Flowjs file upload - AngularJS and Node - node.js

I am using Flowjs, and its ng-flow directive to upload file with NodeJS as backend. When i try to upload file, only file in tem folder is uploaded, but it's note any type of file like JPG or PNG. (flow-135601-juicy_gustinektar02l_10185jpg.1). Here is code:
ANGULARJS
app.config(['flowFactoryProvider', function (flowFactoryProvider) {
flowFactoryProvider.defaults = {
target: 'http://localhost:8086/api/upload/',
permanentErrors: [500, 501],
maxChunkRetries: 1,
chunkRetryInterval: 5000,
simultaneousUploads: 1
};
flowFactoryProvider.on('catchAll', function (event) {
console.log('catchAll', arguments);
});
// Can be used with different implementations of Flow.js
//flowFactoryProvider.factory = fustyFlowFactory;
}]);
NODEJS
// Handle uploads through Flow.js
app.post('/api/upload', function(req, res){
flow.post(req, function(status, filename, original_filename, identifier){
console.log('POST', status, original_filename, identifier);
res.send(200, {
// NOTE: Uncomment this funciton to enable cross-domain request.
//'Access-Control-Allow-Origin': '*'
});
});
});
// Handle cross-domain requests
// NOTE: Uncomment this funciton to enable cross-domain request.
/*
app.options('/upload', function(req, res){
console.log('OPTIONS');
res.send(true, {
'Access-Control-Allow-Origin': '*'
}, 200);
});
*/
// Handle status checks on chunks through Flow.js
app.get('/api/upload', function(req, res){
flow.get(req, function(status, filename, original_filename, identifier){
console.log('GET', status);
res.send(200, (status == 'found' ? 200 : 404));
});
});

Reassembling all chunks is easy, just call this:
var stream = fs.createWriteStream(filename);
r.write(identifier, stream);
And that is it!
But other question is, when this method should be called?
Maybe when all chunks are uploaded and present at tmp folder.
But there is another issue with duplicate calls of the done.
This can be solved by creating and locking the file, once all chunks exists.
Then call
r.write(identifier, stream);
Then clean all chunks, release the lock and close the file.
Same approuch is done in php server side library: https://github.com/flowjs/flow-php-server/blob/master/src/Flow/File.php#L102
Hope this helps, and I hope someone could collaborate and update node.js sample with these fixes.

Related

uploaded file differ with the original

I'm trying to upload a file with node using this simple code:
UpdateController.prototype.uploadUpdateFile = function(req, res)
{
var w = fs.createWriteStream(settings.uploadFolder + settings.updateFile);
req.pipe(w);
w.on('finish', function()
{
res.send(JSON.stringify({
status:0,
filename:settings.uploadFolder + settings.updateFile
}));
}, function()
{
res.send(JSON.stringify({
status:1,
message:"error during file upload, operation failed"
}));
});
}
The file is correctly uploaded but it changes between the original because header (------WebKitForm ... /octet-stream....) and footer (..------WebKitFormBoundary9gOZjMubs9GivcUQ--..) are added to the content.
How to get only the file content ?
You would have to look at the headers of the client request to understand how the client decided to send you the file (how the file was encoded)
You will probably end up using busboy or another package that depends on it : https://www.npmjs.com/package/busboy
such a package will "decode" the data sent by the browser.

node.js server not responding while large file upload with express and busboy

I am developping an file sharing platform with node.js and express.js, using busboy.
It works nicely at this time but uploading large file.
If I do, the server doesn't accept any new request while the large file is uploaded.
I that normal ? If it is, how to improve this behavior, even if that means the upload will take more time...
For now I develop and test on localhost on a pretty good PC (asus i7/8G) on ubuntu.
When I start uploadind a large file, and open a new tab to go to the app, the tab loads only when the upload is completed.
Application loading:
var app = express();
//...
app.use(bodyParser.urlencoded({
extended: true
}));
app.use(bodyParser.json());
app.use(methodOverride());
// Request multipart parsing middleware
app.use(busboy());
// default options, immediately start reading from the request stream and
// parsing
app.use(busboy({ immediate: true }));
My upload method in files controller:
exports.create = function(req, res) {
var _file = new File(req.body);
req.busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
file.on('data', function(data) {
_file.data += data;
});
file.on('end', function() {
_file.save(function(err) {
if (err) {
console.error(err);
return res.status(500).send({
message: errorHandler.getErrorMessage(err)
});
} else {
// doing some stuff on save
}
});
});
});
// req.busboy.on('field', function(key, value, keyTruncated, valueTruncated) {
// console.log('Field [' + key + ']: value: ' + value);
// });
// req.busboy.on('finish', function() {
// console.log('Done parsing form!');
// });
req.pipe(req.busboy);
};
There's at least a few things wrong here:
busboy is being loaded twice. You should remove the app.use(busboy({ immediate: true })); line from your app.js.
Next, the entirety of all files are being buffered in memory (_file.data += data;). You should instead stream the file(s) somewhere, whether that's to disk, to some network storage service like Amazon's S3, or any other place outside of the process.
This one is a moot point really once you switch to streaming, but technically with your current buffering code, you're concatenating multiple files together because the same File object is being used for the entire request. Perhaps that's not an issue if you know for sure only one file will ever be sent (for example, you are always the client), but it is worth noting.

How to stop upload and redirect in busboy if mime type is invalid?

I am fairly new to Node.js, and I am using Express and Busboy-Connect to create a simple file upload form, for wav files only.
Here is what I am trying to do :
- start the upload
- if the mimetype is not wav, redirect to an error page
- else : write the file on the server and redirect back.
If the mimetype is valid, everything works fine, but if it isn't I cannot redirect and the browser is just hanging and eventually times out.
My understanding of it is that the browser doesn't want to redirect because it is waiting for the upload to finish, but how can I cancel the upload then within my js code ?
I could work around the issue and write the file then delete it if it's not the right mimetype, but I think it's a bit stupid to do that, I'd rather find a way to trigger an event that will stop it and redirect immediately.
Here is (a snippet of) my app code :
app.get('/', function (req, res) {
res.render(__dirname + '/public/index.ejs', {error: 0});
});
app.get('/error', function (req, res) {
res.render(__dirname + '/public/index.ejs', {error: 1});
});
app.post('/upload', function (req, res) {
var timestamp = new Date().getTime().toString();
//console.log(timestamp);
var fstream;
req.pipe(req.busboy);
req.busboy.on('file', function (fieldname, file, filename, encoding, mimetype) {
if ("audio/wav" != mimetype)
{
console.log("invalid mimetype"); // that prints ok
// req.busboy.end(); // I tried that but it doesn't work
res.redirect('/error');
}
else
{
console.log("Uploading: " + mimetype);
fstream = fs.createWriteStream(__dirname + '/tmp/' + timestamp + filename);
file.pipe(fstream);
fstream.on('close', function () {
res.redirect('back');
});
}
});
});
Can anyone point me in the right direction?
Thank you for your help !
Alright I found it in the docs of npm, if you think anyone could be interested in finding this answer from a google search you can leave it resolved, otherwise feel free to close/remove this post.
Basically there is a function on the filestream that need to be called to unblock busboy, so all I had to do to make it work is to add
file.resume();
before redirecting to the error page.

mongodb gridfs encoding picture base64

i try to readout an image, saved in mongodb, via gridfs (without temporary file)
it should be directly sent to ajax, which injects it into html
when i use my actual functions a large bit string is formed and sent to client (is saved in ajax response var)
but as it reaches the client, the bits arent correct anymore
so i look for a way to encode the picture before it is sent (into base64)
(or is there any other way?)
Serverside - javascript, gridfs
exports.readFileFromDB = function(req, res, profile, filename, callback){
console.log('Find data from Profile ' + JSON.stringify(profile));
var GridReader = new GridStore(db, filename,"r");
GridReader.open(function(err, gs) {
var streamFile = gs.stream(true);
streamFile.on("end", function(){
});
// Pipe out the data
streamFile.pipe(res);
GridReader.close(function(err, result) {
});
Clientside - javascript ajax call:
function imgUpload(){
var thumb = $("#previewPic");
$('#uploadForm').ajaxSubmit({
beforeSend:function(){
//launchpreloader();
},
error: function(xhr) {
//status('Error: ' + xhr.status);
},
success: function(response) {
console.log(response);
var imageData = $.base64Encode(response);
console.log(imageData);
thumb.attr("src","data:image/png;base64"+imageData);
$("#spanFileName").html("File Uploaded")
}
});
}
I'm doing something similar for a current project, but when the upload is complete, I return a JSON object containing the URL for the uploaded image:
{ success : true, url : '/uploads/GRIDFSID/filename.ext' }
I have a route in Express that handles the /uploads route to retrieve the file from GridFS and stream it back to the client, so I can use the above URL in an IMG SRC. This is effectively what appears in the DOM:
<img src="/uploads/GRIDFSID/filename.ext">
The route handler looks something like this (it uses node-mime and gridfs-stream):
app.get(/^\/uploads\/([a-f0-9]+)\/(.*)$/, function(req, res) {
var id = req.params[0];
var filename = req.params[1];
// Set correct content type.
res.set('Content-type', mime.lookup(filename));
// Find GridFS file by id and pipe it to the response stream.
gridfs
.createReadStream({ _id : id })
.on('error', function(err) {
res.send(404); // or 500
})
.pipe(res);
});
It obviously depends on your exact setup if my solution works for you.

Serving Temporary Files with NodeJs

I am building a NodeJs SOAP client. Originally, I imagined the server (ie the node SOAP client) would allow downloading documents through a REST API (the REST API is authenticated). After a good deal of time on Google and SO, looks like that is not possible.
That means when a document download is requested, I'll have to make a SOAP call for the document and return a URL to the REST client via AJAX.
In order to do this I'll need to:
Temporarily create a file in Node
get its URL and return to web client
When the file is requested and response is sent, delete the file (for security purposes)
Here are my questions:
Is there already a framework that does this? the temp module might be an option, but really I'd like to delete after every request, not after a time period.
If not, can I do this just using the NodeJs File System, and Express static module? Basically we would modify the static module to look like this:
return function static(req, res, next) {
if ('GET' != req.method && 'HEAD' != req.method) return next();
var path = parse(req).pathname;
var pause = utils.pause(req);
/* My Added Code Here */
res.on('end', function(){
// delete file based on req URL
})
/* end additions */
function resume() {
next();
pause.resume();
}
function directory() {
if (!redirect) return resume();
var pathname = url.parse(req.originalUrl).pathname;
res.statusCode = 301;
res.setHeader('Location', pathname + '/');
res.end('Redirecting to ' + utils.escape(pathname) + '/');
}
function error(err) {
if (404 == err.status) return resume();
next(err);
}
send(req, path)
.maxage(options.maxAge || 0)
.root(root)
.hidden(options.hidden)
.on('error', error)
.on('directory', directory)
.pipe(res);
};
Is res.on('end',... vaild? Alternatively,should I create some middleware that does this for URLs pointing to the temporary files?
Found two SO questions that answer my question. So apparently we don't need to use the express.static middleware. We just need the filesystem to download a file:
app.get('/download', function(req, res){
var file = __dirname + '/upload-folder/dramaticpenguin.MOV';
res.download(file); // Set disposition and send it.
});
If we want to stream and then delete follow:
app.get('/download', function(req, res){
var stream = fs.createReadStream('<filepath>/example.pdf', {bufferSize: 64 * 1024})
stream.pipe(res);
var had_error = false;
stream.on('error', function(err){
had_error = true;
});
stream.on('close', function(){
if (!had_error) fs.unlink('<filepath>/example.pdf');
});
If you are visiting this SO page after Dec 2015, you may find that the previous solution isn't working (At least it isn't working for me). I found a different solution so I thought I would provide it here for future readers.
app.get('/download', function(req, res){
res.download(pathToFile, 'fileNameForEndUser.pdf', function(err) {
if (!err) {
fs.unlink(path);
}
});
});

Resources