NodeJS - Formidable 200 MB file limit and crash - node.js

I have nodejs server:
var http = require('http');
var formidable = require('formidable');
var fs = require('fs');
http.createServer(function (req, res) {
if (req.url == '/fileupload') {
console.log(req.rawHeaders)
var form = new formidable.IncomingForm();
form.parse(req, function (err, fields, files) {
if (err) throw err;
var oldpath = files.file.path
var newpath = 'C:/VSI/' + files.file.name;
fs.rename(oldpath, newpath, function (err) {
//if (err) throw err;
//res.write('File uploaded and moved!');
if (err) {
res.write(JSON.stringify({
success: false,
message: 'error happend'
})
);
res.end()
throw err;
}
res.write(JSON.stringify({
success: true,
message: 'Successfully Uploaded'
})
);
res.end();
});
});
} else {
console.log(req.rawHeaders)
res.writeHead(200, { 'Content-Type': 'text/html' });
res.write('<form action="fileupload" method="post" enctype="multipart/form-data">');
res.write('<input type="file" name="filetoupload"><br>');
res.write('<input type="submit">');
res.write('</form>');
return res.end();
}
}).listen(8080, () => {
console.log('server running')
}).on("error", () => console.log("error happend"));
and get
if (err) throw err;
^
Error: maxFileSize exceeded, received 209715683 bytes of file data
How I can upload more 200 MB with formidable?
How properly handle exceptions and why "on error" on("error", () =>) event not work"?
I tried Multer and it doesn't have 200 MB limit and work normally.

Just read the documentation:
https://www.npmjs.com/package/formidable
Default max size is 200MB, change it with (300MB for example) :
form.maxFileSize = 300 * 1024 * 1024;

Related

Requesting file directly with Node JS server returns with nothing

So I am trying to get a node js server to serve a file. Even when I call the file directly in the url (http://localhost:8080/media/file.mp3) it returns no data. When I console.log on the write statement, it returns false, meaning that some or all the of the data isn't being flushed out and sent to the client.
The code:
http.createServer(function(req, res){
res.on('end',()<= {
if (req.url.includes("media")){
res.setHeader('Content-Type','audio/mp3');
res.setHeader('Retry-After', '1');
res.setHeader('method','POST');
res.setHeader('Connection','keep-alive');
res.writeHead(200,{'Content-Type':'audio/mp3'});
console.log("/path_to_file_from_root_to_index"+req.url);
fs.readFile("/path_to_file_from_root_to_index"+req.url,function(err,data){
if (err){
console.log(err);
} else {
res.write(data);
res.end();
}
});
}).listen(8080);
What is being shown in the browser when req.url='/media/file.mp3':
var fs = require('fs'),
http = require('http'),
path = require('path');
http.createServer(function(req, res) {
if (req.url.includes('media')) {
res.writeHead(200, {
'Content-Type': 'audio/mp3'
});
console.log(path.join(__dirname, '/path_to_file_from_root_to_index' + req.url));
fs.readFile(path.join(__dirname, '/path_to_file_from_root_to_index' + req.url), function(err, data) {
if (err) {
console.log(err);
} else {
res.end(data);
}
});
}
}).listen(8080);

cannot read property name of undefined

I'm new to Node.js. I started to learn from yesterday.
I have a form with simple file upload and submit button
JS file
var http = require('http');
var fs = require('fs');
var formidable = require('formidable');
http.createServer(function(req, res){
if (req.url == '/fileupload'){
var form = new formidable.IncomingForm();
form.parse(req, function (err, files, fields){
var oldpath = files.filetoupload.path;
console.log(oldpath);
var newpath = 'F:/' + files.filestoupload.name;
console.log('Destination set!');
fs.readFile(oldpath, function (err, data){
if (err) throw err;
console.log('File read!');
fs.writeFile(newpath, data, function(err){
if (err) throw err;
res.write('Package shipped and moved');
res.end();
console.log('Newpath is' + newpath);
});
fs.unlink(oldpath, function (err){
if (err) throw err;
console.log("Temp file deleted");
});
});
});
} else {
res.writeHead(200, {'Content-Type': 'text/html'});
res.write('<form action="fileupload" method="post">');
res.write('<input type="file" name="filetoupload"><br><br>');
res.write('<input type="submit" name="submit" value="submit">');
res.write('</form>');
return res.end();
}
}).listen(8080);
It somehow runs on the browser, but when I hit upload button an error pops up in the command prompt
TypeError: Cannot read property 'name' of undefined
at F:\Prince\workspace\NODE JS\formidable.js:12:48
at IncomingForm.<anonymous> (F:\Prince\workspace\NODE JS\node_modules\formidable\lib\incoming_form.js:107:9)
at IncomingForm.emit (events.js:182:13)
at IncomingForm._maybeEnd (F:\Prince\workspace\NODE JS\node_modules\formidable\lib\incoming_form.js:557:8)
at QuerystringParser.parser.onEnd (F:\Prince\workspace\NODE JS\node_modules\formidable\lib\incoming_form.js:458:10)
at QuerystringParser.end (F:\Prince\workspace\NODE JS\node_modules\formidable\lib\querystring_parser.js:25:8)
at IncomingMessage.<anonymous> (F:\Prince\workspace\NODE JS\node_modules\formidable\lib\incoming_form.js:132:30)
at IncomingMessage.emit (events.js:182:13)
at endReadableNT (_stream_readable.js:1098:12)
at process.internalTickCallback (internal/process/next_tick.js:72:19)
I'll appreciate if anyone clarifies my mistake.
Ps: read comments too.
After debugging for a while I found the issues here.
There are three mistakes here.
As #Jordan S said there is an typo in line:12
filestoupload should be filestoupload
Then Incorrect callback format in form parse in line:10
form.parse(req, function (err, files, fields) to form.parse(req, function (err, fields, files)
And the final one is in the form enctype is missing
res.write('<form action="fileupload" method="post" enctype="multipart/form-data">');
adding enctype="multipart/form-data" fixed the final issue.
But I still don't know what is the point of adding enctype
Updated JS File
var http = require('http');
var fs = require('fs');
var formidable = require('formidable');
http.createServer(function(req, res){
if (req.url == '/fileupload'){
var form = new formidable.IncomingForm();
form.parse(req, function (err, fields, files){
var oldpath = files.filetoupload.path;
console.log(oldpath);
var newpath = 'F:/' + files.filetoupload.name;
console.log('Destination set!');
fs.readFile(oldpath, function (err, data){
if (err) throw err;
console.log('File read!');
fs.writeFile(newpath, data, function(err){
if (err) throw err;
res.write('Package shipped and moved');
res.end();
console.log('Newpath is' + newpath);
});
fs.unlink(oldpath, function (err){
if (err) throw err;
console.log("Temp file deleted");
});
});
});
} else {
res.writeHead(200, {'Content-Type': 'text/html'});
res.write('<form action="fileupload" method="post" enctype="multipart/form-data">');
res.write('<input type="file" name="filetoupload"><br><br>');
res.write('<input type="submit" name="submit" value="submit">');
res.write('</form>');
return res.end();
}
}).listen(8080);

node.js formidable make variable with uploaded file URL

Hi Im using node and formidable to submit a file in a form, this file's URL I need to save in a global variable to later use it with WATSON IBM image recognition api.
I'm new with node so I'm stuck, the variable name is newpath, I'm able to print it after the form is submitted, but can't access the variable later.
I must be doing something wrong, I really appreciate if you can point me out my mistake.
const http = require('http');
var formidable = require('formidable');
const hostname = '127.0.0.1';
const port = 3500;
var fs = require('fs');
/// WATSON
var VisualRecognitionV3 = require('watson-developer-cloud/visual-recognition/v3');
var visualRecognition = new VisualRecognitionV3({
version: '2018-03-19',
iam_apikey: 'xxxxxxx'
});
// SERVER AND FORM
const server = http.createServer((req, res) => {
if (req.url == '/fileupload') {
var form = new formidable.IncomingForm();
form.parse(req, function (err, fields, files) {
var oldpath = files.filetoupload.path;
var newpath = '/users/myuser/coding/visualr/' + files.filetoupload.name;
fs.rename(oldpath, newpath, function (err) {
if (err) throw err;
res.write('File uploaded and moved!');
// this is the path, variable newpath, but can't be accessed
// outside this function, tried to make it global but didn't work either
res.write('newpath');
res.end();
});
});
} else {
res.writeHead(200, {'Content-Type': 'text/html'});
res.write('<form action="fileupload" method="post" enctype="multipart/form-data">');
res.write('<input type="file" name="filetoupload"><br>');
res.write('<input type="submit">');
res.write('</form>');
return res.end();
}
});
var images_file = fs.createReadStream(newpath);
// I want to put the variable newpath in this function: but it doesn't work...
var params = {
images_file: images_file,
};
visualRecognition.classify(params, function(err, response) {
if (err)
console.log(err);
else
console.log(JSON.stringify(response, null, 2))
});
// ENDS
server.listen(port, hostname, () => {
console.log(`Server running at http://${hostname}:${port}/`);
});
The variable is defined within the context of the if (req.url == '/fileupload') {...} block, so it won't be available outside of that block.
To use the variable everywhere in the code, define it outside of the createServer block:
var newpath; // this variable will have global context for this file
const server = http.createServer((req, res) => {
if (req.url == '/fileupload') {
var form = new formidable.IncomingForm();
form.parse(req, function (err, fields, files) {
var oldpath = files.filetoupload.path;
// set the variable to its intended value here
newpath = '/users/myuser/coding/visualr/' + files.filetoupload.name;
fs.rename(oldpath, newpath, function (err) {
if (err) throw err;
res.write('File uploaded and moved!');
res.write('newpath');
res.end();
});
});
} else {
res.writeHead(200, {'Content-Type': 'text/html'});
res.write('<form action="fileupload" method="post" enctype="multipart/form-data">');
res.write('<input type="file" name="filetoupload"><br>');
res.write('<input type="submit">');
res.write('</form>');
return res.end();
}
});
console.log(newpath); // the variable should be available here, because of its context

How to show ldapatuh node jpegPhoto?

I use ldapauth-fork in node.js to take a user information in ldap but when I try to take a user.jpegPhoto why can't it show this image?
I've tried this:
const jpeg = Buffer.from(user.jpegPhoto, 'binary')
fs.writeFileSync("foto.jpeg", jpeg);
And this:
var img64 = Buffer.from(user.jpegPhoto).toString('base64');
var foto = "<div><img src='data:image/jpeg;base64,"+img64+"'/></div>"
fs.appendFile('index.html', foto, function (err) {
if (err) throw err;
console.log('Saved!');
});
This is all node.js code:
var basicAuth = require('basic-auth');
var LdapAuth = require('ldapauth-fork');
var http = require('http');
var fs = require('fs');
var qs = require('querystring');
var username = "MOCKUSERNAME";
var password = "MOCKPASSWORD";
server = http.createServer( function(req, res) {
if (req.method == 'POST') {
console.log("POST");
var body = '';
req.on('data', function (data) {
body += data;
//console.log(body);
});
req.on('end', function () {
var data = qs.parse(body);
var userDecode = Buffer.from(data.username, 'base64').toString('ascii');
var passDecode = Buffer.from(data.password, 'base64').toString('ascii');
res.setHeader('Access-Control-Allow-Origin', '*');
res.writeHead(200, {'Content-Type':'image/jpeg'});
if (data.username && data.password) {
var options = {
url: 'XXXXXX',
bindDN: 'XXXXX',
bindCredentials: 'XXXXX',
searchBase: 'XXXXX',
includeRaw: true,
searchFilter: '(aoldapkey=*' + userDecode + ')'
// reconnect: true
}
var ldap = new LdapAuth(options);
ldap.authenticate(userDecode, passDecode, function(err, user) {
if (err) {
console.log("login Error: " + err);
res.write(JSON.stringify({ success : false, message : 'Authentication failed', serverError: err }));
res.end('');
} else if(!user.uid) {
console.log("user not found Error: " + err);
res.write(JSON.stringify({ success : false, message :
'Authentication failed, user not found', serverError: err }));
res.end();
} else if(user.uid) {
console.log("success : user "+ user.uid +" found ");
var userJsonObject = {
success : true,
user: {
uid : user.uid,
cn: user.cn,
givenName: user.givenName,
sn: user.sn,
title: user.title,
mobile: user.mobile,
mail: user.mail,
aoJobPosition: user.aoJobPosition,
jpegPhoto: user.jpegPhoto
}
}
//------------------------------dont see the image in index.html
/*var img64 = Buffer.from(user.jpegPhoto).toString('base64');
var foto = "<div><img src='data:image/jpeg;base64,"+img64+"'/>
</div>"
fs.appendFile('index.html', foto, function (err) {
if (err) throw err;
console.log('Saved!');
});*/
//--------------------------------
res.write(JSON.stringify(userJsonObject);
res.end();
}
});
} else {
res.write(JSON.stringify({ success : false, message : 'You must
specify a valid username and password'}));
res.end();
}
});
}
else {
var html = fs.readFileSync('index.html');
res.writeHead(200, {'Content-Type': 'text/html'});
res.end(html);
}
});
server.on('error', function(err) {
console.log("Caught server error: ");
console.log(err.stack);
});
port = 3000;
host = '127.0.0.1';
server.listen(port, host);
console.log('Listening at http://' + host + ':' + port);
This is all node.js code I use ldapauth-fork in node.js to take a user information in ldap but when i try to take a user.jpegPhoto y canĀ“t show this image.
This is index.html :
<div><img src='data:imag/*;base64, 77x73..........3vv70='/></div>
index.html

Error: EXDEV: cross-device link not permitted, rename '/tmp/ on Ubuntu 16.04 LTS

I have problem upload file nodejs. i have read
stack1
and
stack2. but i think this different case
on terminal output like this :
> Unhandled rejection Error: EXDEV: cross-device link not permitted,
> rename '/tmp/31941-53enly.png' ->
> 'public/files/complaint/Screenshot_2016-05-01_01-16-55.png'
> at Error (native)
> at Object.fs.renameSync (fs.js:681:18)
> at null.<anonymous> (/home/faris/myprojects/orchestrale-server/routes/complaintimage.js:163:20)
> at tryCatcher (/home/faris/myprojects/orchestrale-server/node_modules/sequelize/node_modules/bluebird/js/release/util.js:16:23)
> at Promise._settlePromiseFromHandler (/home/faris/myprojects/orchestrale-server/node_modules/sequelize/node_modules/bluebird/js/release/promise.js:502:31)
> at Promise._settlePromise (/home/faris/myprojects/orchestrale-server/node_modules/sequelize/node_modules/bluebird/js/release/promise.js:559:18)
> at Promise._settlePromise0 (/home/faris/myprojects/orchestrale-server/node_modules/sequelize/node_modules/bluebird/js/release/promise.js:604:10)
> at Promise._settlePromises (/home/faris/myprojects/orchestrale-server/node_modules/sequelize/node_modules/bluebird/js/release/promise.js:683:18)
> at Async._drainQueue (/home/faris/myprojects/orchestrale-server/node_modules/sequelize/node_modules/bluebird/js/release/async.js:138:16)
> at Async._drainQueues (/home/faris/myprojects/orchestrale-server/node_modules/sequelize/node_modules/bluebird/js/release/async.js:148:10)
> at Immediate.Async.drainQueues [as _onImmediate] (/home/faris/myprojects/orchestrale-server/node_modules/sequelize/node_modules/bluebird/js/release/async.js:17:14)
> at processImmediate [as _immediateCallback] (timers.js:383:17)
my code :
if (_.isEmpty(req.files) == false) {
var tp = avatar_folder+req.files.file.name;
fs.renameSync(req.files.file.path, tp, function(err){});
var avatar = req.files.file.name;
}
code work on another distro like elementary os, but when i run project on debian jessie or ubuntu 16 LTS that error result on terminal.
any body can't help ? that code just run perfectly on elementary os and arch.
Same problem. Workaround: Read the temp file, write the file in new location and remove the temp file:
// Read the file
fs.readFile(oldpath, function (err, data) {
if (err) throw err;
console.log('File read!');
// Write the file
fs.writeFile(newpath, data, function (err) {
if (err) throw err;
res.write('File uploaded and moved!');
res.end();
console.log('File written!');
});
// Delete the file
fs.unlink(oldpath, function (err) {
if (err) throw err;
console.log('File deleted!');
});
});
You can use 'mv' packgage to resolve this issue
Link NPM: https://www.npmjs.com/package/mv
How to use:
Replace some old code with fs.rename:
fs.rename(temp_path, target_path, (err) => {
if (err) {
console.log('> FileServer.jsx | route: "/files/upload" | err:', err);
throw err;
}
});
With:
const mv = require('mv');
mv(temp_path, target_path, function (err) {
if (err) {
console.log('> FileServer.jsx | route: "/files/upload" | err:', err);
throw err;
}
});
it assisted me when working with nodejs , the concept should be the same https://stackoverflow.com/a/43206506/7878987
var http = require('http');
var formidable = require('formidable');
var fs = require('fs');
var mv = require('mv');
http.createServer(function (req, res) {
if (req.url == '/fileupload') {
var form = new formidable.IncomingForm();
form.parse(req, function (err, fields, files) {
var oldpath = files.filetoupload.path;
console.log(oldpath);
var newpath = 'F:/change path to your project dir/' + files.filetoupload.name;
mv(oldpath, newpath, function (err) {
if (err) throw err;
res.write('File uploaded and moved!');
res.end();
});
});
} else {
res.writeHead(200, {'Content-Type': 'text/html'});
res.write('<form action="fileupload" method="post" enctype="multipart/form-data">');
res.write('<input type="file" name="filetoupload"><br>');
res.write('<input type="submit">');
res.write('</form>');
return res.end();
}enter code here
}).listen(8080);
In order to resolve this problem, you can rewrite as:
fs.writeFile(path_where_to_write, file, function(err) {
if (err) throw err; /*do something else.*/
});
/*and then Remove the file from tmp location*/
fs.unlink(tempPath);
and for multiple files, you could iterate this block based on length of files.
var http = require('http');
var formidable = require('formidable');
var fs = require('fs');
http.createServer(function (req, res) {
if (req.url == '/fileupload') {
var form = new formidable.IncomingForm();
form.parse(req, function (err, fields, files) {
var oldpath = files.filetoupload.path;
console.log(oldpath);
var newpath = 'F:/your folder name/' + files.filetoupload.name;
fs.readFile(oldpath, function (err, data) {
if (err) throw err;
console.log('File read!');
// Write the file
fs.writeFile(newpath, data, function (err) {
if (err) throw err;
res.write('File uploaded and moved!');
res.end();
console.log('File written!');
});
// Delete the file
fs.unlink(oldpath, function (err) {
if (err) throw err;
console.log('File deleted!');
});
});
});
} else {
res.writeHead(200, {'Content-Type': 'text/html'});
res.write('<form action="fileupload" method="post" enctype="multipart/form-
data">');
res.write('<input type="file" name="filetoupload"><br>');
res.write('<input type="submit">');
res.write('</form>');
return res.end();
}
}).listen(8080);
You have to use fs.writeFileSync
otherwise oldfile can be deleted before copied
fs.writeFileSync(newpath, data, function (err) {
if (err) throw err;
res.write('File uploaded and moved!');
res.end();
console.log('File written!');
});
// Delete the file
fs.unlink(oldpath, function (err) {
if (err) throw err;
console.log('File deleted!');
});
This issue occurs when your temp folder is in a different drive and Node code is in a different drive.
It will throw this error.
Solution is in another StackOverflow question .
What does the "EXDEV: cross-device link not permitted" error mean?
solution from this post :
As per the new NodeJS, the solution is don`t move, copy it
fs.copyFileSync(oldpath, newpath);
fs.unlink(oldpath,()=>{});
My windows 11 is in SSD and my Codes are in HDD. The uploaded image will be downloaded in windows drive and I am trying to move it to HDD.
In my case, Its working within same disc(drive) path means within c:/ , Error occurred if new path disk drive is different from temp path disk drive.
yes, you can resole this problem as.
install mv by command
npm install mv --save
add mv in your nodejs file
var mv = require('mv');
move file as:
mv('source_file', 'destination_file,' function(err) {
if (err) { throw err; }
console.log('file moved successfully');
};

Resources