Uploading file using POST request in Node.js - node.js

I have problem uploading file using POST request in Node.js. I have to use request module to accomplish that (no external npms). Server needs it to be multipart request with the file field containing file's data. What seems to be easy it's pretty hard to do in Node.js without using any external module.
I've tried using this example but without success:
request.post({
uri: url,
method: 'POST',
multipart: [{
body: '<FILE_DATA>'
}]
}, function (err, resp, body) {
if (err) {
console.log('Error!');
} else {
console.log('URL: ' + body);
}
});

Looks like you're already using request module.
in this case all you need to post multipart/form-data is to use its form feature:
var req = request.post(url, function (err, resp, body) {
if (err) {
console.log('Error!');
} else {
console.log('URL: ' + body);
}
});
var form = req.form();
form.append('file', '<FILE_DATA>', {
filename: 'myfile.txt',
contentType: 'text/plain'
});
but if you want to post some existing file from your file system, then you may simply pass it as a readable stream:
form.append('file', fs.createReadStream(filepath));
request will extract all related metadata by itself.
For more information on posting multipart/form-data see node-form-data module, which is internally used by request.

An undocumented feature of the formData field that request implements is the ability to pass options to the form-data module it uses:
request({
url: 'http://example.com',
method: 'POST',
formData: {
'regularField': 'someValue',
'regularFile': someFileStream,
'customBufferFile': {
value: fileBufferData,
options: {
filename: 'myfile.bin'
}
}
}
}, handleResponse);
This is useful if you need to avoid calling requestObj.form() but need to upload a buffer as a file. The form-data module also accepts contentType (the MIME type) and knownLength options.
This change was added in October 2014 (so 2 months after this question was asked), so it should be safe to use now (in 2017+). This equates to version v2.46.0 or above of request.

Leonid Beschastny's answer works but I also had to convert ArrayBuffer to Buffer that is used in the Node's request module. After uploading file to the server I had it in the same format that comes from the HTML5 FileAPI (I'm using Meteor). Full code below - maybe it will be helpful for others.
function toBuffer(ab) {
var buffer = new Buffer(ab.byteLength);
var view = new Uint8Array(ab);
for (var i = 0; i < buffer.length; ++i) {
buffer[i] = view[i];
}
return buffer;
}
var req = request.post(url, function (err, resp, body) {
if (err) {
console.log('Error!');
} else {
console.log('URL: ' + body);
}
});
var form = req.form();
form.append('file', toBuffer(file.data), {
filename: file.name,
contentType: file.type
});

You can also use the "custom options" support from the request library. This format allows you to create a multi-part form upload, but with a combined entry for both the file and extra form information, like filename or content-type. I have found that some libraries expect to receive file uploads using this format, specifically libraries like multer.
This approach is officially documented in the forms section of the request docs - https://github.com/request/request#forms
//toUpload is the name of the input file: <input type="file" name="toUpload">
let fileToUpload = req.file;
let formData = {
toUpload: {
value: fs.createReadStream(path.join(__dirname, '..', '..','upload', fileToUpload.filename)),
options: {
filename: fileToUpload.originalname,
contentType: fileToUpload.mimeType
}
}
};
let options = {
url: url,
method: 'POST',
formData: formData
}
request(options, function (err, resp, body) {
if (err)
cb(err);
if (!err && resp.statusCode == 200) {
cb(null, body);
}
});

I did it like this:
// Open file as a readable stream
const fileStream = fs.createReadStream('./my-file.ext');
const form = new FormData();
// Pass file stream directly to form
form.append('my file', fileStream, 'my-file.ext');

const remoteReq = request({
method: 'POST',
uri: 'http://host.com/api/upload',
headers: {
'Authorization': 'Bearer ' + req.query.token,
'Content-Type': req.headers['content-type'] || 'multipart/form-data;'
}
})
req.pipe(remoteReq);
remoteReq.pipe(res);

Related

Autodesk Forge - Reality Capture Issue - Specified Photoscene ID doesn't exist in the database

i'm trying to upload my files as form-data, after i've created a scene. But I receive always the error "Specified Photoscene ID doesn't exist in the database" (which were created directly before).
My upload function:
// Upload Files
async function uploadFiles(access_Token, photoSceneId, files) {
try {
const params = new URLSearchParams({
'photosceneid': photoSceneId,
'type': 'image',
'file': files
})
const headers = Object.assign({
Authorization: 'Bearer ' + access_Token,
'Content-Type': 'multipart/form-data' },
files.getHeaders()
)
let resp = await axios({
method: 'POST',
url: 'https://developer.api.autodesk.com/photo-to-3d/v1/file',
headers: headers,
data: params
})
let data = resp.data;
return data;
} catch (e) {
console.log(e);
}
};
I've also tried a few varaints, e.g. adding the photosceneId to the form data (form.append(..), but doesn't works either.
Any helpful suggestion are appreciated. Thx in advance.
There might be two problems here.
First, I am not sure of it, as I don't have experience of URLSearchParams as a "packer" for POST requests. This might be the reason why you get "Specified Photoscene ID doesn't exist in the database" error - perhaps the way the data are serialized using URLSearchParams is not compatible.
The second problem, I am sure of it, is regarding the way you submit the files.
According to documentation, you have to pass the files one by one, like
"file[0]=http://www.autodesk.com/_MG_9026.jpg" \
"file[1]=http://www.autodesk.com/_MG_9027.jpg"
and not just passing an array to the "file" field.
Having this said, try this approach:
var axios = require('axios');
var FormData = require('form-data');
var fs = require('fs');
var data = new FormData();
var TOKEN = 'some TOKEN';
const photoSceneID = 'some_photoscene_id';
data.append('photosceneid', photoSceneID);
data.append('type', 'image');
data.append('file[0]', fs.createReadStream('/C:/TEMP/Example/DSC_5427.JPG'));
data.append('file[1]', fs.createReadStream('/C:/TEMP/Example/DSC_5428.JPG'));
data.append('file[2]', fs.createReadStream('... and so on ...'));
var config = {
method: 'post',
url: 'https://developer.api.autodesk.com/photo-to-3d/v1/file',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Authorization': 'Bearer ' + TOKEN,
},
data : data
};
axios(config)
.then(function (response) {
console.log(JSON.stringify(response.data));
})
.catch(function (error) {
console.log(error);
});
Also, I always recommend instead of jumping right into the code, to check first the workflow using apps like Postman or Insomnia and then, after you validated the workflow (created the photoscene, all images were properly uploaded and so on), you can translate this into the code.
At the end of this blogpost you will find the link to alrady created Postman collection, but I highly recommend building your own collection, as part of the learning step.
This is the solution that worked for me. Please note that the upload should be limited by a maximum of 20 files per call.
// Upload Files
async function uploadFiles(access_Token, photoSceneId) {
try {
let dataPath = path.join(__dirname, '../../data')
let files = fs.readdirSync(dataPath)
var data = new FormData();
data.append('photosceneid', photoSceneId)
data.append('type', 'image')
for(let i=0; i < files.length; i++) {
let filePath = path.join(dataPath, files[i])
let fileName = 'file[' + i + ']'
data.append(fileName, fs.createReadStream(filePath))
}
const headers = Object.assign({
Authorization: 'Bearer ' + access_Token,
'Content-Type': 'multipart/form-data;boundary=' + data.getBoundary()},
data.getHeaders()
)
let resp = await axios({
method: 'POST',
url: 'https://developer.api.autodesk.com/photo-to-3d/v1/file',
headers: headers,
maxContentLength: Infinity,
maxBodyLength: Infinity,
data: data
})
let dataResp = resp.data;
return dataResp;
} catch (e) {
console.log(e);
}
};

Using request module and passing the body to the next request

I am using request to get an image:
request(req.body.imageUrl, {encoding: null}, function(error, response, body) {
I then want to use the body to pass to an api that uses a multipart form and send that image (which is now in the body). I don't want to write the file to disk and then readstream from the disk again. I basically want to enter the formData of the next request by using the Buffer of the body, but it is not working.
So, for the options in the next request I have:
const options = {
method: "POST",
url: coreURL,
headers: {
"Content-Type": "multipart/form-data"
},
formData : {
file : new Buffer.from(body,'binary')
}
};
And this does not work, if I write the body to a file fs.writeFileSync(fileName, body, 'binary');
And then read in the options formData : { file : fs.createReadStream(fileName)}
it works but I cannot use the disk, so need an alternative way to pass the body into the next post as multipart form data.
Any ideas?
POC:
let request = require('request');
request.post({
url: "http://httpbin.org/anything",
formData: {
myfile: request.get("https://www.gravatar.com/avatar/f056d36f9e273fd4740ec8a5cea1348a"),
}
},
function (err, req, body) {
console.log(body);
}
);

how to convert image raw data to image

I want to show user's document file which is return by spring boot api in nodejs
but the file is not showing.
Here is my code of spring boot api
httpServletResponse.setHeader("Content-Disposition", "inline; filename=" + storeDocName + "");
httpServletResponse.setContentType(fileType);
httpServletResponse.setContentLengthLong(Files.readAllBytes(file.toPath()).length);
httpServletResponse.getOutputStream().write(Files.readAllBytes(file.toPath()));
Here is my node js code
requestMethodGetForImageData: function(url, form, header) {
return new Promise((resolve, reject) => {
//SET ALL THESE PARATMETER TO MAKE REQUEST
request.get({ url: url, qs: form, headers: header }, function(error, response, body) {
var result = {
body: response.body,
header: response.headers['content-type']
};
resolve(result);
}
});
});
},
I am using request module of nodejs to send the request to spring api and in nodejs here, I am calling this request function
router.get("/loadStoreDocument", function(req, res) {
var url = httpProtocol + httpServer + servicePort + serviceUrl;
//SET HEADER
headers = {
'Authorization': "token"
};
//SET FORM DATA
form = {
account_id: "id",
token_type: "token_type",
document_child_id: "id"
};
requestHandling.requestMethodGetForImageData(url, form, headers)
.then((result) => {
res.header("content-type", result.header);
res.send(result.body);
}).catch((err) => {
console.log("ERROR IN GET STORE PROFILE");
console.log(err);
res.send(err);
});
});
But when I call this URL(loadStoreDocument) then it returns image like this
enter image description here
Response of spring api is here(image data)
enter image description here
Please help me, I search this everywhere but couldn't find any solution.
You need to set encoding: null in request.get() options.
request.get({ url: url, qs: form, headers: header, encoding: null })
From request docs:
encoding - encoding to be used on setEncoding of response data. If
null, the body is returned as a Buffer. Anything else (including the
default value of undefined) will be passed as the encoding parameter
to toString() (meaning this is effectively utf8 by default). (Note: if
you expect binary data, you should set encoding: null.)

How to post data using node-http-ntlm?

I am trying to figure out how to actually post data using this node module:
https://github.com/SamDecrock/node-http-ntlm
It looks like posting should be similar to:
https://github.com/SamDecrock/node-httpreq#post
But the documentation for httpreq doesnt actually show POSTing a value, I only see parameters or how to POST an entire file. Im using node and have something along these lines:
NodeClient.prototype.create = function (xml) {
var options = {
url: this.url,
username: this.user,
password: this.pw,
domain: this.domain,
headers: {
'Content-type': 'text/plain'
}
};
return new Promise(function (resolve, reject) {
httpntlm.post(options,
function (err, resp) {
if(err) {
reject(err);
}
resolve(resp.body);
});
});
};
Obviously I never send my xml object, so I need to figure out how to include this. Reading the documentation hasnt lead me anywhere to this point.
You can POST xml like this:
var httpntlm = require('httpntlm');
var xml = '<?xml version="1.0" encoding="UTF-8"?>'; // replace this with your xml
httpntlm.post({
url: "https://someurl.com",
username: 'm$',
password: 'stinks',
workstation: 'choose.something',
domain: '',
body: xml,
headers: { 'Content-Type': 'text/xml' }
}, function (err, res){
if(err) return err;
console.log(res.headers);
console.log(res.body);
});
To add content to the post, you can include the following options:
json: if you want to send json directly (content-type is set to application/json)
files: an object of files to upload (content-type is set to multipart/form-data; boundary=xxx)
body: custom body content you want to send. If used, previous options will be ignored and your custom body will be sent. (content-type will not be set)

NodeJS: sending/uploading a local file to a remote server

I have used the Winston module to create a daily log file for my offline app. I now need to be able to send or upload that file to a remote server via POST (that part already exists)
I know I need to write the file in chunks so it doesn't hog the memory so I'm using fs.createReadStream however I seem to only get a 503 response, even if sending just sample text.
EDIT
I worked out that the receiver was expecting the data to be named 'data'. I have removed the createReadSteam as I could only get it to work with 'application/x-www-form-urlencoded' and a synchronous fs.readFileSync. If I change this to 'multipart/form-data' on the php server would I be able to use createReadStream again, or is that only if I change to physically uploading the json file.
I've only been learning node for the past couple of weeks so any pointers would be gratefully received.
var http = require('http'),
fs = require('fs');
var post_options = {
host: 'logger.mysite.co.uk',
path: '/',
port: 80,
timeout: 120000,
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded'
}
}
var sender = http.request(post_options, function(res) {
if (res.statusCode < 399) {
var text = ""
res.on('data', function(chunk) {
text += chunk
})
res.on('end', function(data) {
console.log(text)
})
} else {
console.log("ERROR", res.statusCode)
}
})
var POST_DATA = 'data={['
POST_DATA += fs.readFileSync('./path/file.log').toString().replace(/\,+$/,'')
POST_DATA += ']}'
console.log(POST_DATA)
sender.write(POST_DATA)
sender.end()
After gazillion of trial-failure this worked for me. Using FormData with node-fetch. Oh, and request deprecated two days ago, btw.
const FormData = require('form-data');
const fetch = require('node-fetch');
function uploadImage(imageBuffer) {
const form = new FormData();
form.append('file', imageBuffer, {
contentType: 'image/jpeg',
filename: 'dummy.jpg',
});
return fetch(`myserver.cz/upload`, { method: 'POST', body: form })
};
In place of imageBuffer there can be numerous things. I had a buffer containing the image, but you can also pass the result of fs.createReadStream('/foo/bar.jpg') to upload a file from drive.
copied from https://github.com/mikeal/request#forms
var r = request.post('http://service.com/upload', function optionalCallback (err, httpResponse, body) {
if (err) {
return console.error('upload failed:', err);
}
console.log('Upload successful! Server responded with:', body);
})
var form = r.form()
form.append('my_field1', 'my_value23_321')
form.append('my_field2', '123123sdas')
form.append('my_file', fs.createReadStream(path.join(__dirname, 'doodle.png')))
Have a look at the request module.
It will provide you the ability to stream a file to POST requests.

Resources