POST file using request, gridfs-stream and multer - node.js

In my node.js-application, I need to read data from a MongoDB with GridFS and upload it to another server via POST request, I am using the modules gridfs-stream, request and multer.
According to the request docs I can simply add a stream for multipart/form-data, so I'm doing this:
var fileUuid = "myFilename";
var options = {
url: "url.to/my/target/",
method: "POST",
header: {"Content-Type": "multipart/form-data"},
formData: {
filedata: gfs.createReadStream({filename: fileUuid})
}
}
request(options, function(error, response, body) {
if (error) {
console.log("[" + getDateTime() + "] Error replicating file:");
console.log(error);
return;
}
console.log('statusCode: ', response && response.statusCode);
console.log('body: ', body);
});
On my receiving server, I have the following code:
var upload = multer({ dest: './tmp_upload/' })
app.post('/my/target', upload.single('filedata'), function(req, res) {
console.log('['+getDateTime()+'] Request received');
}
But whenever I am executing the request, I get the following error on my sending server:
[2019:07:26:16:52:00] Error replicating file:
Error: write ECONNRESET
and this on my receiving server:
Error: Unexpected end of multipart data
at c:\Users\...\node_modules\dicer\lib\Dicer.js:62:28
at process._tickCallback (internal/process/next_tick.js:61:11)
Can anybody please give me a hint how to POST my file from one server to another?
EDIT:
The issue seems to be the gfs.createReadStream({filename: fileUuid}) type of stream... When I write the file to the filesystem first and instead put an fs.createReadStream(...) into the form as formData, then it works.
But is there an option to direct the stream directly into the formData, without having to write it to disk first?

So I ended up working with a temporary file that I read from the database, write it to disk and create a new readStream that I then put into the request. After the request was successful, I delete the temporary file from my disk and continue with my code:
var fsstreamwrite = fs.createWriteStream("./tmp/" + myFile);
var readstream = gfs.createReadStream( {filename: myFile} );
readstream.pipe(fsstreamwrite);
readstream.on("close", function () {
console.log("File read successfully from database");
var options = {
url: "url.to/my/target/",
method: "POST",
enctype: "multipart/form-data",
formData: {
"filedata": fs.createReadStream('./tmp/' + myFile)
}
}
request(options, function(error, response, body) {
if (error) {
// error handling
return;
}
// delete temporary file
fs.unlinkSync('./tmp/' + myFile)
// continue with my stuff...
});
});

Related

How to read and save pdf file from response in Node JS client

I have created a Node JS server that pushes the PDF to client as below
app.post("/home", function(req, res, next) {
// to download the file automatically
/*
var file = fs.createReadStream('E:\\test\\conversion\\310-output.pdf');
var stat = fs.statSync('E:\\test\\conversion\\310-output.pdf');
res.setHeader('Content-Length', stat.size);
res.setHeader('Content-Type', 'application/pdf');
res.setHeader('Content-Disposition', 'attachment; filename=output.pdf');
file.pipe(res);
*/
// To Stream file to browser
var data =fs.readFileSync('E:\\test\\conversion\\310-output.pdf');
res.contentType("application/pdf");
res.send(data);
});
In client script, i am trying to call above post command and want to fetch the pdf and save it locally from response. Not sure how to do it. Below is client script. In browser, i was able to see the PDF but i need to access it through client script
var request = require('request');
request.post(
'http://localhost:3000/home',
{},
function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(" Response :: "+response);
// console.log(" Body :: "+body);
}
}
);
To recieve your pdf response in proper format mension "responseType": 'arraybuffer', and "responseEncoding": 'binary' while calling the api
const config = {
baseURL: "<your base url>",
url: "<api endpoint>",
headers: {},
method: "<method>",
responseType: "arraybuffe",
responseEncoding: "binary"
}
let response = await axios(config)
//response.data will be binary encoded buffer Array of pdf data
//*To convert to base64*
var base64 =result.data.toString("base64")
//to convert to binary
var binary =result.data.toString("binary")
//to write to file
fs.writeFileSync(`${name}.pdf`, response.data,'binary');
To download a file by request lib, we will to many ways to do that (I think so!)
Below is a simple way: Create a pipe stream from request response, then save data from stream to a file with a write stream
const request = require('request');
const fs = require('fs');
const req = request.post(
'http://localhost:3000/home',
{},
);
req.on('response', function (res) {
res.pipe(fs.createWriteStream('./filename_to_save.pdf'));
});

Sending a buffer in multipart/form-data POST request Node | Express | request

I have a buffer that I have taken from users uploaded image, that I then want to send on to another API in a multipart/form-data POST request.
I'm having issues however with the request object. I want to send a stream or buffer rather than accessing local server file system / creating temp files. I'm fairly new to the concept of streams.
I get the correct response from API sending
image_file: fs.createReadStream('image.png')
But when I try:
image_file: data // buffer
I get an error from the API saying I'm missing the image_file parameter.
Please help!
Docs for the API I'm using (Face++)
I'm using request to make the post request.
Here is my code in question:
app.post('/', (req, res) => {
const url = 'https://api-us.faceplusplus.com/facepp/v3/detect';
let data = [];
req.on('data', (chunk) => {
data.push(chunk)
})
req.on('end', (req, res) => {
data = Buffer.concat(data);
const formData = {
api_key: process.env.FACEPP_API_KEY,
api_secret: process.env.FACEPP_API_SECRET,
// image_file: fs.createReadStream('image.png') // works
image_file: data // doesnt work
}
const options = {
uri: url,
method: 'POST',
formData
}
request(options, (err, response, body) => {
if (err) console.log(err)
console.log(body)
})
})
})
After a little playing about, I have the following code, it's working nicely for me. I used the Multer middleware (https://github.com/expressjs/multer) for the original multipart upload. As a matter of interest, request doesn't seem to play nice uploading files unless you specify a filename option.
const multer = require('multer');
const upload = multer();
app.post('/', upload.any(), (req, res) => {
const url = 'https://api-us.faceplusplus.com/facepp/v3/detect';
console.log('Image upload complete, creating request to: ' + url);
var formData = {
api_key: process.env.FACEPP_API_KEY,
api_secret: process.env.FACEPP_API_SECRET,
image_file: {
value: req.files[0].buffer, // Upload the first file in the multi-part post
options: {
filename: 'image_file'
}
}
};
const options = {
uri: url,
formData: formData,
method: 'POST'
}
request(options, (err, response, body) => {
console.log('Request complete');
if (err) console.log('Request err: ', err);
console.log(body)
})
})
I'm getting a response that looks like this:
{
"image_id": "GuF0MUPoaTcL/rbbcg+2kA==",
"request_id": "1520789753,d913cce4-118e-4893-a1ee-d1ace2b6a65b",
"time_used": 142,
"faces": [{
"face_rectangle": {
"width": 183,
"top": 125,
"left": 220,
"height": 183
},
"face_token": "8b8e327edfc10730f344b1465934a478"
}]
}
I tested the image upload to my local server using curl like so:
curl -v -i -F "data=#smiling_woman.jpg" -H "Content-Type: multipart/form-data" -X POST http://localhost:3000/

Upload AJAX response (image blob) to Azure Blob Storage: Error 414

I am able to successfully upload a file via an upload button to my vendors API. My vendors API also returns a .png file in blob format that I need to upload to Azure Blob Storage. I have tried a few approaches, but am getting the following error in my Node console:
[Error] statusCode: 414
My front end code is in an Angular Controller which passes data back to my Node backend that contains my Azure Blob Storage calls. I have the formidable and request modules installed and required, but am not using them in my current backend code since the data I receive is already in blob format.
Here is my front end upload code. The success "result" is the blob data I am returned:
$scope.sendToProduction = function () {
var parts = document.getElementById("file").value.split("\\");
var uploadedfilename = parts[parts.length - 1];
var basefilename = uploadedfilename.split(".")[0];
var fileextension = uploadedfilename.split(".")[1];
var filename = basefilename + '.' + fileextension;
var file = document.getElementById("file").files[0];
var formdata = new FormData();
formdata.append(filename, file);
$.ajax({
url: 'http://myvendorsapi/fileuploadendpoint',
type: "POST",
data: formdata,
mimeType: "multipart/form-data",
processData: false,
contentType: false,
crossDomain: true,
success: function (result) {
var filename = 'Test.png';
var file = result;
console.log(file);
$http.post('/postAdvanced', {filename: filename, file: file }).success(function (data) {
console.log(data);
}, function (err) {
console.log(err);
});
},
error: function (error) {
console.log("Something went wrong!");
}
});
};
Here is my node backend for uploading to Azure Blob Storage:
app.post('/postAdvanced', function (req, res, next) {
var filename = req.body.filename;
var file = req.body.file;
blobSvc.createBlockBlobFromText('blob5', file, filename, function (error, result, response) {
if (!error) {
console.log("Uploaded" + result);
}
else {
console.log(error);
}
});
})
How do I upload an AJAX response into Azure Blob Storage?
The problem is that in this line of code:
blobSvc.createBlockBlobFromText('blob5', file, filename, function (error, result, response) {
you have the wrong parameter order. It should be:
blobSvc.createBlockBlobFromText('blob5', filename, file, function (error, result, response) {
HTTP status code 414 means "Request-URI Too Long". Did you pass the correct blob name into blobSvc.createBlockBlobFromText?

Uploading file using POST request in Node.js

I have problem uploading file using POST request in Node.js. I have to use request module to accomplish that (no external npms). Server needs it to be multipart request with the file field containing file's data. What seems to be easy it's pretty hard to do in Node.js without using any external module.
I've tried using this example but without success:
request.post({
uri: url,
method: 'POST',
multipart: [{
body: '<FILE_DATA>'
}]
}, function (err, resp, body) {
if (err) {
console.log('Error!');
} else {
console.log('URL: ' + body);
}
});
Looks like you're already using request module.
in this case all you need to post multipart/form-data is to use its form feature:
var req = request.post(url, function (err, resp, body) {
if (err) {
console.log('Error!');
} else {
console.log('URL: ' + body);
}
});
var form = req.form();
form.append('file', '<FILE_DATA>', {
filename: 'myfile.txt',
contentType: 'text/plain'
});
but if you want to post some existing file from your file system, then you may simply pass it as a readable stream:
form.append('file', fs.createReadStream(filepath));
request will extract all related metadata by itself.
For more information on posting multipart/form-data see node-form-data module, which is internally used by request.
An undocumented feature of the formData field that request implements is the ability to pass options to the form-data module it uses:
request({
url: 'http://example.com',
method: 'POST',
formData: {
'regularField': 'someValue',
'regularFile': someFileStream,
'customBufferFile': {
value: fileBufferData,
options: {
filename: 'myfile.bin'
}
}
}
}, handleResponse);
This is useful if you need to avoid calling requestObj.form() but need to upload a buffer as a file. The form-data module also accepts contentType (the MIME type) and knownLength options.
This change was added in October 2014 (so 2 months after this question was asked), so it should be safe to use now (in 2017+). This equates to version v2.46.0 or above of request.
Leonid Beschastny's answer works but I also had to convert ArrayBuffer to Buffer that is used in the Node's request module. After uploading file to the server I had it in the same format that comes from the HTML5 FileAPI (I'm using Meteor). Full code below - maybe it will be helpful for others.
function toBuffer(ab) {
var buffer = new Buffer(ab.byteLength);
var view = new Uint8Array(ab);
for (var i = 0; i < buffer.length; ++i) {
buffer[i] = view[i];
}
return buffer;
}
var req = request.post(url, function (err, resp, body) {
if (err) {
console.log('Error!');
} else {
console.log('URL: ' + body);
}
});
var form = req.form();
form.append('file', toBuffer(file.data), {
filename: file.name,
contentType: file.type
});
You can also use the "custom options" support from the request library. This format allows you to create a multi-part form upload, but with a combined entry for both the file and extra form information, like filename or content-type. I have found that some libraries expect to receive file uploads using this format, specifically libraries like multer.
This approach is officially documented in the forms section of the request docs - https://github.com/request/request#forms
//toUpload is the name of the input file: <input type="file" name="toUpload">
let fileToUpload = req.file;
let formData = {
toUpload: {
value: fs.createReadStream(path.join(__dirname, '..', '..','upload', fileToUpload.filename)),
options: {
filename: fileToUpload.originalname,
contentType: fileToUpload.mimeType
}
}
};
let options = {
url: url,
method: 'POST',
formData: formData
}
request(options, function (err, resp, body) {
if (err)
cb(err);
if (!err && resp.statusCode == 200) {
cb(null, body);
}
});
I did it like this:
// Open file as a readable stream
const fileStream = fs.createReadStream('./my-file.ext');
const form = new FormData();
// Pass file stream directly to form
form.append('my file', fileStream, 'my-file.ext');
const remoteReq = request({
method: 'POST',
uri: 'http://host.com/api/upload',
headers: {
'Authorization': 'Bearer ' + req.query.token,
'Content-Type': req.headers['content-type'] || 'multipart/form-data;'
}
})
req.pipe(remoteReq);
remoteReq.pipe(res);

NodeJS: sending/uploading a local file to a remote server

I have used the Winston module to create a daily log file for my offline app. I now need to be able to send or upload that file to a remote server via POST (that part already exists)
I know I need to write the file in chunks so it doesn't hog the memory so I'm using fs.createReadStream however I seem to only get a 503 response, even if sending just sample text.
EDIT
I worked out that the receiver was expecting the data to be named 'data'. I have removed the createReadSteam as I could only get it to work with 'application/x-www-form-urlencoded' and a synchronous fs.readFileSync. If I change this to 'multipart/form-data' on the php server would I be able to use createReadStream again, or is that only if I change to physically uploading the json file.
I've only been learning node for the past couple of weeks so any pointers would be gratefully received.
var http = require('http'),
fs = require('fs');
var post_options = {
host: 'logger.mysite.co.uk',
path: '/',
port: 80,
timeout: 120000,
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded'
}
}
var sender = http.request(post_options, function(res) {
if (res.statusCode < 399) {
var text = ""
res.on('data', function(chunk) {
text += chunk
})
res.on('end', function(data) {
console.log(text)
})
} else {
console.log("ERROR", res.statusCode)
}
})
var POST_DATA = 'data={['
POST_DATA += fs.readFileSync('./path/file.log').toString().replace(/\,+$/,'')
POST_DATA += ']}'
console.log(POST_DATA)
sender.write(POST_DATA)
sender.end()
After gazillion of trial-failure this worked for me. Using FormData with node-fetch. Oh, and request deprecated two days ago, btw.
const FormData = require('form-data');
const fetch = require('node-fetch');
function uploadImage(imageBuffer) {
const form = new FormData();
form.append('file', imageBuffer, {
contentType: 'image/jpeg',
filename: 'dummy.jpg',
});
return fetch(`myserver.cz/upload`, { method: 'POST', body: form })
};
In place of imageBuffer there can be numerous things. I had a buffer containing the image, but you can also pass the result of fs.createReadStream('/foo/bar.jpg') to upload a file from drive.
copied from https://github.com/mikeal/request#forms
var r = request.post('http://service.com/upload', function optionalCallback (err, httpResponse, body) {
if (err) {
return console.error('upload failed:', err);
}
console.log('Upload successful! Server responded with:', body);
})
var form = r.form()
form.append('my_field1', 'my_value23_321')
form.append('my_field2', '123123sdas')
form.append('my_file', fs.createReadStream(path.join(__dirname, 'doodle.png')))
Have a look at the request module.
It will provide you the ability to stream a file to POST requests.

Resources