I have used the Winston module to create a daily log file for my offline app. I now need to be able to send or upload that file to a remote server via POST (that part already exists)
I know I need to write the file in chunks so it doesn't hog the memory so I'm using fs.createReadStream however I seem to only get a 503 response, even if sending just sample text.
EDIT
I worked out that the receiver was expecting the data to be named 'data'. I have removed the createReadSteam as I could only get it to work with 'application/x-www-form-urlencoded' and a synchronous fs.readFileSync. If I change this to 'multipart/form-data' on the php server would I be able to use createReadStream again, or is that only if I change to physically uploading the json file.
I've only been learning node for the past couple of weeks so any pointers would be gratefully received.
var http = require('http'),
fs = require('fs');
var post_options = {
host: 'logger.mysite.co.uk',
path: '/',
port: 80,
timeout: 120000,
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded'
}
}
var sender = http.request(post_options, function(res) {
if (res.statusCode < 399) {
var text = ""
res.on('data', function(chunk) {
text += chunk
})
res.on('end', function(data) {
console.log(text)
})
} else {
console.log("ERROR", res.statusCode)
}
})
var POST_DATA = 'data={['
POST_DATA += fs.readFileSync('./path/file.log').toString().replace(/\,+$/,'')
POST_DATA += ']}'
console.log(POST_DATA)
sender.write(POST_DATA)
sender.end()
After gazillion of trial-failure this worked for me. Using FormData with node-fetch. Oh, and request deprecated two days ago, btw.
const FormData = require('form-data');
const fetch = require('node-fetch');
function uploadImage(imageBuffer) {
const form = new FormData();
form.append('file', imageBuffer, {
contentType: 'image/jpeg',
filename: 'dummy.jpg',
});
return fetch(`myserver.cz/upload`, { method: 'POST', body: form })
};
In place of imageBuffer there can be numerous things. I had a buffer containing the image, but you can also pass the result of fs.createReadStream('/foo/bar.jpg') to upload a file from drive.
copied from https://github.com/mikeal/request#forms
var r = request.post('http://service.com/upload', function optionalCallback (err, httpResponse, body) {
if (err) {
return console.error('upload failed:', err);
}
console.log('Upload successful! Server responded with:', body);
})
var form = r.form()
form.append('my_field1', 'my_value23_321')
form.append('my_field2', '123123sdas')
form.append('my_file', fs.createReadStream(path.join(__dirname, 'doodle.png')))
Have a look at the request module.
It will provide you the ability to stream a file to POST requests.
Related
Stack Overflow community, greetings. I'm trying to pass the response of a new request on the request object using the Node HTTP Module for a basic autocomplete search app for my website (i.e using Node as a proxy that will transform and redirect the requests within the server).
The flow basically is:
Client Browser - Node - ElasticSearch - Node - Client Browser
I've started with:
Listen to requests with http.createServer (function (req,res)
Get the body from the req object and use it in a new request with http.request(options, function (newReqResponse)
Get the body from that newReqResponse object and send it back to the client on the res object
The problem is that the content of newReqResponse is always outdated (trails behind the last typed character). i.e.:
If I type "te", the content of newReqResponse corresponds to that if I had typed only "t".
If I type "test", it corresponds to that if I had typed "tes".
And so on.
I've tried to solve it using Node.js streams and using the file system module to write and read files sync and async, but the result is the same. Here's a sample of the whole -code- picture:
var http = require('http');
var fs = require('fs');
http.createServer(function (req, res) {
var reqBody = '';
var newReqResponseBody = "";
req.on('data', function (chunk) {
reqBody += chunk;
fs.writeFile('reqbody.json', reqBody, function(err) {
if (err) {
throw err;
}
});
var options = {
hostname: '127.0.0.1',
port: 9500,
method: 'GET',
path: '/_search',
headers: { host: 'es',
'content-length': Buffer.byteLength(reqBody),
'content-type': 'application/json',
accept: 'application/json' },
};
var newReq = http.request(options, function (newReqResponse) {
newReqResponse.setEncoding("UTF-8");
newReqResponse.on('data', function (ch) {
newReqResponseBody += ch;
});
newReqResponse.on("end", function() {
fs.writeFile("newReqResponseBody.json", newReqResponseBody, function(err) {
if (err) {
throw err;
}
});
});
});
newReq.on("error", function(err) {
console.log(`problem with request: ${err.message}`);
});
newReq.write(reqBody);
newReq.end();
});
req.on('end', function() {
var responseBody = fs.readFileSync('newReqResponseBody.json', 'utf8');
console.log(responseBody);
res.end(responseBody);
});
}).listen(3000, '127.0.0.1');
Is there a workaround to work with requests and responses within the http server? If there isn't, I'll be very grateful if you give me any directions on how to solve this.
Since the planned use for Node is rather basic, I prefer to stick with core modules rather than having to get new ones from npm, unless that it's necessary.
Thanks in advance.
EDIT:
All I had to do was to call res.end(responseBody) within the newReqResponse.on("end") callback, which is totally counterintuitive for me, but... it works.
Glad you solved your own problem. However, I see room for improvement (not sure if you're new), especially if you're transferring data. Which you can do with streams.
You can see that I didn't calculate the content length, you're asked not to and should get ignore (for this specific case) according to HTTP specification as streams pass data in chunks with 'Transfer-Encoding': 'chunked' header.
const fs = require('fs');
const http = require('http');
const options = {
hostname: '127.0.0.1',
port: 9500,
method: 'GET',
path: '/_search',
headers: {
'Content-Type': 'application/json'
}
};
http.createServer((req, res) => {
req.pipe(fs.createWriteStream('reqBody.json'));
let request = http.request(options, (newRes) => {
newRes.pipe(res);
});
fs.createReadStream('reqBody.json').pipe(request);
res.setHeader('Content-Type', 'application/json');
}).listen(3000, '127.0.0.1');
You can shorten this snippet more if you don't want your data saved in the future and only want to pipe the req stream to request.
I am receiving a PDF file from a node server (it is running jsreport in this server) and i need to download this PDF in the client (i am using react in the client) but the problem is that when i download the file, it comes all blank and the title some strange symbols. After a lot of tests and researchs, i found that the problem may be that the file is coming enconded as chunked (i can see that in the headers of the response) and i need to decode do become a file again.
So, how to decode this chunked string to a file again?
In the client i am just downloading the file that comes in the responde:
handleGerarRelatorioButtonClick(){
axios.post(`${REQUEST_URL}/relatorios`, this.state.selectedExam).then((response) => {
fileDownload(response.data, this.state.selectedExam.cliente.nome.replace(' ', '_') + ".pdf");
});
}
In my server, i am making a request to my jsreport that is other node server and it returns the report as a PDF:
app.post('/relatorios', (request, response) => {
var exame = new Exame(request.body);
var pdf = '';
var body = {
"template": {
"shortid": "S1C9birB-",
"data": exame
}
};
var options = {
hostname: 'localhost',
port: 5488,
path: '/api/report',
method: 'POST',
headers: {
'Content-Type': 'application/json'
}
};
var bodyparts = [];
var bodylength = 0;
var post = http.request(options, (res) => {
res.on('data', (chunk) => {
bodyparts.push(chunk);
bodylength += chunk.length;
});
res.on('end', () => {
var pdf = new Buffer(bodylength);
var pdfPos = 0;
for(var i=0;i<bodyparts.length;i++){
bodyparts[i].copy(pdf, pdfPos, 0, bodyparts[i].length);
pdfPos += bodyparts[i].length;
}
response.setHeader('Content-Type', 'application/pdf');
response.setHeader('Content-disposition', exame._id + '.pdf');
response.setHeader('Content-Length', bodylength);
response.end(Buffer.from(pdf));
});
});
post.write(JSON.stringify(body));
post.end();
});
I am sure that my report is being rendered as expected because if i make a request from postman, it returns the PDF just fine.
Your solution is simply relaying data chunks but you are not telling your front end what to expect of these chunks or how to assemble them. At a minimum you should be setting the the Content-Type response header to application/pdf and to be complete should also be sending the Content-disposition as well as Content-Length. You may need to collect the PDF from your 3rd party source into a buffer and then send that buffer to your client if you are not able to set headers and pipe to response successfully.
[edit] - I'm not familiar with jsreport but it is possible (and likely) that the response they send is a buffer. If that is the case you could use something like this in place of your response to the client:
myGetPDFFunction(params, (err, res) => {
if (err) {
//handle it
} else {
response.writeHead(200, {
'Content-Type': 'application/pdf',
'Content-Length': [your buffer's content length]
});
response.end(Buffer.from([the res PDF buffer]));
}
}
What you haven't shown is the request made to obtain that PDF, so I couldn't be more specific at this time. You should look into the documentation of jsreport to see what it sends in its response, and you can also read up on buffers here
This is rough pseudo code but the point is to respond with the PDF buffer after setting the headers to their proper values.
I would like to get upload progress information during http post request using node js http.
my code is simple and it works but I am only notified when there is an error or the upload is completed.
But I need to show a progress bar to the user and to do that I need to check the progress every second or so.
Basically I need to have upload speed and how much of the file has been uploaded at given time.
const http = require('http');
const fs = require('fs');
const FormData = require('form-data');
function myUploadFunction(authToken,uploadPath, file) {
const form = new FormData();
form.append('file', fs.createReadStream(file.path));
form.append('filename', file.name);
form.append('parent_dir', '/');
const requestHeaders = {
Accept: '*/*',
Authorization: `Token ${authToken}`,
};
const options = {
host: 'myHost',
port: 'myPort',
path: uploadPath,
method: 'POST',
headers: requestHeaders,
};
form.submit(options, (error, response) => {
response.setEncoding('utf8');
response.on('data', (chunk) => {
// this code runs only when the upload is completed and the server only send some info about the file. And only if this info is large it will send it in chunks
logger.debug(`BODY: ${chunk}`);
});
response.on('end', () => {
logger.debug('No more data in response.');
});
});
}
When I upload a file, nothing happens till upload is completed then I get the response and it executes response.onData and afetr that response.onEnd.
You just need to add another listener to handle the progress
form.on('progress', (bytesReceived, bytesExpected) => {
logger.warn('progress bytesReceived: ', bytesReceived);
logger.warn('progress bytesExpected: ', bytesExpected);
});
I have a video file on s3 that I want to post to a user Facebook account:
https.get(signedUrlOfObjectInS3, function(httpRes){
var form = new formData(); // that's form-data module https://github.com/felixge/node-form-data
form.append('source', httpRes);
var options = {
method: 'post',
host: 'graph-video.facebook.com',
path: '/me/videos?access_token=' + user_access_token,
headers: form.getHeaders(),
}
var buffer = '';
var apiCall = https.request(options, function (res){
res.on('data',function(chunk){
buffer += chunk;
});
res.on('end',function(){
var data = JSON.parse(buffer);
console.log('data from fb is: ' + util.inspect(data));
});
});
form.pipe(apiCall);
});
The response I get from Facebook is:
(#352) Sorry, the video file you selected is in a format that we don\'t support.
The video file on s3 is a mov file with a content type of video/quicktime.
OK, so apparently Facebook ignores the content type in the headers and guesses it from the file name. Since the s3 signed url doesn't end with filename.mov for example, it doesn't get it...
All I had to do was concating a '&f=filename.mov' to the end of the signedUrl, and now Facebook get that...
The following code actually worked fine for me for me:
https.get(signedUrlOfObjectInS3, function(httpRes) {
var form = new FormData(); // that's form-data module https://github.com/felixge/node-form-data
form.append('source', httpRes);
var options = {
method: 'post',
host: 'graph-video.facebook.com',
path: '/me/videos?access_token=' + user_access_token,
headers: form.getHeaders(),
}
var apiCall = https.request(options);
form.pipe(apiCall);
apiCall.on('response', function(res) {
console.log(res.statusCode);
});
});
try making the small difference in apiCall postback ,
other explanation might be that i've used a public amazon URL...
I have problem uploading file using POST request in Node.js. I have to use request module to accomplish that (no external npms). Server needs it to be multipart request with the file field containing file's data. What seems to be easy it's pretty hard to do in Node.js without using any external module.
I've tried using this example but without success:
request.post({
uri: url,
method: 'POST',
multipart: [{
body: '<FILE_DATA>'
}]
}, function (err, resp, body) {
if (err) {
console.log('Error!');
} else {
console.log('URL: ' + body);
}
});
Looks like you're already using request module.
in this case all you need to post multipart/form-data is to use its form feature:
var req = request.post(url, function (err, resp, body) {
if (err) {
console.log('Error!');
} else {
console.log('URL: ' + body);
}
});
var form = req.form();
form.append('file', '<FILE_DATA>', {
filename: 'myfile.txt',
contentType: 'text/plain'
});
but if you want to post some existing file from your file system, then you may simply pass it as a readable stream:
form.append('file', fs.createReadStream(filepath));
request will extract all related metadata by itself.
For more information on posting multipart/form-data see node-form-data module, which is internally used by request.
An undocumented feature of the formData field that request implements is the ability to pass options to the form-data module it uses:
request({
url: 'http://example.com',
method: 'POST',
formData: {
'regularField': 'someValue',
'regularFile': someFileStream,
'customBufferFile': {
value: fileBufferData,
options: {
filename: 'myfile.bin'
}
}
}
}, handleResponse);
This is useful if you need to avoid calling requestObj.form() but need to upload a buffer as a file. The form-data module also accepts contentType (the MIME type) and knownLength options.
This change was added in October 2014 (so 2 months after this question was asked), so it should be safe to use now (in 2017+). This equates to version v2.46.0 or above of request.
Leonid Beschastny's answer works but I also had to convert ArrayBuffer to Buffer that is used in the Node's request module. After uploading file to the server I had it in the same format that comes from the HTML5 FileAPI (I'm using Meteor). Full code below - maybe it will be helpful for others.
function toBuffer(ab) {
var buffer = new Buffer(ab.byteLength);
var view = new Uint8Array(ab);
for (var i = 0; i < buffer.length; ++i) {
buffer[i] = view[i];
}
return buffer;
}
var req = request.post(url, function (err, resp, body) {
if (err) {
console.log('Error!');
} else {
console.log('URL: ' + body);
}
});
var form = req.form();
form.append('file', toBuffer(file.data), {
filename: file.name,
contentType: file.type
});
You can also use the "custom options" support from the request library. This format allows you to create a multi-part form upload, but with a combined entry for both the file and extra form information, like filename or content-type. I have found that some libraries expect to receive file uploads using this format, specifically libraries like multer.
This approach is officially documented in the forms section of the request docs - https://github.com/request/request#forms
//toUpload is the name of the input file: <input type="file" name="toUpload">
let fileToUpload = req.file;
let formData = {
toUpload: {
value: fs.createReadStream(path.join(__dirname, '..', '..','upload', fileToUpload.filename)),
options: {
filename: fileToUpload.originalname,
contentType: fileToUpload.mimeType
}
}
};
let options = {
url: url,
method: 'POST',
formData: formData
}
request(options, function (err, resp, body) {
if (err)
cb(err);
if (!err && resp.statusCode == 200) {
cb(null, body);
}
});
I did it like this:
// Open file as a readable stream
const fileStream = fs.createReadStream('./my-file.ext');
const form = new FormData();
// Pass file stream directly to form
form.append('my file', fileStream, 'my-file.ext');
const remoteReq = request({
method: 'POST',
uri: 'http://host.com/api/upload',
headers: {
'Authorization': 'Bearer ' + req.query.token,
'Content-Type': req.headers['content-type'] || 'multipart/form-data;'
}
})
req.pipe(remoteReq);
remoteReq.pipe(res);