Callback / asynchronous request in nodejs - node.js

I do a http request in nodejs with the following code
var buffer = "";
var postToPHP = function(data, path){
var httpreq = require('http');
var querystring = require("querystring");
var data = querystring.stringify(data);
var options = {
host : 'localhost',
path : path,
method : 'POST',
headers : {
'Content-Type' : 'application/x-www-form-urlencoded',
'Content-Length' : data.length
}
};
var buffer = "";
var reqPost = httpreq.request(options, function(res) {
res.on('data', function(d) {
buffer = buffer+d;
});
res.on('end', function() {
console.log("buffer",buffer); //this logs the buffer correctly
return buffer;
});
});
reqPost.write(data);
reqPost.end();
}
var buffer = postToPHP(message,path); //this buffer displays nothing because the call is async
I'd like to know exactly what is the standard procedure to "wait" for the server response in the nodejs or how to implement a callback that would react accordingly to what I want after I receive the message... Could someone give me an example of a callback on this please?

You have to pass in a callback if you're performing an asynchronous task inside a function:
var http = require('http'),
querystring = require('querystring');
function postToPHP(data, path, cb) {
var data = querystring.stringify(data);
var options = {
host: 'localhost',
path: path,
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': Buffer.byteLength(data.length)
}
};
http.request(options, function(res) {
var buffer = '',
calledBack = false;
res.on('data', function(d) {
buffer += d;
}).on('error', function(err) {
if (!calledBack) {
calledBack = true;
cb(err);
}
}).on('end', function() {
if (!calledBack) {
calledBack = true;
cb(null, buffer);
}
});
}).on('error', function(err) {
if (!calledBack) {
calledBack = true;
cb(err);
}
}).end(data)
}
Then use it:
postToPHP({ foo: 'bar' }, '/foo', function(err, data) {
if (err) throw err;
console.dir(data);
});

Related

How do I do a request.write(body) without the body?

I am using the code as shown below. However I want to include the body in the query string (which I can do fine) - however, I am unsure how to restructure thehttps.request so as to remove request.write(body) - simply using request.write() does not work as it requires a string.
Can someone help?
Thanks
var body = JSON.stringify( json.text );
const params = {
'q': body,
};
var requestUrl = url.parse( URL + queryStringify(params) );
const requestOptions = {
hostname: requestUrl.hostname,
path: requestUrl.path,
method: 'POST',
headers: {
'Content-Type': 'application/json',
}
};
var request = https.request(requestOptions, function(res) {
var data = "";
res.on('data', function (chunk) {
//do stuff
});
res.on('end', function () {
//do stuff
});
});
request.write(body);
request.end();
To do that, you can set your params in path field, with querystring like this :
Set your query string params into requestOptions
const querystring = require('querystring');
requestOptions.path = `/your/Path?${querystring.stringify({firstName: 'John', lastName: 'doe'})}`;
// Result is '/your/Path?firstName=John&lastName=doe'
Then, do your request
var request = https.request(requestOptions, function(res) {
res.on('data', function(chunk) {
// Do stuff
});
res.on('end', function() {
// Do stuff
});
});
request.on('error', function (err) {
// Throw err
});
request.end();
Hope it helps.

NodeJS http post request read timeout

I'm trying to execute the following code inside AWS Lambda which only makes a POST http request to an ElasticSearch.
The problem I'm facing is that it seems the nodejs request has a read timeout and the response is almost always cut and an error is thrown. I've checked that the problem is not related with AWS Lambda timeout which is set to 10 seconds and the code throws an error in less than a second.
As you can see, I've tried to put a timeout to 5secs but I think that's a connection timeout and not a read timeout.
What am I doing wrong?
var http = require('http');
exports.handler = (event, context, callback) => {
var options = {
hostname: '172.31.40.10',
port: 9200,
path: '/articles/es/_search?_source=reference',
method: 'POST',
headers: {
'Content-Type': 'application/json',
}
};
var req = http.request(options, function(res) {
res.setEncoding('utf8');
res.on('data', function (body) {
var parsed = JSON.parse(body);
var b = [];
for (var i = 0; i < parsed.hits.hits.length; i++) {
b.push(parsed.hits.hits[i]._source.reference);
}
var response = {
statusCode: '200',
body: JSON.stringify(b),
headers: {
'Content-Type': 'application/json',
}
};
callback(null, response);
});
});
req.on('error', function(e) {
callback(new Error('fallo'));
});
req.setTimeout(5000, function() {req.abort;})
req.on('socket', function (socket) {
socket.setTimeout(5000);
socket.on('timeout', function() {
req.abort();
});
});
req.write(MY_QUERY_HERE);
req.end();
};
I think you should let the stream of incoming data finish before performing any data manipulation.
Example :
var http = require('http');
//var _ = require('underscore');
function MyPostRequest(callback) {
var options = {
hostname:'172.31.40.10',
port:9200,
path:'/articles/es/_search?_source=reference',
method:'POST',
headers:{'Content-Type':'application/json'}
};
http.request(options, function(res) {
var tmpstore = ''; //temp. data storage
//:Store the continuous incoming data stream
res.on('data', function(d){tmpstore += d;});
//:Data reception is done, use it now...
res.on('end', function() {
var parsed = JSON.parse(tmpstore); var b = [];
for (var i = 0; i < parsed.hits.hits.length; i++) {
b.push(parsed.hits.hits[i]._source.reference);
}
/* //I suggest using underscore module :
_.each(parsed.hits.hits,function(element, index, list){
b.push(element._source.reference);
});
*/
var response = {
statusCode:'200',
body:JSON.stringify(b),
headers:{'Content-Type':'application/json'}
};
callback(null, response);
});
//:Response contained an error
res.on('error', function(e){/*error handling*/callback(e,null);});
});
}

AWS Lambda - NodeJS POST request and asynch write/read file

I am new to NodeJS and inside of AWS Lambda I am trying to make a POST request that calls an external API with a JSON object, creates a document with the response and then reads the contents of the file.
Coming from a Ruby background, I'm thinking the problem stems from my unfamiliarity with asynchronous programming, but I've tried using callbacks and readfileSync just to debug with no luck.
Any help would be appreciated.
var querystring = require('querystring');
var https = require('https');
var fs = require('fs');
exports.handler = function(event, context) {
console.log('Received event:', JSON.stringify(event, null, 2));
var operation = event.operation;
delete event.operation;
var accessKey = event.accessKey;
delete event.accessKey;
var templateName = event.templateName;
delete event.templateName;
var outputName = event.outputName;
delete event.outputName;
var req = {
"accessKey": accessKey,
"templateName": templateName,
"outputName": outputName,
"data": event.data
};
function doPost(data, callback) {
// Build the post string from an object
var post_data = JSON.stringify(data);
// An object of options to indicate where to post to
var post_options = {
host: 'hostname.com',
port: '443',
path: '/path/to/api',
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Content-Length': post_data.length
}
};
// Set up the request
var file = fs.createWriteStream(outputName);
var post_req = https.request(post_options, function(res) {
res.setEncoding('utf8');
res.pipe(file);
res.on('response', function(response) {
console.log(response);
});
res.on('error', function(e) {
context.fail('error:' + e.message);
})
res.on('end', function() {
context.succeed('success, end request listener');
});
});
// post the data
post_req.write(post_data);
post_req.end();
callback();
}
function printFileContents() {
fs.readFileSync(outputName, 'utf8', function (err, data) {
console.log('file contents:' + data);
});
}
switch (operation) {
case 'create':
// Make sure there's data before we post it
if(req) {
doPost(req, printFileContents);
printFileContents();
}
break;
...
}
};
In general, I'd recommend starting like this:
var querystring = require('querystring');
var https = require('https');
var fs = require('fs');
exports.handler = function(event, context) {
console.info('Received event', event);
var data = {
"accessKey": accessKey,
"templateName": templateName,
"outputName": outputName,
"data": event.data
};
// Build the post string from an object
var post_data = JSON.stringify(data);
// An object of options to indicate where to post to
var post_options = {
host: 'hostname.com',
port: '443',
path: '/path/to/api',
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Content-Length': post_data.length
}
};
var post_request = https.request(post_options, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
context.done(body);
});
res.on('error', function(e) {
context.fail('error:' + e.message);
});
});
// post the data
post_request.write(post_data);
post_request.end();
};
You can see I simplified your code quite a bit. I'd recommend avoiding the file system since that would slow down your program. I'm also not sure about what is the real goal of your function so I just return the HTTP response.

How can you synchronize this process using nodejs?

I need to iterate on an array, for each item I apply an operation by calling an HTTP call.
The difficulty is that i need to syncronize this process in order to call a callback after the loop (containing the array after all the operation executed by the HTTP call).
Let's consider this short example:
function customName(name, callback) {
var option = {
host:'changename.com',
path: '/'+name,
headers: { 'Content-Type': 'application/json' },
port: 80,
method:'POST'
};
var req = http.request(option, function(res) {
var output = "";
res.on('data', function (chunk) {
output += chunk;
});
res.on('end', function() {
var obj = JSON.parse(output);
callback(obj.res);
});
});
req.on('error', function(e) {
console.error(e.message);
});
req.end();
}
function changeNames(OldNames, callback) {
var Res = [];
for (name in OldNames) {
customName(OldNames[name], function(new_name) { Res.push(new_name); });
});
callback(Res);
}
var Names = ['toto', 'tata', 'titi'];
changeNames(Names, function(Names) {
//...
});
Here the loop is over before the first HTTP call, so the Res array is empty.
How can we synchronize this execution?
I know it's not very good to synchronize treatments in nodejs. Do you think it would be better to communicate the names one by one with the client and not building an array?
You can use async.map for that. You pass it your list of names, it will run the getOriginalName function (which you mistakenly called customName, I think) for each name and gather the result, and in the end it will call a function with an array of results:
var http = require('http');
var async = require('async');
function getOriginalName(name, callback) {
var option = {
host:'changename.com',
path: '/'+name,
headers: { 'Content-Type': 'application/json' },
port: 80,
method:'POST'
};
var req = http.request(option, function(res) {
var output = "";
res.on('data', function (chunk) {
output += chunk;
});
res.on('end', function() {
var obj = JSON.parse(output);
callback(null, obj.res);
});
});
req.on('error', function(e) {
callback(e);
});
req.end();
}
function changeNames(OldNames, callback) {
async.map(OldNames, getOriginalName, callback);
}
var Names = ['toto', 'tata', 'titi'];
changeNames(Names, function(err, NewNames) {
console.log('N', NewNames);
});

Uploading video to Vimeo with nodejs

I'm trying to upload a video to vimeo with nodejs (https://developer.vimeo.com/apis/advanced/upload at step 3) . This is what I currently do:
Firstly I call the function to read the file:
var options = {
hostname : dataObject.ticket.host,
path : '/upload?ticket_id=' + dataObject.ticket.id,
port : 8080,
method: 'POST'
}
postMovie(options);
I get these parameters from my object:
{
"generated_in": "0.0308",
"stat": "ok",
"ticket": {
"endpoint": "http://126535.cloud.vimeo.com:8080/upload?ticket_id=9d818e8bd066dfd54e53f1be2fa3f958",
"endpoint_secure": "https://126535.cloud.vimeo.com/upload?ticket_id=9d818e8bd066dfd54e53f1be2fa3f958",
"host": "126535.cloud.vimeo.com",
"id": "9d818e8bd066dfd54e53f1be2fa3f958",
"max_file_size": "26843545600"
}
}
This function is called :
function postMovie(options){
// This is an async file read
fs.readFile('public/uploads/4363066343.mp4', function (err, data) {
if (err) {
console.log("FATAL An error occurred trying to read in the file: " + err);
process.exit(-2);
}
// Make sure there's data before we post it
if(data) {
PostData(data,options);
}
else {
console.log("No data to post");
process.exit(-1);
}
});
};
When the file is read:
function PostData(data,options) {
var headers = {
'Content-Type': 'video/mp4',
'Content-Length': data.length
}
options.headers = headers
console.log(options)
// Set up the request
var post_req = http.request(options, function(res) {
res.on('data', function (chunk) {
console.log('Response: ' + chunk);
});
});
// post the data
post_req.write(data);
post_req.end();
post_req.on('error', function(e) {
console.log('problem with request: ' + e.message);
});
}
My post_req.on(error) logs this:
problem with request: write EPIPE
problem with request: write EPIPE
I understand this is because of a time-out at the serverside.
I assume my request is not well formed.
Can someone point out what I did wrong ?
The upload operation will be much simpler with the request module.
var inspect = require('eyespect').inspector();
var request = require('request')
var path = require('path')
var fs = require('fs')
var filePath = path.join(__dirname, '../public/uploads/foo.mp4')
fs.stat(filePath, function(err, stats) {
if (err) {
inspect(err, 'error stating file')
return
}
var fileSize = stats.size
var url = 'https://126535.cloud.vimeo.com/upload?ticket_id=9d818e8bd066dfd54e53f1be2fa3f958'
var opts = {
url: url,
method: 'post',
headers: {
'Content-Length': fileSize,
'Content-Type': 'foo'
}
var r = request(opts)
// pipe the file on disk to vimeo
var readStream = fs.createReadStream(filePath)
readStream.pipe(r)
readStream.on('error', function (err) {
inspect(err, 'error uploading file')
})
readStream.on('end', function (err) {
inspect('file uploaded correctly')
})
})
Request also allows you to set the timeout option as well if the file is big and thus takes a long time to upload

Resources