NodeJS http post request read timeout - node.js

I'm trying to execute the following code inside AWS Lambda which only makes a POST http request to an ElasticSearch.
The problem I'm facing is that it seems the nodejs request has a read timeout and the response is almost always cut and an error is thrown. I've checked that the problem is not related with AWS Lambda timeout which is set to 10 seconds and the code throws an error in less than a second.
As you can see, I've tried to put a timeout to 5secs but I think that's a connection timeout and not a read timeout.
What am I doing wrong?
var http = require('http');
exports.handler = (event, context, callback) => {
var options = {
hostname: '172.31.40.10',
port: 9200,
path: '/articles/es/_search?_source=reference',
method: 'POST',
headers: {
'Content-Type': 'application/json',
}
};
var req = http.request(options, function(res) {
res.setEncoding('utf8');
res.on('data', function (body) {
var parsed = JSON.parse(body);
var b = [];
for (var i = 0; i < parsed.hits.hits.length; i++) {
b.push(parsed.hits.hits[i]._source.reference);
}
var response = {
statusCode: '200',
body: JSON.stringify(b),
headers: {
'Content-Type': 'application/json',
}
};
callback(null, response);
});
});
req.on('error', function(e) {
callback(new Error('fallo'));
});
req.setTimeout(5000, function() {req.abort;})
req.on('socket', function (socket) {
socket.setTimeout(5000);
socket.on('timeout', function() {
req.abort();
});
});
req.write(MY_QUERY_HERE);
req.end();
};

I think you should let the stream of incoming data finish before performing any data manipulation.
Example :
var http = require('http');
//var _ = require('underscore');
function MyPostRequest(callback) {
var options = {
hostname:'172.31.40.10',
port:9200,
path:'/articles/es/_search?_source=reference',
method:'POST',
headers:{'Content-Type':'application/json'}
};
http.request(options, function(res) {
var tmpstore = ''; //temp. data storage
//:Store the continuous incoming data stream
res.on('data', function(d){tmpstore += d;});
//:Data reception is done, use it now...
res.on('end', function() {
var parsed = JSON.parse(tmpstore); var b = [];
for (var i = 0; i < parsed.hits.hits.length; i++) {
b.push(parsed.hits.hits[i]._source.reference);
}
/* //I suggest using underscore module :
_.each(parsed.hits.hits,function(element, index, list){
b.push(element._source.reference);
});
*/
var response = {
statusCode:'200',
body:JSON.stringify(b),
headers:{'Content-Type':'application/json'}
};
callback(null, response);
});
//:Response contained an error
res.on('error', function(e){/*error handling*/callback(e,null);});
});
}

Related

NodeJs Error: Can't set headers after they are sent

I am new to nodejs and stuck up with this Error:Can't set headers after they are sent. Below is my code. Please help. I used postman to test this code. It is working for the first 2 hits but on the 3rd hit this error is coming.
const http = require('http');
const fs = require('fs')
module.exports.verifyPyeval5 = function(request, response){
let filePath = "D:/PyEval/NewPyEvalRequests/solution.txt";
fs.readFile(filePath, 'utf8', function (err,data) {
var json = JSON.parse(data);
post_call(json,function(res){
response.status(200)
.json(res);
});
});
};
var post_call = function(new_val,callback){
var post_req = http.request(post_options, function(res) {
res.setEncoding('utf8');
res.on('data', function (chunk) {
callback(chunk);
});
});
post_req.write(JSON.stringify(new_val));
post_req.end();
};
var post_options = {
host: 'acclabserv',
port: '8080',
path: '/generate',
method: 'POST',
headers: {
'Content-Type': 'application/json'
}
};
I got it the issue is with callback function it gets called more then once.
var post_call = function(new_val,callback){
var post_req = http.request(post_options, function(res) {
res.setEncoding('utf8');
var str = null;
res.on('data', function (chunk) {
str += chunk;
});
res.on('end', function () {
callback(str);
});
});
post_req.write(JSON.stringify(new_val));
post_req.end();
};
One more thing to note here I dont know the type of chuck if its an object and you want array of object in response then you can use this
var str = []; //in req.on('data') str.push(chunck)

Limit 60 requests by minute in for loop with node JS

I'm working with zoho crm api, two make two api calls, one to get the ID and the other to update a record, the problem is that i have a 60 calls per minute.
I've an array with lots of info that I use to update the records.
Is there any way I can call 60 times per minute to avoid getting blocked? or any other work around?
var censados = [...] <-- big array
for (var i = 0; i < censados.length; i++) {
var elcorreo = censados[i].email;
var elpais = censados[i].country;
var criteria = "(Account Name:"+elcorreo+")"
var criteria = encodeURI(criteria);
var options = {
hostname: 'crm.zoho.com',
port: 443,
path: '/crm/private/json/Potentials/searchRecords?authtoken=apikey&scope=crmapi&newFormat=1&selectColumns=Potentials(POTENTIALID)&criteria=' + criteria,
method: 'GET'
};
https.request(options, function(response) {
var responseData = '';
response.setEncoding('utf8');
response.on('data', function(chunk) {
responseData += chunk;
jsonObject = JSON.parse(chunk);
});
response.once('error', function(err) {
// Some error handling here, e.g.:
res.serverError(err);
});
response.on('end', function() {
try {
//console.log(responseData.response.result.Potentials.row.FL.content)
console.log(jsonObject.response.result)
if (jsonObject.response.result.Potentials.row.FL.content) {
var IdPotential = jsonObject.response.result.Potentials.row.FL.content;
// START UPDATER
var xmlData = '\
<Potentials>\
<row no="1">\
<FL val="PaĆ­s de Recidencia">'+elpais+'</FL>\
</row>\
</Potentials>';
var xmlData = encodeURI(xmlData);
var options = {
hostname: 'crm.zoho.com',
port: 443,
path: '/crm/private/xml/Potentials/updateRecords?authtoken=apikey&scope=crmapi&id='+IdPotential+'&xmlData='+xmlData,
method: 'POST'
};
https.request(options, function(response) {
response.setEncoding('utf8');
response.on('data', function(chunk) {
responseData += chunk;
//jsonObject = JSON.parse(chunk);
});
response.once('error', function(err) {
// Some error handling here, e.g.:
res.serverError(err);
});
response.on('end', function() {
try {
console.log(responseData)
} catch (e) {
console.log(e)
}
});
}).end();
};
//END UPDATE
} catch (e) {
console.log(e)
}
});
}).end();
}
Seems that your code in not valid. You have to provide APIKEY for your account
Generate Auth Token
API calls limited on your account. As usual 250-500 requests /user depends on your subscription API Limit, so I suggest you use mass update call (version = 4).
You can play with API using Zoho CRM Console

AWS Lambda - NodeJS POST request and asynch write/read file

I am new to NodeJS and inside of AWS Lambda I am trying to make a POST request that calls an external API with a JSON object, creates a document with the response and then reads the contents of the file.
Coming from a Ruby background, I'm thinking the problem stems from my unfamiliarity with asynchronous programming, but I've tried using callbacks and readfileSync just to debug with no luck.
Any help would be appreciated.
var querystring = require('querystring');
var https = require('https');
var fs = require('fs');
exports.handler = function(event, context) {
console.log('Received event:', JSON.stringify(event, null, 2));
var operation = event.operation;
delete event.operation;
var accessKey = event.accessKey;
delete event.accessKey;
var templateName = event.templateName;
delete event.templateName;
var outputName = event.outputName;
delete event.outputName;
var req = {
"accessKey": accessKey,
"templateName": templateName,
"outputName": outputName,
"data": event.data
};
function doPost(data, callback) {
// Build the post string from an object
var post_data = JSON.stringify(data);
// An object of options to indicate where to post to
var post_options = {
host: 'hostname.com',
port: '443',
path: '/path/to/api',
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Content-Length': post_data.length
}
};
// Set up the request
var file = fs.createWriteStream(outputName);
var post_req = https.request(post_options, function(res) {
res.setEncoding('utf8');
res.pipe(file);
res.on('response', function(response) {
console.log(response);
});
res.on('error', function(e) {
context.fail('error:' + e.message);
})
res.on('end', function() {
context.succeed('success, end request listener');
});
});
// post the data
post_req.write(post_data);
post_req.end();
callback();
}
function printFileContents() {
fs.readFileSync(outputName, 'utf8', function (err, data) {
console.log('file contents:' + data);
});
}
switch (operation) {
case 'create':
// Make sure there's data before we post it
if(req) {
doPost(req, printFileContents);
printFileContents();
}
break;
...
}
};
In general, I'd recommend starting like this:
var querystring = require('querystring');
var https = require('https');
var fs = require('fs');
exports.handler = function(event, context) {
console.info('Received event', event);
var data = {
"accessKey": accessKey,
"templateName": templateName,
"outputName": outputName,
"data": event.data
};
// Build the post string from an object
var post_data = JSON.stringify(data);
// An object of options to indicate where to post to
var post_options = {
host: 'hostname.com',
port: '443',
path: '/path/to/api',
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Content-Length': post_data.length
}
};
var post_request = https.request(post_options, function(res) {
var body = '';
res.on('data', function(chunk) {
body += chunk;
});
res.on('end', function() {
context.done(body);
});
res.on('error', function(e) {
context.fail('error:' + e.message);
});
});
// post the data
post_request.write(post_data);
post_request.end();
};
You can see I simplified your code quite a bit. I'd recommend avoiding the file system since that would slow down your program. I'm also not sure about what is the real goal of your function so I just return the HTTP response.

Callback / asynchronous request in nodejs

I do a http request in nodejs with the following code
var buffer = "";
var postToPHP = function(data, path){
var httpreq = require('http');
var querystring = require("querystring");
var data = querystring.stringify(data);
var options = {
host : 'localhost',
path : path,
method : 'POST',
headers : {
'Content-Type' : 'application/x-www-form-urlencoded',
'Content-Length' : data.length
}
};
var buffer = "";
var reqPost = httpreq.request(options, function(res) {
res.on('data', function(d) {
buffer = buffer+d;
});
res.on('end', function() {
console.log("buffer",buffer); //this logs the buffer correctly
return buffer;
});
});
reqPost.write(data);
reqPost.end();
}
var buffer = postToPHP(message,path); //this buffer displays nothing because the call is async
I'd like to know exactly what is the standard procedure to "wait" for the server response in the nodejs or how to implement a callback that would react accordingly to what I want after I receive the message... Could someone give me an example of a callback on this please?
You have to pass in a callback if you're performing an asynchronous task inside a function:
var http = require('http'),
querystring = require('querystring');
function postToPHP(data, path, cb) {
var data = querystring.stringify(data);
var options = {
host: 'localhost',
path: path,
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': Buffer.byteLength(data.length)
}
};
http.request(options, function(res) {
var buffer = '',
calledBack = false;
res.on('data', function(d) {
buffer += d;
}).on('error', function(err) {
if (!calledBack) {
calledBack = true;
cb(err);
}
}).on('end', function() {
if (!calledBack) {
calledBack = true;
cb(null, buffer);
}
});
}).on('error', function(err) {
if (!calledBack) {
calledBack = true;
cb(err);
}
}).end(data)
}
Then use it:
postToPHP({ foo: 'bar' }, '/foo', function(err, data) {
if (err) throw err;
console.dir(data);
});

How can you synchronize this process using nodejs?

I need to iterate on an array, for each item I apply an operation by calling an HTTP call.
The difficulty is that i need to syncronize this process in order to call a callback after the loop (containing the array after all the operation executed by the HTTP call).
Let's consider this short example:
function customName(name, callback) {
var option = {
host:'changename.com',
path: '/'+name,
headers: { 'Content-Type': 'application/json' },
port: 80,
method:'POST'
};
var req = http.request(option, function(res) {
var output = "";
res.on('data', function (chunk) {
output += chunk;
});
res.on('end', function() {
var obj = JSON.parse(output);
callback(obj.res);
});
});
req.on('error', function(e) {
console.error(e.message);
});
req.end();
}
function changeNames(OldNames, callback) {
var Res = [];
for (name in OldNames) {
customName(OldNames[name], function(new_name) { Res.push(new_name); });
});
callback(Res);
}
var Names = ['toto', 'tata', 'titi'];
changeNames(Names, function(Names) {
//...
});
Here the loop is over before the first HTTP call, so the Res array is empty.
How can we synchronize this execution?
I know it's not very good to synchronize treatments in nodejs. Do you think it would be better to communicate the names one by one with the client and not building an array?
You can use async.map for that. You pass it your list of names, it will run the getOriginalName function (which you mistakenly called customName, I think) for each name and gather the result, and in the end it will call a function with an array of results:
var http = require('http');
var async = require('async');
function getOriginalName(name, callback) {
var option = {
host:'changename.com',
path: '/'+name,
headers: { 'Content-Type': 'application/json' },
port: 80,
method:'POST'
};
var req = http.request(option, function(res) {
var output = "";
res.on('data', function (chunk) {
output += chunk;
});
res.on('end', function() {
var obj = JSON.parse(output);
callback(null, obj.res);
});
});
req.on('error', function(e) {
callback(e);
});
req.end();
}
function changeNames(OldNames, callback) {
async.map(OldNames, getOriginalName, callback);
}
var Names = ['toto', 'tata', 'titi'];
changeNames(Names, function(err, NewNames) {
console.log('N', NewNames);
});

Resources