For some reason when the status code is not 200 and I return then call the callback the executed script just hangs there, not exiting. Why is that?
var http = require('http');
var qs = require('querystring');
var args = {
q: 'dfdfdf'
};
var opts = {
hostname: 'api.openweathermap.org',
path: '/data/2.5/weather?' + qs.stringify(args)
};
function cb(err, result) {
console.log(err, result);
}
http.get(opts, function(res) {
var buffer = new Buffer(0);
if (res.statusCode !== 200) return cb(new Error('Unable to fulfill request.'));
res.on('readable', function() {
return buffer = Buffer.concat([buffer, this.read()]);
});
res.on('end', function() {
return cb(null, JSON.parse(buffer.toString('utf8')));
});
});
Command line:
$ node plugins/weather.js
[Error: Unable to fulfill request.] undefined
# I have to ctrl+c at this point
You still need to consume the stream until it emits end event:
http.get(opts, function(res) {
var buffer = new Buffer(0);
res.on('readable', function() {
return buffer = Buffer.concat([buffer, this.read()]);
});
res.on('end', function() {
if (res.statusCode !== 200) return cb(new Error('Unable to fulfill request.'))
return cb(null, buffer.toString());
});
});
You never call res.end(). You should also do some res.write calls before res.end().
Related
Following is the node-js code used for HTTP requests. This code is giving "This deferred has already been resolved" error on production servers when I try to use requestPromise.resolve(str) in request end. Can someone please suggest what might be the error?
Libraries used : http and node-promise
var Promise = require('node-promise').Promise;
var requestPromise = new Promise();
callback = function (response) {
var str = '';
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function () {
if (!(response && response.statusCode >= 200 && response.statusCode < 300)) {
requestPromise.resolve(str);
return;
}
var resp;
try {
resp = JSON.parse(str);
} catch (ex) {
resp = str;
}
requestPromise.resolve(str);
});
});
var request = http.request(options, callback);
request.on('error', function (err) {
requestPromise.resolve(err);
});
request.write(postObject);
request.end();
I think you cannot use new Promise() (because it need resolver).
You can use this code:
new Promise(function(resolve, reject) {
callback = function (response) {
var str = '';
response.on('data', function (chunk) {
str += chunk;
});
response.on('end', function () {
if (!(response && response.statusCode >= 200 && response.statusCode < 300)) {
resolve(str);
return;
}
var resp;
try {
resp = JSON.parse(str);
} catch (ex) {
resp = str;
}
resolve(resp);
});
});
var request = http.request(options, callback);
request.on('error', function (err) {
reject(err);
});
request.write(postObject);
request.end();
});
I need to implement system which
Get data from parent collection.
Check if particular key found in redis
If no then do a http call and get json data then set cache
if yes then get data from cache
save data into child collection for parent id.
I have working solution using callback something like that.
MongoClient.connect(dsn).then(function(db) {
parentcollection.findOne({"_id" : new ObjectId(pid)}, function(err, data) {
var redis = require("redis"),
client = redis.createClient();
client.on("error", function (err) {
console.log("Error " + err);
});
// If not set
client.get(cacheKey, function(err, data) {
// data is null if the key doesn't exist
if(err || data === null) {
var options = {
host: HOST,
port: 80,
path: URI
};
var req = http.get(options, function(res) {
res.setEncoding('utf8');
res.on('data', function (chunk) {
body += chunk;
//console.log('CHUNK: ' + chunk);
});
res.on('end', function () {
data = JSON.parse(body);
// Get childdata After process of data
childcollection.save(childdata, {w:1}, function(cerr, inserted) {
db.close();
});
});
});
} else {
// Get childdata from cache data
childcollection.save(childdata, {w:1}, function(cerr, inserted) {
db.close();
});
}
});
});
I want to use promise (native one, not external ones like bluebird / request ) instead of callbacks. I checkout manuals and thinking if I need to implement like that
var promise1 = new Promise((resolve, reject) => {
MongoClient.connect(dsn).then(function(db) {
parentcollection.findOne({"_id" : new ObjectId(pid)}, function(err, data) {
});
}}.then(function(data){
var promise2 = new Promise((resolve, reject) => {
var redis = require("redis"),
client = redis.createClient();
client.on("error", function (err) {
console.log("Error " + err);
});
// If not set
client.get(cacheKey, function(err, data) {
// data is null if the key doesn't exist
if(err || data === null) {
var options = {
host: HOST,
port: 80,
path: URI
};
var promise3 = new Promise((resolve, reject) => {
var req = http.get(options, function(res) {
res.setEncoding('utf8');
res.on('data', function (chunk) {
body += chunk;
//console.log('CHUNK: ' + chunk);
});
res.on('end', function () {
data = JSON.parse(body);
// Get childdata After process of data
});
})
}).then(function(data){
childcollection.save(childdata, {w:1}, function(cerr, inserted) {
db.close();
});
});
} else {
// Get childdata from cache data
childcollection.save(childdata, {w:1}, function(cerr, inserted) {
db.close();
});
}
});
}}.then(function(data){
});
});
Which look like as dirty as callback hell or any better approach which not used promises like above ?
One issue is that you never call the resolve functions provided to the promise constructor callbacks. Without calling them, promises never resolve.
I would suggest creating those new promises in separate, reusable functions. On the other hand, some MongoDb methods return promises already when you don't provide the callback argument.
You could do it like below.
// Two promisifying functions:
function promiseClientData(client, key) {
return new Promise(function (resolve, reject) {
return client.get(key, function (err, data) {
return err ? reject(err) : resolve(data); // fulfull the promise
});
});
}
function promiseHttpData(options) {
return new Promise(function (resolve, reject) {
return http.get(options, function(res) {
var body = ''; // You need to initialise this...
res.setEncoding('utf8');
res.on('data', function (chunk) {
body += chunk;
//console.log('CHUNK: ' + chunk);
});
res.on('end', function () {
data = JSON.parse(body);
resolve(data); // fulfull the promise
});
);
});
}
// Declare the db variable outside of the promise chain to avoid
// having to pass it through
var db;
// The actual promise chain:
MongoClient.connect(dsn).then(function (dbArg) {
db = dbArg;
return parentcollection.findOne({"_id" : new ObjectId(pid)}); // returns a promise
}).then(function (data) {
var redis = require("redis"),
client = redis.createClient();
client.on("error", function (err) {
console.log("Error " + err);
});
// Get somehow cacheKey...
// ...
return promiseClientData(client, cacheKey);
}).then(function (data) {
// If not set: data is null if the key doesn't exist
// Throwing an error will trigger the next `catch` callback
if(data === null) throw "key does not exist";
return data;
}).catch(function (err) {
var options = {
host: HOST,
port: 80,
path: URI
};
return promiseHttpData(options);
}).then(function (data) {
// Get childdata by processing data (in either case)
// ....
// ....
return childcollection.save(childdata, {w:1}); // returns a promise
}).then(function () {
db.close();
});
I assume that the promises returned by MongoDb are fully compliant. In doubt, you can turn them into native JavaScript promises by calling Promise.resolve() on them, for instance like this:
return Promise.resolve(parentcollection.findOne({"_id" : new ObjectId(pid)}));
or:
return Promise.resolve(childcollection.save(childdata, {w:1}));
I am getting following error while running code in AWS-Lambda.
TypeError: first argument must be a string or Buffer at ClientRequest.OutgoingMessage.write (_http_outgoing.js:447:11) at EventEmitter. (/var/task/index.js:52:13) at emitOne (events.js:77:13) at EventEmitter.emit (events.js:169:7) at exports.handler.eventEmitter.on.offset (/var/task/index.js:57:18)
'use strict';
let https = require('https');
exports.handler = (event, context, callback) => {
var ratesData =[];
var totalRecords =0;
var events = require('events');
var options = {
hostname: 'encrypted.google.com',
port: 443,
path: '/',
method: 'GET'
};
// Create an eventEmitter object
var eventEmitter = new events.EventEmitter();
eventEmitter.on('getJson', function(offset)
{
const req = https.request(options.toString(), (res) => {
let body = '';
console.log('Status:', res.statusCode);
console.log('Headers:', JSON.stringify(res.headers));
res.setEncoding('utf8');
res.on('data', (chunk) => body += chunk);
res.on('end', () => {
console.log('Successfully processed HTTPS response');
// If we know it's JSON, parse it
if (res.headers['content-type'] === 'application/json')
{
var requestedJson = JSON.parse(body);
body = requestedJson.records;
totalRecords = requestedJson.total_records;
body.forEach(function(record)
{
ratesData.push(record);
});
}
callback(null, body);
});
});
req.on('error', callback);
req.write(JSON.stringify(event.data));
req.end();
});
console.log('in calling');
// Bind the connection event with the handler
eventEmitter.emit('getJson',0);
for(var i=1;i < (totalRecords/100)+1;i++)
{
eventEmitter.emit('getJson',i);
}
console.log(ratesData);
};
For one you have toString() on your options object. Remove that.
I'd also recommend testing locally as it's easier to debug.
// filename: test.js - run as node test.js
// Require your lambda
require('./index.js');
// Dummy context
var context = { };
// The event object - make sure of its format. S3 buckets pass a Records array for example
var e = { };
// Call the Lambda function
lambda.handler(e, context, function(err, result) {
console.log('------------');
console.log('Context done');
console.log(' error:', err);
console.log(' result:', result);
});
how to use Q to make it wait until previous response has come from the server.
What I am looking to do here is compare the response from test server and production server for the same request.
I get the responses back from both the servers, but unable to compare them since the assert statement is executed before the response comes back.
Any one know what I am doing wrong. heres the code.
var Q = require('q');
var path='';
var prodResponse = '';
var tstReponse = '';
Q.fcall(readFile())
.then(secondFunction())
.then(thirdFunction())
.then(function(){
console.log("prodResponse: "+prodResponse);
console.log("tstResponse: "+tstResponse);
assert.strictEqual(prodResponse, tstResponse)
})
.catch(function(){
console.log('error occurred');
})
.done();
function readFile(){
fs.readFile('hostname.json', function (err, data) {
if (err) return console.error(err);
path = JSON.parse(data);
return JSON.parse(data);
});
}
function secondFunction(){
var prodOptions = {
hostname: 'somehostname.com',
port: 80,
path: "/path?"+path.path,
method: 'POST',
headers: {
'Content-Type': 'application/json;charset=UTF-8'
},
auth : ''
};
return http.request(prodOptions, function(res) {
console.log('Prod');
res.setEncoding('utf8');
res.on('data', function (chunk) {
prodResponse = chunk;
return chunk;
});
res.on('end', function() {
console.log('No more data in response.');
})
}).on('error', function(e) {
console.log('problem with request: ' + e.message);
}).end();
}
function thirdFunction(){
// same a second, only difference is the response http.
}
There is multiple errors in your code
Q.fcall(readFile())
Your q variable is q and not Q. So this line will crash because Q is undefined (javascript is case sensitive).
Then, readFile doesn't return any promise (in fact, it returns nothing). So the q library can't use anything to wait the end of any asynchronous work. The then callbacks will be fired immediatly.
You can use Q.ninvoke to make your readFile function return a promise, and you can use Q.defer to create and return a promise from your secondFunction:
var Q = require('q');
var path='';
var prodResponse = [];
var tstReponse = '';
readFile()
.then(secondFunction())
.then(thirdFunction())
.then(function(){
console.log("prodResponse: "+prodResponse);
console.log("tstResponse: "+tstResponse);
assert.strictEqual(prodResponse, tstResponse)
})
.catch(function(){
console.log('error occurred');
})
.done();
function readFile(){
return Q.ninvoke(fs, 'readFile', 'hostname.json').then(function (data) {
path = JSON.parse(data);
return path;
}, function (err) {
console.error(err);
});
}
function secondFunction(){
var prodOptions = {
hostname: 'somehostname.com',
port: 80,
path: "/path?"+path.path,
method: 'POST',
headers: {
'Content-Type': 'application/json;charset=UTF-8'
},
auth : ''
};
var defer = Q.defer();
var chunks = [];
http.request(prodOptions, function(res) {
console.log('Prod');
res.setEncoding('utf8');
res.on('data', function (chunk) {
chunks.push(chunk);
});
res.on('end', function() {
console.log('No more data in response.');
prodResponse = chunks.join('');
defer.resolve(prodResponse);
})
}).on('error', function(e) {
console.log('problem with request: ' + e.message);
defer.reject(e);
}).end();
return defer.promise;
}
function thirdFunction(){
// same a second, only difference is the response http.
}
I'm trying to limit my use of an external API in my node.js code.
I've set up node rate limiter, but it doesn't seem to be working. I still hit 429's. What else should I be doing that I'm not?
var RateLimiter = require('limiter').RateLimiter; // Rate limits
var limiter = new RateLimiter(1, 2000); // one call every two seconds
self.riotAPI = function(options, cb){
limiter.removeTokens(1, function() {
https.request(options, function(response) {
// Error handling
response.on('error', function (e) {
console.log(e);
});
var str = '';
// Another chunk of data has been recieved, so append it to `str`
response.on('data', function (chunk) {
str += chunk;
});
// Parse and return the object
response.on('end', function () {
if(response.statusCode >= 400) {
var err = "HTTP response "+response.statusCode;
console.log(err);
cb(new Error("err"), null);
}
else {
cb(null, JSON.parse(str));
}
});
}).end();
});
}
I switched to Bottleneck and got everything functioning as desired.
self.riotAPI = function(options, cb){
limiter.submit( function(lcb) {
https.request(options, function(response) {
// Error handling
response.on('error', function (e) {
console.log(e);
});
var str = '';
// Another chunk of data has been recieved, so append it to `str`
response.on('data', function (chunk) {
str += chunk;
});
// Parse and return the object
response.on('end', function () {
if(response.statusCode >= 400) {
var err = "HTTP response "+response.statusCode;
console.log(err);
// If it's a 429, retry
if(response.statusCode == 429) {
console.log("retrying...");
self.riotAPI(options, cb);
}
// If not, fail
else {
cb(new Error("err"), null);
lcb();
}
}
else {
cb(null, JSON.parse(str));
lcb();
}
});
}).end();
}, null);
}