Export HTTP request as module function Node.js - node.js

I've tried (with success) to do an http request. I have some REST Api's, like graph.facebook.com as target. The functionality is the same. When i make an HTTP request with node.js as a simple program i can do it.
Really i want to make a little module, and i have this code:
// file: facebook.js
var http = require('http');
var Facebook = (function() {
function Facebook(access_token) {
this.access_token = access_token;
}
Facebook.prototype.getObject = function(id) {
var options;
this.id = id;
options = {
host: 'graph.facebook.com',
port: 80,
path: '/' + id + '?access_token=' + this.access_token
};
return http.request(options, function(response) {
response.on('data', function(d) {
return JSON.parse(d);
});
request.end();
return request.on('error', function(err) {
return {
error: 'An error ocurred.'
};
});
});
};
return Facebook;
})();
module.exports = Facebook;
After, when i write a program i can do this:
var facebook = require('./facebook.js');
var fb = facebook('my_Access_token')
// Here's the problem:
var response = fb.getObject('My_facebook_profile_ID')
I get a response like
{ domain: null,
_events:
{ response: { [Function: g] listener: [Function] },
socket: { [Function: g] listener: [Function] } },
....
When i must have something like
{
"id": "MyIDNumer",
"first_name": "Orlando",
"gender": "male",
"last_name": "Sanchez",
"link": "https://www.facebook.com/MyFacebookName",
"locale": "es_LA",
"name": "Orlando S\u00e1nchez",
"username": "MyUsername"
}
What can i do?

The first thing you should do is rewrite the module to not use the same function name twice ('Facebook').
Secondly, there's no need for the closure, just export the constructor.
Thirdly, you are trying to return a value from an asynchronous callback. This particular issue is common with people coming from the synchronous world.
Assuming you wanted to do a GET request, here's your module refactored after the above and other things are fixed:
// file: facebook.js
var http = require('http');
function Facebook(access_token) {
if (!(this instanceof Facebook))
return new Facebook(access_token);
this.access_token = access_token;
}
Facebook.prototype.getObject = function(id, cb) {
var options;
this.id = id;
options = {
host: 'graph.facebook.com',
port: 80,
path: '/' + id + '?access_token=' + this.access_token
};
http.get(options, function(res) {
var buf = '',
hadError = false;
res.on('data', function(d) {
buf += d;
}).on('error', function(err) {
hadError = true;
cb(err);
}).on('end', function() {
if (hadError)
return;
var val;
try {
val = JSON.parse(buf);
} catch (err) {
return cb(err);
}
cb(null, val);
});
});
};
module.exports = Facebook;
Then use it like so:
var facebook = require('./facebook.js');
var fb = facebook('my_Access_token');
fb.getObject('My_facebook_profile_ID', function(err, response) {
if (err) {
// include better error handling here
return console.log(err);
}
// use response here
console.dir(response);
});

Related

Synchronous api calls in Node.js

I've a cronjob that runs every 10 secs. It requests for machines for a single client and does computation based on the response and then has to update or create documents with those computations in a for loop. But, the api calls after '***' in the code don't happen until the for loop has executed and the data sent to the api calls is that of the last machine which is wrong. I want to solve this by this way or some other way possible. My code looks like this:
// index.js
const cron = require("node-cron");
const express = require("express");
const fs = require("fs");
const request = require("request");
app = express();
var clientId = 'ABCD';
var apiUrl = 'http://example.com:3001/';
var getMachines = apiUrl + 'getMachines/',
updateMachine = apiUrl + 'updateMachine/',
getControlRoomStatus = apiUrl + 'getControlRoomStatus/',
registerControlRoomStatus = apiUrl + 'registerControlRoomStatus/',
updateControlRoomStatus = apiUrl + 'updateControlRoomStatus/';
cron.schedule("*/10 * * * * *", function() {
APICall(getMachines, { 'clientId' : clientId }, 'POST', function(err, machines) {
if (err) {
console.log(err);
} else {
console.log('--------------------------------------------------------------------------------------------------');
var allMachines = machines;
var currentDateTime = IST();
for (var i = 0; i < 2; i++) {
var lastCycleTime = new Date(allMachines[i]['lastCycleTime']);
var lastHeartbeat = new Date(allMachines[i]['lastHeartbeat']);
var machineData;
var controlRoomData;
var machineId = {
'machineId' : allMachines[i]['machineId']
};
console.log(machineId);
if (allMachines[i]['downtimeStatus'] == '0') {
if ((currentDateTime - lastCycleTime)>300000) {
if ((currentDateTime - lastHeartbeat)>300000) {
console.log(allMachines[i]['machineId'] ,' No Internet');
controlRoomData = {
'clientId': clientId,
'lastTimeStamp': allMachines[i]['lastCycleTime'],
'status': 'Inactive',
'type': 'No Internet/Power'
};
} else {
console.log(allMachines[i]['machineId'] ,' No button pressed');
controlRoomData = {
'clientId': clientId,
'lastTimeStamp': allMachines[i]['lastCycleTime'],
'status': 'Inactive',
'type': 'No Button Pressed'
};
}
machineData = {
'status' : 'Inactive'
};
} else {
console.log(allMachines[i]['machineId'] ,' Active');
machineData = {
'status' : 'Active'
};
controlRoomData = {
'clientId': clientId,
'lastTimeStamp': allMachines[i]['lastCycleTime'],
'status': 'Active',
'type': 'N.A'
};
}
} else {
if ((currentDateTime - lastHeartbeat)>300000) {
console.log(allMachines[i]['machineId'] ,' button pressed ',' No Internet');
controlRoomData = {
'clientId': clientId,
'lastTimeStamp': allMachines[i]['lastCycleTime'],
'status': 'Inactive',
'type': 'No Internet/Power'
};
} else {
var downtimeLength = allMachines[i]['downtimeData'].length - 1;
console.log(allMachines[i]['machineId'] ,' button pressed ',allMachines[i]['downtimeData'][downtimeLength]['downtimeType']);
controlRoomData = {
'clientId': clientId,
'lastTimeStamp': allMachines[i]['lastCycleTime'],
'status': 'Inactive',
'type': allMachines[i]['downtimeData'][downtimeLength]['downtimeType']
};
}
machineData = {
'status' : 'Inactive'
};
}
***
APICall(getControlRoomStatus, machineId, 'POST', function(err, controlRoom) {
if (err) {
console.log(err);
} else {
console.log(machineId,controlRoomData);
if (controlRoom == null ) {
APICall(registerControlRoomStatus, controlRoomData, 'POST', function(err, body) {
if (err) {
console.log(err);
} else {
// console.log(body);
}
});
} else {
var updateControlRooomUrl = (updateControlRoomStatus+''+controlRoom['_id']+'');
// console.log(updateControlRooomUrl);
APICall(updateControlRooomUrl, controlRoomData, 'PUT', function(err, body) {
if (err) {
console.log(err);
} else {
// console.log(body);
}
});
}
}
});
var updateMachineUrl = (updateMachine+''+allMachines[i]['_id']+'');
// console.log(updateMachineUrl);
APICall(updateMachineUrl, machineData, 'PUT', function(err, body) {
if (err) {
console.log(err);
} else {
console.log(i,machineId);
// console.log(body);
}
});
}
}
});
});
function APICall(url, requestData, method, callback) {
request({
url: url,
form: requestData,
method: method
}, function (error, response, body) {
if (error || response.statusCode !== 200) {
return callback(error || {statusCode: response.statusCode});
}
callback(null, JSON.parse(body));
});
}
function IST(){
var dateUTC = new Date();
var dateUTC = dateUTC.getTime();
var dateIST = new Date(dateUTC);
dateIST.setHours(dateIST.getHours() + 5);
dateIST.setMinutes(dateIST.getMinutes() + 30);
return dateIST;
}
app.listen(3128);
Thank you in advance.
I used a different method to do things and now it's working just as it's supposed to. I used 'async' and replaced the for loop with the following:
var async = require('async');
...
async.map(allMachines , function(machine, callback) {
...
});
...
You can try the following package:
sync-request
You can find it here on NPM.
Here is an example how to use it (from the docs):
var request = require('sync-request');
var res = request('GET', 'http://example.com');
console.log(res.getBody());
As stated in the documentation, don't use it in production code, since this will terribly block your server and it will slow down considerably (if you are running a HTTP server which you are using express).
If you have asynchronous code and you want to execute some code after the asynchronous you also can use:
Observables (not native need to use a package, RxJS for example)
Promises (native ES6 JS)

Node.js Async mapLimit and memory

SOLVED, see the answer please.
I have a list of urls I fetch using request and for some reason I am unable to save more than 1720 records into my database when I try to fetch 2000 or more URL's at a time.
If I try 1000 to 2000 and 2000 to 3000, I get 3000 results in total. But when I try 1000 to 3000 or 4000 to 6000, my script stops after fetching the 1720th result.
What could be the reason for that?
I use mapLimit in order to limit concurrent connections.
app.get('/asynctest', function(req, res) {
var people = [];
for (var a = 1000; a < 3000; a++) {
people.push("http://www.example.com/" + a + "/person.html");
}
async.mapLimit(people, 20, function(url, callback) {
// iterator function
var options2 = {
url: url,
headers: {
'User-Agent': req.headers['user-agent'],
'Content-Type': 'application/json; charset=utf-8'
}
};
request(options2, function(error, response, body) {
if (!error && response.statusCode == 200) {
async.series([
// add this person into database
function(callback) {
var $ = cheerio.load(body);
var name = entities.decodeHTML($('span[itemprop="name"]').text());
new person({
name: name,
url: url
}).save();
callback();
},
function(callback) {
async.waterfall([
function(callback) {
var $ = cheerio.load(body);
var jobs = $('span[itemprop="jobtitle"]').length;
if (jobs == 0) {
console.log("no job");
var jobsArr = 0;
} else {
var jobsArr = [];
for (var aa = 0; aa < jobs; aa++) {
jobsArr.push(entities.decodeHTML($('span[itemprop="jobtitle"]').eq(aa).text()));
}
}
callback(null, jobsArr);
},
function(jobsArr, callback) {
if (jobsArr == 0) {
console.log("this person has no jobs");
} else {
async.map(jobsArr, function(jobs, callback) {
personRole.where('job_name', jobs).fetch({
require: true
}).then(function(data1) {
data1 = data1.toJSON();
person.where('url', url).fetch().then(function(data2) {
data2 = data2.toJSON();
new personPersonRole({
person_id: data2.id,
personrole_id: data1.id
}).save();
});
}).catch(function(err) {
new personRole({
job_name: jobs
}).save().then(function(data3) {
data3 = data3.toJSON();
person.where('url', url).fetch().then(function(data4) {
data4 = data4.toJSON();
new personPersonRole({
person_id: data4.id,
personrole_id: data3.id
}).save();
});
});
});
});
}
callback(null, "yes");
}
], function(err, result) {
if (err) {
console.log(err);
}
});
callback();
}
], function(err, result) {
if (err) {
console.log("err3");
}
});
} else {
console.log("err4");
}
});
callback();
});
});
EDIT #2
The following code is also problematic, adds only 1747 records and it stops after that. If I stop my node app and start again, it also stops at 1747.
var person = require('./models').person;
app.get('/asynctest', function(req, res) {
var people = [];
for (var a = 18000; a < 20000; a++) {
people.push("random url");
}
async.mapLimit(people, 20, function(url, callback) {
new person({
name: "YES",
url: url
}).save();
callback();
});
});
db.js
var knex = require('knex')({
client: 'mysql',
connection: {
host: '127.0.0.1',
port: 8889,
user: 'root',
password: 'root',
database: 'mydatabase',
charset: 'utf8'
},
pool: {
min: 0,
max: 100
}
});
var db = require('bookshelf')(knex);
module.exports = db;
models.js
var person = db.Model.extend({
tableName: 'people'
});
module.exports = {
person : person
};
EDIT #3
Okay, I think I've found the solution.
18K-18.5K - no problem
18K-19K - no problem
18K-19.7K - no problem
18K-20K - RangeError: Maximum call stack size exceeded at new Object
(native)
I just wrapped my callbacks into a wrapper, like below.
async.setImmediate(function () {
callback();
});
app.get('/async22', function(req, res) {
var people = [];
for (var a = 18000; a < 20000; a++) {
people.push("yes");
}
async.mapLimit(people, 20, function(url, callback) {
new person({
name: "YES",
url: url
}).save();
async.setImmediate(function () {
callback();
});
});
});
It was in front of my eyes all the time, actually this solution isn't unique, it's already included async library's database.
https://github.com/caolan/async#common-pitfalls-stackoverflow
Here's how you do it.
async.setImmediate(function () {
callback();
});
Example
app.get('/async22', function(req, res) {
var people = [];
for (var a = 18000; a < 20000; a++) {
people.push("yes");
}
async.mapLimit(people, 20, function(url, callback) {
new person({
name: "YES",
url: url
}).save();
async.setImmediate(function () {
callback();
});
});
});
This is still not an answer, but it is too big for the comment.
I suggest to reduce the code to some minimal example and try if it works (example is below and it works for me).
Second thing - is to add a monitoring route (see the /apptest below), so you can check if you app still works and the processing progress.
If the minimal sample works, start to gradually add more code with your logic to it and check if it still works.
The code, server.js:
var util = require('util');
var express = require('express');
var async = require('async');
var request = require('request');
var cheerio = require('cheerio');
var app = express.createServer();
app.successCount = 0;
app.errorCount = 0;
app.get('/apptest', function(req, res) {
res.send(
util.format(
'I am OK, successCount: %s, errorCount: %s',
app.successCount, app.errorCount
), 200
);
});
app.get('/asynctest', function(req, res) {
var people = [];
for (var a = 1000; a < 3000; a++) {
people.push("http://www.example.com/" + a + "/person.html");
}
async.mapLimit(people, 20, function(url, callback) {
// iterator function
var options2 = {
url: url,
headers: {
'User-Agent': req.headers['user-agent'],
'Content-Type': 'application/json; charset=utf-8'
}
};
request(options2, function(error, response, body) {
if (!error) {
console.log('success requesting: ' + options2.url);
var $ = cheerio.load(body);
app.successCount += 1;
} else {
console.log(
'error requesting: %s, error: %s, status: %s',
options2.url, error, response.statusCode
);
app.errorCount += 1;
}
callback();
});
});
});
app.listen(3000, function() {
console.log(
"Express server listening on port %d in %s mode",
app.address().port, app.settings.env
);
});
Dependencies, package.json:
{
"name": "application-name",
"version": "0.0.1",
"private": true,
"dependencies": {
"async": "^1.5.2",
"cheerio": "^0.19.0",
"express": "2.5.8",
"request": "^2.67.0"
},
"devDependencies": {}
}
Run the example as node server.js and then open http://localhost:3000/asynctest in the browser, you should see success requesting: xxxx in the console. While it is running (or when it stops running) - open http://localhost:3000/apptest to check if app is OK and how many urls are processed.

Why is this not working, And how do you debug code in nodejs?

First I'm trying to learn nodejs, and for that I am writing like a router. As you have in express.
This is my code:
function sirus() {
var util = utilFuncs(),
paths = {};
this.router = util.handleReq;
this.con = {
get: function (path, fn, options) {
options = (options || {}).method = "GET";
util.makeRequest(path, fn, options);
},
post: function (path, fn, options) {
options = (options || {}).method = "POST";
util.makeRequest(path, fn, options);
}
};
this.req = {
get: function (path, fn) {
util.addPath("GET", path, fn);
},
post: function (path, fn) {
util.addPath("POST", path, fn);
}
};
function utilFuncs() {
function handleReq(req, res) {
var url = parsePath(req.url);
var path = paths[req.method + url];
// console.log(req.url, url +' requested');
if (typeof path != "function") {
res.writeHead(404);
} else {
path(req, res);
}
res.end();
}
function addPath(type, path, callback) {
// add path as a key of object for easier look up, root=/, /a=/a, /a/=/a, /a?=/a, /a/?..=/a so there are no duplicates path=parsePath(path);
paths[type + path] = callback;
}
function parsePath(path) {
path = url.parse(path).pathname;
if ((/[^\/]+\/(?=$)/igm).test(path)) path = path.substring(0, path.length - 1);
return path;
}
function makeRequest(path, fn, options) {
var urls = url.parse(path);
var d = {
host: null,
hostname: null,
method: "GET",
path: '/',
port: 80,
headers: {},
auth: null,
agent: false,
keepAlive: false
};
for (var k in options) d[k] = options[k];
d.host = urls.host;
d.hostname = urls.hostname;
d.path = urls.pathname;
d.headers['Content-Type'] = 'application/x-www-form-urlencoded';
d.headers['Content-Length'] = ((d || {}).body || {}).length || '';
var req = http.request(options, function (res) {
// console.log('STATUS: ' + res.statusCode);
// console.log('HEADERS: ' + JSON.stringify(res.headers));
res.setEncoding('utf8');
var data = '';
res.on('data', function (chunk) {
data += chunk;
});
res.on('end', function (chunk) {
data += chunk;
});
res.on('response', function () {
fn(res);
});
});
req.on('error', function (e) {
console.log('problem with request: ' + e.message);
});
req.write(d.body);
req.end();
}
return {
makeRequest: makeRequest,
handleReq: handleReq,
addPath: addPath
};
}
}
and i use it like this:
var http = require('http'),
url = require('url'),
app = new sirus();
http.createServer(function (req, res) {
app.router(req, res);
}).listen(80, '127.0.0.1');
app.req.get('/', function (req, res) {
res.writeHead(200, {'Content-Type': 'text/html'});
var link = "https://www.reddit.com/";
res.write('Click here');
});
app.con.get('http://www.google.com/', function (res) {
console.log('from req: ' + res);
});
The error i get is The First argument must be string or buffer
Receiving part works correctly, but when i started to add make request part something isn't right.
Also since I'm new to JS and Nodejs. I'd like general advice or anything that catch your eye something that i can be improved about this code. Since I'm note sure i am doing things the best way.
On line 85
req.write(d.body);
there is no member named "body" in the "d" object.
1 - Why is this not working - you get exactly error that tells you why it is not working. If you check more details about error it shows the file and which line error happened at. Perhaps the arguments you pass to function call are not what Express is expecting.
2 - How to debug - look at node.js package called nodev.

How to make tedious Async in Meteor

I am using this module tedious to connect. I am having issues when I try to populate a collection with the data from MSSQL.
My code thus far:
http://pastebin.com/q4ByRCbW
Meteor.startup(function () {
var Request = Meteor.require('tedious').Request;
var Connection = Meteor.require('tedious').Connection;
var config = {
userName: 'xxxxx',
password: 'xxxx',
server: '197.xxx.xxx.xxx',
// If you're on Windows Azure, you will need this:
options: {
encrypt: true,
debug: {
packet: true,
data: true,
payload: true,
token: false,
log: true
}
}
};
var connection = new Connection(config);
var asnycWrapFunc = Async.wrap(connection.execSql);
var rettarr = [];
function executeStatement() {
Fiber(function(){
request = new Request("select * from AccountSummary", function(err, rowCount) {
if (err) {
console.log(err);
} else {
console.log(rowCount + ' rows');
}
});
request.on('row', function(columns) {
aaary = []; cnting = 0;
columns.forEach(function(column) {
console.log(column.value);
aaary.push(column.value);
});
if (AccountSummary.find().count() === 0){
AccountSummary.insert({ID:aaary[0], ClientNo:aaary[1], ClientName:aaary[2]});
}
});
//rettarr.push(aaary);
}).run();
asnycWrapFunc(request);
//return rettarr;
}
connection.on('connect', function(err) {
// If no error, then good to go...
var res = executeStatement();
// aaary = res[0];
console.log(res);
errr = err;
});
});
I have found that you have to use Future if you want to you a package like Tedious.
This mini tutorial has the answer

How can this function be tested with vows?

How can the following function, intended to add routes to express.js app based on an object hierarchy, be tested using vows.js cleanly without breaking vows' separation of the topic and the vow?
var addRoutes = function(routeObject, app, path) {
var httpVerbs = ['get', 'post', 'put', 'delete'];
path = path || '';
for(var property in routeObject){
var routesAdded = false;
for (var verbIndex in httpVerbs) {
var verb = httpVerbs[verbIndex];
var completePath, handler;
if (property === verb) {
if (typeof(routeObject[verb]) === 'function') {
handler = routeObject[verb];
completePath = path;
} else {
handler = routeObject[verb].handler;
completePath = path + (routeObject[verb].params || '');
}
app[verb](completePath, handler);
routesAdded = true;
}
}
if (!routesAdded) {
addRoutes(routeObject[property], app, path + '/' + property);
}
}
};
The test should confirm that functions corresponding to http verbs are called on the app object. e.g. if routeObject is:
{
tracker: {
message: {
get: {
handler: function (req, res) {
res.send("respond with a resource");
}
}
}
}
}
then
app.get('/tracker/message', function (req, res) {
res.send("respond with a resource");
});
should be called.
Something like this should work:
var assert = require('assert'),
request = require('request');
var verbs = ['get', 'post', 'put', 'delete'],
app = require('your/app.js'),
port = 8080;
//
// Your route would look something like this
//
var example = {
tracker: {
message: {
get: {
status: 200,
body: 'respond with a resource'
}
}
}
};
function testRoutes(suite, route, app, path) {
path = path || '/';
Object.keys(route).forEach(function (key) {
if (~verbs.indexOf(key)) {
return testRoutes(suite, route[key], app, [path, key].join('/'));
}
var test = [key, path].join(' ');
suite[test] = {
topic: function () {
request({
method: key,
uri: 'http://localhost' + port + path
}, this.callback);
},
'should respond correctly': function (err, res, body) {
assert.isNull(err);
assert.equal(res.statusCode, route[key].status);
assert.equal(body, route[key].body);
}
}
});
return suite;
}
var suite = vows.describe('api');
testRoutes(suite, example, app);
suite.export(module);

Resources