Synchronous api calls in Node.js - node.js

I've a cronjob that runs every 10 secs. It requests for machines for a single client and does computation based on the response and then has to update or create documents with those computations in a for loop. But, the api calls after '***' in the code don't happen until the for loop has executed and the data sent to the api calls is that of the last machine which is wrong. I want to solve this by this way or some other way possible. My code looks like this:
// index.js
const cron = require("node-cron");
const express = require("express");
const fs = require("fs");
const request = require("request");
app = express();
var clientId = 'ABCD';
var apiUrl = 'http://example.com:3001/';
var getMachines = apiUrl + 'getMachines/',
updateMachine = apiUrl + 'updateMachine/',
getControlRoomStatus = apiUrl + 'getControlRoomStatus/',
registerControlRoomStatus = apiUrl + 'registerControlRoomStatus/',
updateControlRoomStatus = apiUrl + 'updateControlRoomStatus/';
cron.schedule("*/10 * * * * *", function() {
APICall(getMachines, { 'clientId' : clientId }, 'POST', function(err, machines) {
if (err) {
console.log(err);
} else {
console.log('--------------------------------------------------------------------------------------------------');
var allMachines = machines;
var currentDateTime = IST();
for (var i = 0; i < 2; i++) {
var lastCycleTime = new Date(allMachines[i]['lastCycleTime']);
var lastHeartbeat = new Date(allMachines[i]['lastHeartbeat']);
var machineData;
var controlRoomData;
var machineId = {
'machineId' : allMachines[i]['machineId']
};
console.log(machineId);
if (allMachines[i]['downtimeStatus'] == '0') {
if ((currentDateTime - lastCycleTime)>300000) {
if ((currentDateTime - lastHeartbeat)>300000) {
console.log(allMachines[i]['machineId'] ,' No Internet');
controlRoomData = {
'clientId': clientId,
'lastTimeStamp': allMachines[i]['lastCycleTime'],
'status': 'Inactive',
'type': 'No Internet/Power'
};
} else {
console.log(allMachines[i]['machineId'] ,' No button pressed');
controlRoomData = {
'clientId': clientId,
'lastTimeStamp': allMachines[i]['lastCycleTime'],
'status': 'Inactive',
'type': 'No Button Pressed'
};
}
machineData = {
'status' : 'Inactive'
};
} else {
console.log(allMachines[i]['machineId'] ,' Active');
machineData = {
'status' : 'Active'
};
controlRoomData = {
'clientId': clientId,
'lastTimeStamp': allMachines[i]['lastCycleTime'],
'status': 'Active',
'type': 'N.A'
};
}
} else {
if ((currentDateTime - lastHeartbeat)>300000) {
console.log(allMachines[i]['machineId'] ,' button pressed ',' No Internet');
controlRoomData = {
'clientId': clientId,
'lastTimeStamp': allMachines[i]['lastCycleTime'],
'status': 'Inactive',
'type': 'No Internet/Power'
};
} else {
var downtimeLength = allMachines[i]['downtimeData'].length - 1;
console.log(allMachines[i]['machineId'] ,' button pressed ',allMachines[i]['downtimeData'][downtimeLength]['downtimeType']);
controlRoomData = {
'clientId': clientId,
'lastTimeStamp': allMachines[i]['lastCycleTime'],
'status': 'Inactive',
'type': allMachines[i]['downtimeData'][downtimeLength]['downtimeType']
};
}
machineData = {
'status' : 'Inactive'
};
}
***
APICall(getControlRoomStatus, machineId, 'POST', function(err, controlRoom) {
if (err) {
console.log(err);
} else {
console.log(machineId,controlRoomData);
if (controlRoom == null ) {
APICall(registerControlRoomStatus, controlRoomData, 'POST', function(err, body) {
if (err) {
console.log(err);
} else {
// console.log(body);
}
});
} else {
var updateControlRooomUrl = (updateControlRoomStatus+''+controlRoom['_id']+'');
// console.log(updateControlRooomUrl);
APICall(updateControlRooomUrl, controlRoomData, 'PUT', function(err, body) {
if (err) {
console.log(err);
} else {
// console.log(body);
}
});
}
}
});
var updateMachineUrl = (updateMachine+''+allMachines[i]['_id']+'');
// console.log(updateMachineUrl);
APICall(updateMachineUrl, machineData, 'PUT', function(err, body) {
if (err) {
console.log(err);
} else {
console.log(i,machineId);
// console.log(body);
}
});
}
}
});
});
function APICall(url, requestData, method, callback) {
request({
url: url,
form: requestData,
method: method
}, function (error, response, body) {
if (error || response.statusCode !== 200) {
return callback(error || {statusCode: response.statusCode});
}
callback(null, JSON.parse(body));
});
}
function IST(){
var dateUTC = new Date();
var dateUTC = dateUTC.getTime();
var dateIST = new Date(dateUTC);
dateIST.setHours(dateIST.getHours() + 5);
dateIST.setMinutes(dateIST.getMinutes() + 30);
return dateIST;
}
app.listen(3128);
Thank you in advance.

I used a different method to do things and now it's working just as it's supposed to. I used 'async' and replaced the for loop with the following:
var async = require('async');
...
async.map(allMachines , function(machine, callback) {
...
});
...

You can try the following package:
sync-request
You can find it here on NPM.
Here is an example how to use it (from the docs):
var request = require('sync-request');
var res = request('GET', 'http://example.com');
console.log(res.getBody());
As stated in the documentation, don't use it in production code, since this will terribly block your server and it will slow down considerably (if you are running a HTTP server which you are using express).
If you have asynchronous code and you want to execute some code after the asynchronous you also can use:
Observables (not native need to use a package, RxJS for example)
Promises (native ES6 JS)

Related

getting 403 from lambda calling api gateway

I have an api post end point which would update a customer's information in dynamodb. It is set to authenticate using AWS_IAM. I am getting 403 from my lambda when calling this api. I have allowed execute-api:Invoke permission to the api for the role lambda uses. I see in this post that I need to create a canonical request. I was able to come up with the below code and I still get a 403. I can't figure out what is missing and wish if a different eye can spot the problem. Please help!
"use strict";
const https = require("https");
const crypto = require("crypto");
exports.handler = async (event, context, callback) => {
try {
var attributes = {
customerId: 1,
body: { firstName: "abc", lastName: "xyz" }
};
await updateUsingApi(attributes.customerId, attributes.body)
.then((result) => {
var jsonResult = JSON.parse(result);
if (jsonResult.statusCode === 200) {
callback(null, {
statusCode: jsonResult.statusCode,
statusMessage: "Attributes saved successfully!"
});
} else {
callback(null, jsonResult);
}
})
.catch((err) => {
console.log("error: ", err);
callback(null, err);
});
} catch (error) {
console.error("error: ", error);
callback(null, error);
}
};
function sign(key, message) {
return crypto.createHmac("sha256", key).update(message).digest();
}
function getSignatureKey(key, dateStamp, regionName, serviceName) {
var kDate = sign("AWS4" + key, dateStamp);
var kRegion = sign(kDate, regionName);
var kService = sign(kRegion, serviceName);
var kSigning = sign(kService, "aws4_request");
return kSigning;
}
function updateUsingApi(customerId, newAttributes) {
var request = {
partitionKey: `MY_CUSTOM_PREFIX_${customerId}`,
sortKey: customerId,
payLoad: newAttributes
};
var data = JSON.stringify(request);
var apiHost = new URL(process.env.REST_API_INVOKE_URL).hostname;
var apiMethod = "POST";
var path = `/stage/postEndPoint`;
var { amzdate, authorization, contentType } = getHeaders(host, method, path);
const options = {
host: host,
path: path,
method: method,
headers: {
"X-Amz-Date": amzdate,
Authorization: authorization,
"Content-Type": contentType,
"Content-Length": data.length
}
};
return new Promise((resolve, reject) => {
const req = https.request(options, (res) => {
if (res && res.statusCode !== 200) {
console.log("response from api", res);
}
var response = {
statusCode: res.statusCode,
statusMessage: res.statusMessage
};
resolve(JSON.stringify(response));
});
req.on("error", (e) => {
console.log("error", e);
reject(e.message);
});
req.write(data);
req.end();
});
}
function getHeaders(host, method, path) {
var algorithm = "AWS4-HMAC-SHA256";
var region = "us-east-1";
var serviceName = "execute-api";
var secretKey = process.env.AWS_SECRET_ACCESS_KEY;
var accessKey = process.env.AWS_ACCESS_KEY_ID;
var contentType = "application/x-amz-json-1.0";
var now = new Date();
var amzdate = now
.toJSON()
.replace(/[-:]/g, "")
.replace(/\.[0-9]*/, "");
var datestamp = now.toJSON().replace(/-/g, "").replace(/T.*/, "");
var canonicalHeaders = `content-type:${contentType}\nhost:${host}\nx-amz-date:${amzdate}\n`;
var signedHeaders = "content-type;host;x-amz-date";
var payloadHash = crypto.createHash("sha256").update("").digest("hex");
var canonicalRequest = [
method,
path,
canonicalHeaders,
signedHeaders,
payloadHash
].join("/n");
var credentialScope = [datestamp, region, serviceName, "aws4_request"].join(
"/"
);
const sha56 = crypto
.createHash("sha256")
.update(canonicalRequest)
.digest("hex");
var stringToSign = [algorithm, amzdate, credentialScope, sha56].join("\n");
var signingKey = getSignatureKey(secretKey, datestamp, region, serviceName);
var signature = crypto
.createHmac("sha256", signingKey)
.update(stringToSign)
.digest("hex");
var authorization = `${algorithm} Credential=${accessKey}/${credentialScope}, SignedHeaders=${signedHeaders}, Signature=${signature}`;
return { amzdate, authorization, contentType };
}

Cannot view data on page after adding API requests nodejs

I've added API requests which would retrieve data from MongoDB I believe. Whilst no issues arise when running the code, the pages do not show any data; I just get an error message (retrieved from a separate piece of code) stating the page cannot be found.
This is the code from my Locations controller:
module.exports.homelist = function(req, res){
var requestOptions, path;
path = '/api/locations';
requestOptions = {
url : apiOptions.server + path,
method : "GET",
json : {},
qs : {
lng : -0.7992599,
lat : 51.378091,
maxDistance : 20
}
};
request (
requestOptions,
function(err, response, body) {
var i, data;
data = body;
if(response.statusCode === 200 && data.length){
for (i=0; i<data.length; i++) {
data[i].distance = _formatDistance(data[i].distance);
}
}
renderLocHomepage(req, res, body);
}
);
var _formatDistance = function (distance){
var numDistance, unit;
if (distance > 1){
numDistance = parseFloat(distance).toFixed(1);
unit = 'km';
} else {
numDistance = parseInt(distance * 1000,10);
unit = 'm';
}
return numDistance + unit;
}
};
var renderLocHomepage = function(req, res, responseBody){
var message;
if(!(responseBody instanceof Array)){
message = "API lookup error";
responseBody = [];
} else {
if (!responseBody.length) {
message = "No places found nearby";
}
}
res.render('location', {
title: 'Location Title 2',
pageHeader: {
title: 'fuiewhf',
strapline: 'sdjkcdivfdvdj'
},
sidebar: "efvibjdmkdsuidjvifdvhiskfbvzd",
locations: responseBody,
message: message
});
};
/* GET 'Locations List' page*/
module.exports.locationsList = function(req, res) {
res.render('locations-list', {title: 'Locations List'});
};
module.exports.locationInfo = function(req, res){
renderDetailPage(req, res);
};
var renderDetailPage = function (req, res, locDetail){
res.render('location-info', {
title: locDetail.name,
pageHeader: {title: locDetail.name},
sidebar: {
context: 'is on Loc8r because it has some accessible wifi and space to sit',
callToAction: 'If you\'ve been and you like it or don\'t, please leave a review'
},
location: locDetail
});
};
/*GET 'Location info' page */
module.exports.locationInfo = function(req, res){
var requestOptions, path;
path = "/api/locations/" + req.params.locationid;
requestOptions = {
url : apiOptions.server + path,
method : "GET",
json : {}
};
request (requestOptions, function(err, response, body) {
var data = body;
if (response.statusCode === 200){
data.coords = {
lng : body.coords[0],
lat : body.coords[1],
};
renderDetailPage(req, res, data);
} else{
_showError(req, res, response.statusCode);
}
}
);
};
module.exports.locationsListByDistance = function(req, res){
var lng = parseFloat(req.query.lng);
var lat = parseFloat(req.query.lat);
var point = {
type: "Point",
coordinates: [lng, lat]
};
var geoOptions = {
spherical: true,
maxDistance: theEarth.getRadsFromDistance(20),
num: 10
};
if ((!lng && lng!==0) || (!lat && lat!==0)) {
sendJsonResponse(res, 404, {
"message": "lng and lat query parameters are required"
});
return;
}
Location.geoNear(point, geoOptions, function(err, results, stats){
var locations = [];
if (err) {
sendJsonResponse(res, 404, err);
} else {
results.forEach(function(doc) {
locations.push({
distance: theEarth.getDistanceFromRads(doc.dis),
name: doc.obj.name,
address: doc.obj.address,
rating: doc.obj.rating,
facilities: doc.obj.facilities,
_id: doc.obj._id
});
});
sendJsonResponse(res, 200, locations);
}
});
};
var _formatDistance = function (distance){
var numDistance, unit;
if (distance > 1){
numDistance = parseFloat(distance).toFixed(1);
unit = 'km';
} else {
numDistance = parseInt(distance * 1000,10);
unit = 'm';
}
return numDistance + unit;
};
//defining a call to an API
var requestOptions = {
url: "", //define URL of API call to be made
method: "GET", //set request method
json: {},
qs: {
offset: 20
}
};
request(requestOptions, function(err, response, body) {
if(err){
console.log(err);
} else if(response.statusCode === 200) {
console.log(body);
} else {
console.log(response.statusCode);
}
});
//Create a new reusable function to get location information
var getLocationInfo = function(req, res, callback) {
var requestOptions, path;
path = "/api/locations/" + req.params.locationid;
requestOptions = {
url : apiOptions.server + path,
method : "GET",
json : {}
};
request (
requestOptions,
function(err, response, body) {
var data = body;
if (response.statusCode === 200) {
data.coords = {
lng : body.coords[0],
lat : body.coords[1]
};
callback(req, res, data);
} else {
_showError(req, res, response.statusCode);
}
}
);
};
module.exports.locationInfo = function(req, res){
getLocationInfo(req, res, function (req, res, responseData){
renderDetailPage(req, res, responseData);
});
};
In renderHomepage you have res.render('/'... but res.render takes as first parameter the name of the file you want to render (minus the file extension):
res.render('index', ...
if your homepage is under views/index.ejs for example.

Request body sometimes null

The request body sometimes (less than 1% of the time) is null when pulled into lambda. I am processing on the order of 14,000 request bodies one at a time. Any request bodies erring out have to be handled manually. Why is the body randomly coming in null?
Sample code ran from prompt (node index):
const async = require('async');
const _ = require('lodash');
const moment = require('moment');
const Client = require('node-rest-client').Client;
const fs = require('fs');
const input = require('./TestFull.json');
module.exports = () => {
const filename = `./eventfulFails-${new moment().format("YYYY-MM-DD-HHmmss")}.json`;
console.log('Start Time: ', new moment().format("HH:mm:ss"));
let failedObjects = {
events: [],
venues: [],
performers: []
};
async.parallel([
async.apply(processVenues, input.venues, failedObjects),
async.apply(processPerformers, input.performers, failedObjects)
], (lookupErr) => {
if (lookupErr) {
return console.error('Error processing venues and performers.', lookupErr);
}
console.log('Start Events: ', new moment().format("HH:mm:ss"));
async.waterfall([
async.apply(processEvents, input.events, failedObjects)
], (eventErr) => {
if (eventErr) {
console.log('Time of Failure: ', new moment().format("HH:mm:ss"));
return console.error('Error processing events.', eventErr);
}
console.log('End Time: ', new moment().format("HH:mm:ss"));
if (failedObjects.events.length || failedObjects.venues.length || failedObjects.performers.length) {
const stream = fs.createWriteStream(filename);
stream.once('open', function(fd) {
stream.write(JSON.stringify(failedObjects));
stream.end();
});
}
});
});
};
function processVenues(venues, failedObjects, callback) {
const calls = [];
for (let i = 0; i < venues.length; i++) {
const v = venues[i];
calls.push(async.apply((venue, postCallback) => {
const client = new Client();
const args = {
data: venue,
headers: {"Content-Type": "application/json"}
};
client.post('https://hm1br4yo34.execute-api.us-west-2.amazonaws.com/dev/eventful-venue', args, (data, response) => {
if (response.statusCode !== 200 && response.statusCode !== 201) {
failedObjects.venues.push({
venue,
response
});
console.log('venue status code: ', response);
console.log('venue data: ', venue);
}
return postCallback(null);
});
}, v));
}
async.waterfall(calls, callback);
}
function processPerformers(performers, failedObjects, callback) {
const calls = [];
for (let i = 0; i < performers.length; i++) {
const v = performers[i];
calls.push(async.apply((performer, postCallback) => {
const client = new Client();
const args = {
data: performer,
headers: {"Content-Type": "application/json"}
};
client.post('https://hm1br4yo34.execute-api.us-west-2.amazonaws.com/dev/eventful-performer', args, (data, response) => {
if (response.statusCode !== 200 && response.statusCode !== 201) {
failedObjects.performers.push({
performer,
response
});
console.log('performer status code: ', response);
console.log('performer data: ', performer);
}
return postCallback(null);
});
}, v));
}
async.waterfall(calls, callback);
}
function processEvents(events, failedObjects, callback) {
const calls = [];
for (let i = 0; i < events.length; i++) {
const v = events[i];
calls.push(async.apply((event, postCallback) => {
const client = new Client();
const args = {
data: event,
headers: {"Content-Type": "application/json"}
};
client.post('https://hm1br4yo34.execute-api.us-west-2.amazonaws.com/dev/eventful', args, (data, response) => {
if (response.statusCode !== 200 && response.statusCode !== 201) {
failedObjects.events.push({
event,
response
});
console.log('event status code: ', response);
console.log('event data: ', event);
}
return postCallback(null);
});
}, v));
}
async.waterfall(calls, callback);
}
if (!module.parent) {
module.exports();
}
Code of function processVenues (eventful-venue-load) that is called:
const _ = require('lodash');
const AWS = require('aws-sdk');
const async = require('async');
const sdk = require('#consultwithmikellc/withify-sdk');
const host = process.env.aurora_host;
const user = process.env.aurora_user;
const database = process.env.aurora_database;
let decryptedPassword;
const lambda = new AWS.Lambda({
region: 'us-west-2' //your region
});
class WithifyEventCreate extends sdk.Lambda {
constructor(event, context, keysToDecrypt) {
super(event, context, keysToDecrypt);
this.getLocation = this.getLocation.bind(this);
this.insertLocations = this.insertLocations.bind(this);
this.insertLocationImages = this.insertLocationImages.bind(this);
}
decryptedKey(key, value) {
switch (key) {
case 'aurora_password':
decryptedPassword = value;
break;
}
}
initializeComplete() {
this.connect(host, user, decryptedPassword, database, true);
}
connectComplete() {
async.waterfall(
[
this.getLocation,
this.insertLocations,
this.insertLocationImages
]
);
}
getLocation(callback) {
const {id: eventfulLocationID} = this.body;
this.connection.query('SELECT * FROM `Location` WHERE `eventfulLocationID` = ?',
[eventfulLocationID],
(err, results) => {
if (err) {
// error call block
return this.sendResponse(err, this.createResponse(500));
} else if (results.length === 1) {
console.log('Invoking withify-eventful-venue-update...');
lambda.invoke({
FunctionName: 'withify-eventful-venue-update',
Payload: JSON.stringify(this.event)
}, (error, data) => {
return this.sendResponse(null, JSON.parse(data.Payload));
});
} else if (results.length > 1) {
return this.sendResponse(`The location lookup produced multiple results. event:${JSON.stringify(this.body)}`, this.createResponse(500));
} else {
return callback(null);
}
}
);
}
insertLocations(callback) {
const {name: locationName, address: street, city, region_abbr: state, postal_code,
description, id: eventfulLocationID, latitude: lat, longitude: lng, withdrawn: locationWithdrawn} = this.body;
let addresses = street.concat(', ', city, ', ', state, ', ', postal_code);
if (!description.length){
var phones = "";
}else{
var re = /(([\(][0-9]{3}[\)][\s][0-9]{3}[-][0-9]{4})|([0-9]{3}[-][0-9]{3}[-][0-9]{4})|([0-9]{3}[\.][0-9]{3}[\.][0-9]{4}))/i;
this.body.found = description.match(re);
if (!this.body.found){
var phone = "";
}else{
if (!this.body.found.length){
var phone = "";
}else{
var phone = this.body.found[0];
}
}
}
this.connection.query('INSERT IGNORE INTO `Location` (`locationName`, `address`, ' +
'`phone`, `lat`, `lng`, `eventfulLocationID`, `locationWithdrawn`) VALUES (?, ?, ?, ?, ?, ?, ?)',
[locationName, addresses, phone, lat, lng, eventfulLocationID, locationWithdrawn],
(err, results) => {
if (err) {
return this.sendResponse(err, this.createResponse(500));
}
this.body.locationID = results.insertId;
return callback(null);
}
);
}
insertLocationImages(callback) {
var altText = "";
const images = _.flatten(this.body.images.map(im => {
return _.map(im.sizes, (ims, idx) => {
const title = `Image ${idx}`;
return [
this.body.locationID,
this.body.name,
ims.url,
null,
null,
this.body.id,
ims.width,
ims.height
];
});
}));
if(!images[0]){
return this.sendResponse(null, this.createResponse(201, this.body));
}
this.connection.query('INSERT IGNORE INTO `LocationImage` (`locationID`, `imageTitle`, `imageUrl`, ' +
'`imageName`, `altText`, `eventfulLocationID`, `width`, `height`) VALUES ?',
[images],
(err, results) => {
if (err) {
return this.sendResponse(err, this.createResponse(500));
} else if (results.affectedRows !== images.length) {
return this.sendResponse('The image inserts did not affect the right number' +
' of rows.', this.createResponse(500));
}
return this.sendResponse(null, this.createResponse(201, this.body));
}
);
}
}
exports.handler = (event, context) => {
const withifyEventCreate = new WithifyEventCreate(event, context, ['aurora_password']);
withifyEventCreate.initialize([decryptedPassword]);
};

Paypal express checkout in Node.js (error 10410)

I have been unable to to successfully parse the returned token to authenticate final Paypal sandbox payment process. Any help or input would be greatly appreciated :)
REF:
https://github.com/petersirka/node-paypal-express-checkout
https://developer.paypal.com/docs/classic/express-checkout/in-context/integration/
Here is the error I am returned:
ACK: 'Failure',
VERSION: '52.0',
BUILD: '22708556'
L_ERRORCODE0: '10410',
L_SHORTMESSAGE0: 'Invalid token',
L_LONGMESSAGE0: 'Invalid token.',
L_SEVERITYCODE0: 'Error'
Client side:
<form id="myContainer" method="post" action="/api/payment/payone"></form>
<script>
window.paypalCheckoutReady = function () {
paypal.checkout.setup('XXX', {
environment: 'sandbox',
container: 'myContainer'
});
};
</script>
<script src="//www.paypalobjects.com/api/checkout.js" async></script>
API:
payone: function(req, res) {
var paypal = require('paypal-express-checkout').init('username', 'password', 'signature', 'return url', 'cancel url', 'debug');
paypal.pay('20130001', 10.00, 'XX', 'USD', false, function(err, url) {
if (err) {
console.log(err);
return;
}
res.redirect(url);
});
paypal.detail( "token", "PayerID", function(err, data, invoiceNumber, price) {
if (err) {
console.log(err);
return;
}
});
}
Finally, paypal-express doc:
var parser = require('url');
var https = require('https');
var qs = require('querystring');
function Paypal(username, password, signature, returnUrl, cancelUrl, debug) {
this.username = "XXX";
this.password = "XXX";
this.solutiontype = 'Mark';
this.signature = "XXX";
this.debug = debug || false;
this.returnUrl = 'XXX';
this.cancelUrl = 'XXX';
this.url = 'https://' + 'api-3t.sandbox.paypal.com' + '/nvp'; //'https://' + (debug ? 'api-3t.sandbox.paypal.com' : 'api-3t.paypal.com') + '/nvp';
this.redirect = 'https://' + 'www.sandbox.paypal.com/cgi-bin/webscr'; //https://' + (debug ? 'www.sandbox.paypal.com/cgi-bin/webscr' : 'www.paypal.com/cgi-bin/webscr');
};
Paypal.prototype.params = function() {
var self = this;
return {
USER: self.username,
PWD: self.password,
SIGNATURE: self.signature,
SOLUTIONTYPE: self.solutiontype,
VERSION: '52.0'
};
console.log(self);
};
Paypal.prototype.detail = function(token, payer, callback) {
if (token.get !== undefined && typeof(payer) === 'function') {
callback = payer;
payer = token.get.PayerID;
token = token.get.token;
}
console.log(token);
var self = this;
var params = self.params();
params.TOKEN = token;
params.METHOD = 'GetExpressCheckoutDetails';
self.request(self.url, 'POST', params, function(err, data) {
if (err) {
callback(err, data);
return;
}
if (typeof(data.CUSTOM) === 'undefined') {
callback(data, null);
return;
}
console.log('3.3');
var custom = data.CUSTOM.split('|');
var params = self.params();
params.PAYMENTACTION = 'Sale';
params.PAYERID = payer;
params.TOKEN = token;
params.AMT = custom[1];
params.CURRENCYCODE = custom[2];
params.METHOD = 'DoExpressCheckoutPayment';
self.request(self.url, 'POST', params, function(err, data) {
if (err) {
callback(err, data);
return;
}
console.log('3.4');
callback(null, data, custom[0], custom[1]);
});
});
return self;
};
Paypal.prototype.request = function(url, method, data, callback) {
var self = this;
var params = qs.stringify(data);
if (method === 'GET')
url += '?' + params;
var uri = parser.parse(url);
var headers = {};
headers['Content-Type'] = method === 'POST' ? 'application/x-www-form-urlencoded' : 'text/plain';
headers['Content-Length'] = params.length;
var location = '';
var options = { protocol: uri.protocol, auth: uri.auth, method: method || 'GET', hostname: uri.hostname, port: uri.port, path: uri.path, agent: false, headers: headers };
var response = function (res) {
var buffer = '';
res.on('data', function(chunk) {
buffer += chunk.toString('utf8');
})
req.setTimeout(exports.timeout, function() {
callback(new Error('timeout'), null);
});
res.on('end', function() {
var error = null;
var data = '';
if (res.statusCode > 200) {
error = new Error(res.statusCode);
data = buffer;
} else
data = qs.parse(buffer);
callback(error, data);
});
};
var req = https.request(options, response);
req.on('error', function(err) {
callback(err, null);
});
if (method === 'POST')
req.end(params);
else
req.end();
return self;
};
Paypal.prototype.pay = function(invoiceNumber, amount, description, currency, requireAddress, callback) {
// Backward compatibility
if (typeof(requireAddress) === 'function') {
callback = requireAddress;
requireAddress = false;
}
var self = this;
var params = self.params();
params.PAYMENTACTION = 'Sale';
params.AMT = prepareNumber(amount);
params.RETURNURL = self.returnUrl;
params.CANCELURL = self.cancelUrl;
params.DESC = description;
params.NOSHIPPING = requireAddress ? 0 : 1;
params.ALLOWNOTE = 1;
params.CURRENCYCODE = currency;
params.METHOD = 'SetExpressCheckout';
params.INVNUM = invoiceNumber;
params.CUSTOM = invoiceNumber + '|' + params.AMT + '|' + currency;
self.request(self.url, 'POST', params, function(err, data) {
if (err) {
callback(err, null);
return;
}
if (data.ACK === 'Success') {
callback(null, self.redirect + '?cmd=_express- checkout&useraction=commit&token=' + data.TOKEN);
return;
}
callback(new Error('ACK ' + data.ACK + ': ' + data.L_LONGMESSAGE0), null);
});
return self;
console.log(self);
};
function prepareNumber(num, doubleZero) {
var str = num.toString().replace(',', '.');
var index = str.indexOf('.');
if (index > -1) {
var len = str.substring(index + 1).length;
if (len === 1)
str += '0';
if (len > 2)
str = str.substring(0, index + 3);
} else {
if (doubleZero || true)
str += '.00';
}
return str;
};
exports.timeout = 10000;
exports.Paypal = Paypal;
exports.init = function(username, password, signature, returnUrl, cancelUrl, debug) {
return new Paypal(username, password, signature, returnUrl, cancelUrl, debug);
};
exports.create = function(username, password, signature, returnUrl, cancelUrl, debug) {
return exports.init(username, password, signature, returnUrl, cancelUrl, debug);
};
Usually we encounter Error 10410 if there is incorrect token or no token passed.
please make sure that you are passing the right PayPal Express token.
for information on error codes you may refer: https://developer.paypal.com/docs/classic/api/errorcodes/

Node.js Async mapLimit and memory

SOLVED, see the answer please.
I have a list of urls I fetch using request and for some reason I am unable to save more than 1720 records into my database when I try to fetch 2000 or more URL's at a time.
If I try 1000 to 2000 and 2000 to 3000, I get 3000 results in total. But when I try 1000 to 3000 or 4000 to 6000, my script stops after fetching the 1720th result.
What could be the reason for that?
I use mapLimit in order to limit concurrent connections.
app.get('/asynctest', function(req, res) {
var people = [];
for (var a = 1000; a < 3000; a++) {
people.push("http://www.example.com/" + a + "/person.html");
}
async.mapLimit(people, 20, function(url, callback) {
// iterator function
var options2 = {
url: url,
headers: {
'User-Agent': req.headers['user-agent'],
'Content-Type': 'application/json; charset=utf-8'
}
};
request(options2, function(error, response, body) {
if (!error && response.statusCode == 200) {
async.series([
// add this person into database
function(callback) {
var $ = cheerio.load(body);
var name = entities.decodeHTML($('span[itemprop="name"]').text());
new person({
name: name,
url: url
}).save();
callback();
},
function(callback) {
async.waterfall([
function(callback) {
var $ = cheerio.load(body);
var jobs = $('span[itemprop="jobtitle"]').length;
if (jobs == 0) {
console.log("no job");
var jobsArr = 0;
} else {
var jobsArr = [];
for (var aa = 0; aa < jobs; aa++) {
jobsArr.push(entities.decodeHTML($('span[itemprop="jobtitle"]').eq(aa).text()));
}
}
callback(null, jobsArr);
},
function(jobsArr, callback) {
if (jobsArr == 0) {
console.log("this person has no jobs");
} else {
async.map(jobsArr, function(jobs, callback) {
personRole.where('job_name', jobs).fetch({
require: true
}).then(function(data1) {
data1 = data1.toJSON();
person.where('url', url).fetch().then(function(data2) {
data2 = data2.toJSON();
new personPersonRole({
person_id: data2.id,
personrole_id: data1.id
}).save();
});
}).catch(function(err) {
new personRole({
job_name: jobs
}).save().then(function(data3) {
data3 = data3.toJSON();
person.where('url', url).fetch().then(function(data4) {
data4 = data4.toJSON();
new personPersonRole({
person_id: data4.id,
personrole_id: data3.id
}).save();
});
});
});
});
}
callback(null, "yes");
}
], function(err, result) {
if (err) {
console.log(err);
}
});
callback();
}
], function(err, result) {
if (err) {
console.log("err3");
}
});
} else {
console.log("err4");
}
});
callback();
});
});
EDIT #2
The following code is also problematic, adds only 1747 records and it stops after that. If I stop my node app and start again, it also stops at 1747.
var person = require('./models').person;
app.get('/asynctest', function(req, res) {
var people = [];
for (var a = 18000; a < 20000; a++) {
people.push("random url");
}
async.mapLimit(people, 20, function(url, callback) {
new person({
name: "YES",
url: url
}).save();
callback();
});
});
db.js
var knex = require('knex')({
client: 'mysql',
connection: {
host: '127.0.0.1',
port: 8889,
user: 'root',
password: 'root',
database: 'mydatabase',
charset: 'utf8'
},
pool: {
min: 0,
max: 100
}
});
var db = require('bookshelf')(knex);
module.exports = db;
models.js
var person = db.Model.extend({
tableName: 'people'
});
module.exports = {
person : person
};
EDIT #3
Okay, I think I've found the solution.
18K-18.5K - no problem
18K-19K - no problem
18K-19.7K - no problem
18K-20K - RangeError: Maximum call stack size exceeded at new Object
(native)
I just wrapped my callbacks into a wrapper, like below.
async.setImmediate(function () {
callback();
});
app.get('/async22', function(req, res) {
var people = [];
for (var a = 18000; a < 20000; a++) {
people.push("yes");
}
async.mapLimit(people, 20, function(url, callback) {
new person({
name: "YES",
url: url
}).save();
async.setImmediate(function () {
callback();
});
});
});
It was in front of my eyes all the time, actually this solution isn't unique, it's already included async library's database.
https://github.com/caolan/async#common-pitfalls-stackoverflow
Here's how you do it.
async.setImmediate(function () {
callback();
});
Example
app.get('/async22', function(req, res) {
var people = [];
for (var a = 18000; a < 20000; a++) {
people.push("yes");
}
async.mapLimit(people, 20, function(url, callback) {
new person({
name: "YES",
url: url
}).save();
async.setImmediate(function () {
callback();
});
});
});
This is still not an answer, but it is too big for the comment.
I suggest to reduce the code to some minimal example and try if it works (example is below and it works for me).
Second thing - is to add a monitoring route (see the /apptest below), so you can check if you app still works and the processing progress.
If the minimal sample works, start to gradually add more code with your logic to it and check if it still works.
The code, server.js:
var util = require('util');
var express = require('express');
var async = require('async');
var request = require('request');
var cheerio = require('cheerio');
var app = express.createServer();
app.successCount = 0;
app.errorCount = 0;
app.get('/apptest', function(req, res) {
res.send(
util.format(
'I am OK, successCount: %s, errorCount: %s',
app.successCount, app.errorCount
), 200
);
});
app.get('/asynctest', function(req, res) {
var people = [];
for (var a = 1000; a < 3000; a++) {
people.push("http://www.example.com/" + a + "/person.html");
}
async.mapLimit(people, 20, function(url, callback) {
// iterator function
var options2 = {
url: url,
headers: {
'User-Agent': req.headers['user-agent'],
'Content-Type': 'application/json; charset=utf-8'
}
};
request(options2, function(error, response, body) {
if (!error) {
console.log('success requesting: ' + options2.url);
var $ = cheerio.load(body);
app.successCount += 1;
} else {
console.log(
'error requesting: %s, error: %s, status: %s',
options2.url, error, response.statusCode
);
app.errorCount += 1;
}
callback();
});
});
});
app.listen(3000, function() {
console.log(
"Express server listening on port %d in %s mode",
app.address().port, app.settings.env
);
});
Dependencies, package.json:
{
"name": "application-name",
"version": "0.0.1",
"private": true,
"dependencies": {
"async": "^1.5.2",
"cheerio": "^0.19.0",
"express": "2.5.8",
"request": "^2.67.0"
},
"devDependencies": {}
}
Run the example as node server.js and then open http://localhost:3000/asynctest in the browser, you should see success requesting: xxxx in the console. While it is running (or when it stops running) - open http://localhost:3000/apptest to check if app is OK and how many urls are processed.

Resources