how to handle async call in lambda nodejs - node.js

I am creating a chess engine using nodejs in lambda but due to asynchronous call it showing timeout error on lambda every time. This is just partial part of the function. It is working fine on local nodejs console but not on the lambda. Please someone suggest something as I am new to this.
var chessjs = require('./chess');
var engine = require('uci');
var uciengine = new engine(process.env['LAMBDA_TASK_ROOT'] + '/stockfish');
var fs = require("fs");
var match;
function moveEngine() {
var curfen = "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1";
var depth = '20';
uciengine.runProcess().then(
() => {
console.log("Started.");
return uciengine.uciCommand();
}
).then(
() => {
console.log("Is Ready?");
return uciengine.isReadyCommand();
}
).then(
() => {
console.log("New game.");
return uciengine.uciNewGameCommand();
}
).then(
() => {
console.log("Setting position.");
return uciengine.positionCommand(curfen);
}
).then(
() => {
console.log('Starting position set');
console.log('Starting analysis');
return uciengine.depthLimitedGoCommand(depth, (info) => {
});
}
).then((bestmove) => {
console.log('Bestmove: ');
console.log(bestmove);
return uciengine.quitCommand();
}).then(() => {
console.log('Stopped');
response.sessionAttributes = {};
context.succeed(response);
}).done();
}
async call code
var chessjs = require('./chess');
var engine = require('uci');
var async= require('async');
var uciengine = new engine('/var/task/stockfish');
var fs = require("fs");
var match;
function moveEngine() {
var curfen = "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1";
var depth = '20';
async.auto({
runProcess: function(next, results) {
uciengine.runProcess(next,results);
},
checkUiEngineReady:['runProcess',function(next,results) {
uciengine.checkUiEngineReady(next,results);
}],
newGameCommand:['checkUiEngineReady',function(next,results) {
uciengine.newGameCommand(next,results);
}],
position:['newGameCommand',function(next,results) {
uciengine.positionCommand(curfen,next,results);
}],
godepth:['position',function(next,results) {
uciengine.depthLimitedGoCommand(depth,next,results);
}]
}, function(err, response) {
if (err) {
next(err);
} else {
console.log(response);
uciengine.quitCommand();
context.succeed(response);
}
});
}
moveEngine();
async call is giving the same error like before and i think it is probably wrong.

You can handle the Async call in Lambda using async npm module which is a utility module to handle asynchronous programming in Nodejs.
You can install the async module with npm install --save async.
async.auto function will be useful to manage the above call.
Here is an example through which you can manage your code.
async.auto({
runProcess: function(next, results) {
runProcess(next,results);
},
checkUiEngineReady:['runProcess',function(next,results) {
checkUiEngineReady(next,results);
}],
newGameCommand:['checkUiEngineReady',function(next,results) {
newGameCommand(next,results);
}]
}, function(err, response) {
if (err) {
next(err);
} else {
context.succeed(response);
}
});
Thanks

Related

Error: server instance pool gets destroyed in nested db collection functions

I have searched for the solution of the error specified in title.
MongoError: server instance pool was destroyed
I believe it is because misplacement of db.close(). But I am nesting dbo.collection and unable to get the exact solution of this error.
Firstly, I am fetching data (array of ids having status 0) from database and then I am concatenating (each app-id) them one by one with URL to get desired appUrl which will be used for crawling data one by one and then crawled data is meant to be stored into another collection of mongoDB. This process will repeat for each id in the array. But my code is having error of "server instance pool gets destroyed" before storing data into collection. I am doing misplacement of db.close() but I am unable to resolve this. Please help me resolving this error
Here is my code
///* global sitehead */
const request = require('request');
const cheerio = require('cheerio');
//const response = require('response');
const fs = require('fs');
const express = require('express');
const app = express();
var MongoClient = require('mongodb').MongoClient;
var url = "mongodb://localhost:27017/";
var dateTime = require('node-datetime');
MongoClient.connect(url, {useNewUrlParser: true}, function (err, db) {
if (err) {
throw err;
} else {
var dbo = db.db("WebCrawler");
var app_id;
var appUrl;
let arr = [];
dbo.collection("Unique_Apps").find({"Post_Status": 0}, {projection: {_id: 0, App_Id: 1}}).toArray(function (err, result)
{
// console.log(result);
if (err) {
throw err;
// console.log(err);
} else {
for (var i = 0; i < result.length; i++)
{
arr[i] = result[i];
}
arr.forEach((el) => {
app_id = el.App_Id;
//console.log(app_id);
appUrl = 'https://play.google.com/store/apps/details?id=' + app_id;
console.log(appUrl);
request(appUrl, function (error, response, html) {
if (!error && response.statusCode === 200) {
//START Crawling ###########
const $ = cheerio.load(html); //cheerio
const appTitle = $('.AHFaub');
const iconUrl = $('.T75of.sHb2Xb').attr("src");
const developedBy = $('.T32cc.UAO9ie').children().eq(0);
const category = $('.T32cc.UAO9ie').children().eq(1);
//store in database collection: "Single_App_Data_Post"
var curr_obj = {App_Id: app_id, App_Name: appTitle.text(),
Icon_Url: iconUrl, Price: "Free", Developed_By: developedBy.text(),
Category: category.text()
};
dbo.collection("Single_App_Data_Post").insertOne(curr_obj, function (err, res) {
console.log("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
if (err) {
throw err;
// console.log(err);
} else {
console.log("inserted....");
} //main else
});
dbo.collection("Unique_Apps").updateOne({App_Id: app_id}, {$set: {Post_Status: 0}}, function (err, res) {
if (err)
throw err;
console.log("1 document updated");
//dbo.close();
});
} else
{
throw error;
}
});
});
}
db.close();
});
} //else
}); //mongoClient connect db
Output
The following is a good start about how to turn callback into promises. Try to use it, execute the code block, by block, understand it and then add your updateOne/insertOne requests into it.
const request = require('request');
const cheerio = require('cheerio');
const fs = require('fs');
const express = require('express');
const app = express();
const MongoClient = require('mongodb').MongoClient;
const dateTime = require('node-datetime');
// Class used to handle the database basic interractions
class DB {
constructor() {
this.db = false;
this.url = "mongodb://localhost:27017/";
}
// Do connect to the database
connect() {
return new Promise((resolve, reject) => {
MongoClient.connect(this.url, {
useNewUrlParser: true,
}, (err, db) => {
if (err) {
console.log('error mongodb connect');
return reject(err);
}
this.db = db;
return resolve(db);
});
});
}
disconnect() {
db.close();
this.db = false;
}
getCollection(name) {
return this.db.db(name);
}
}
// Get the data from the database
function getAppsIds(dbObj) {
return new Promise((resolve, reject) => {
const dbo = dbObj.getCollection('WebCrawler');
dbo.collection('Unique_Apps').find({
'Post_Status': 0,
}, {
projection: {
_id: 0,
App_Id: 1,
}
}).toArray(function(err, result) {
if (err) {
return reject(err);
}
return resolve(result);
});
});
}
function requestPlayStore(idApp) {
return new Promise((resolve, reject) => {
const appUrl = `https://play.google.com/store/apps/details?id=${app_id}`;
request(appUrl, function(error, response, html) {
if (error || response.statusCode !== 200) {
return reject(error);
}
return resolve({
response,
html,
});
});
});
}
// Do treat one id app at a time
function treatOneIdApp(dbObj, idApp) {
return requestPlayStore(idApp)
.then(({
response,
html,
}) => {
// Perform your requests here updateOne and insertOne ...
});
}
const dbObj = new DB();
dbObj.connect()
.then(() => getAppsIds(dbObj))
.then(rets => Promise.all(rets.map(x => treatOneIdApp(dbObj, x.App_Id))))
.then(() => dbObj.disconnect())
.catch((err) => {
console.log(err);
});

Can't return data from callback function at NodeJS

When i create new variable and assign callback function, But data cannot return from callback function. Undefined is occurring at new variable.
const nedb = require('nedb');
const user = new nedb({ filename: './builds/database/users.db', autoload: true });
var s = user.find({}, function (err,docs) {
if(docs.length == 0) {
var data = false;
} else {
var data = docs;
}
return data;
});
console.log(s);
var s is undefined! ....
You are mixing up callback and Promise which are two different way to handle asynchronous calls.
I recommend you to use of Promises because they are simpler and the present and future of javascript.
Using async/await which is the next step after Promises
const user = {
find: () => ['jon', 'kalessi', 'jorah'],
};
async function getUsers() {
return (await user.find({})) || [];
}
async function myJob() {
const users = await getUsers();
console.log(users);
// Put your next code here
}
myJob();
Full promise :
const user = {
find: () => new Promise((resolve) => resolve(['jon', 'kalessi', 'jorah'])),
};
user.find({})
.then((docs = []) => {
console.log(docs);
// Put you next code here, you can use of the variable docs
})
.catch((err) => {
console.log(err);
});
Full callback :
const user = {
find: (_, callback) => callback(false, ['jon', 'kalessi', 'jorah']),
};
user.find({}, (err, docs = []) => {
if (err) {
console.log(err);
} else {
console.log(docs);
// Put you next code here, you can use of the variable docs
}
});
I think user.find returning the promise. you can do like this.
const nedb = require('nedb');
const user = new nedb({ filename: './builds/database/users.db', autoload: true });
var s = user.find({}, function (err,docs) {
if(docs.length == 0) {
var data = false;
} else {
var data = docs;
}
return data;
});
Promise.all(s)
.then(result => {
console.log(result);
});
Otherwise you can also use await Like this:
async function abc(){
const nedb = require('nedb');
const user = new nedb({ filename: './builds/database/users.db', autoload: true });
var s = await user.find({}, function (err,docs) {
if(docs.length == 0) {
var data = false;
} else {
var data = docs;
}
return data;
});
}
because await worked with async thats why i put it into async function.

I am having issues with exporting in node

I am trying to make an api endpoint for data coming from dynamoDB. I believe that I have everything connected but when I run postman to check the api (api/db) it doesn't recognize the functions from the db.js in the db.js (for routes). I have run a test on api/test and am getting the information back. Here is the code from both files:
1. This scans the database and I'm trying to export it to another file.
var AWS = require('aws-sdk');
var params = {
TableName : "iotbuttonsn",
//KeyConditionExpression: "serialNumber =:serialNumber",
//ExpressionAttributeValues: {
// ":serialNumber":"*"
//},
ScanIndexForward: false,
Limit: 3,
Select: 'ALL_ATTRIBUTES'
};
AWS.config.update({
region: "us-east-1",
endpoint: "https://dynamodb.us-east-1.amazonaws.com"
});
var docClient = new AWS.DynamoDB.DocumentClient();
var getDatabase = (function(){
return {
scanDB: function(){
docClient.scan(params, onScan);
var onScan = function(err, data){
if (err) {
console.log(err.message);
} else {
console.log('scan success');
len = data.Items.length;
for (n=0; n<len; n++) {
clickTypes[n] = data.Items[n].payload.clickType;
serialNums[n] = data.Items[n].serialNumber;
}
}
};
},
clickTypes: [],
serialNums: []
};
})();
module.exports = getDatabase;
2. This is where I'm trying to input but db.scanDB() isn't working:
var router = require('express').Router();
var db = require('../routes/db.js');
router.get('/', function(req, res){
db.scanDB();
buttons =
[
iot_buttonOne = {
serialNum: db.serialNum[0],
clickType: db.clickTypes[0]
},
iot_buttonTwo = {
serialNum: db.serialNum[1],
clickType: db.clickTypes[1]
}
]
.then(
function scanSuccess(data){
res.json(data);
},
function scanError(err){
res.send(500, err.message);
}
);
});
module.exports = router;
Change your db.scan() function to properly return an asynchronous result:
// db.js
module.exports = {
scanDB: function(cb){
docClient.scan(params, function(err, data) {
var clickTypes = [], serialNums = [];
if (err) {
console.log(err.message);
cb(err);
} else {
console.log('scan success');
len = data.Items.length;
for (n=0; n<len; n++) {
clickTypes[n] = data.Items[n].payload.clickType;
serialNums[n] = data.Items[n].serialNumber;
}
cb(null, {clickTypes, serialNums});
}
});
}
};
Then, when you use it:
var db = require('../routes/db.js');
db.scanDB(function(err, data) {
if (!err) {
// data.clickTypes
// data.serialNums
} else {
// process error
}
});
It really does not good to put the scanDB result on the DB object the way you were doing because there was no way for the caller to know when the asynchronous operation was done. So, since you have to provide some notification for the caller when the async operation is done (either via callback or promise), you may as well just pass the results there too.
Also, the .then() handler in your router.get(...) handler does not belong there. I don't know why it's there at all as there are no promises involved in the code you show. Perhaps a cut/paste error when creating the question?
Note, I removed the IIFE from your getDatabase() definition since there was no benefit to it other than a little more complicated code.

Mockgoose: how to simulate a error in mongoose?

I am trying to do unit test around a mongoose powered application.
While mockgoose does a great work at simulating mongoose so I can test around it, I didn t find a way to push it to fail a call, so I can test the error handling logic.
Is it a supported use case? Or should I find another framework?
Code:
var mongoose = require('mongoose');
var Test = {},
Doc = require('./model/Doc.js');
var dbURL = 'mongodb://127.0.0.1/',
dbName = 'Test';
function connect(callback) {
Test = mongoose.createConnection(dbURL + dbName); //<-- Push this to fail
Test.on('error', (err) => {
callback(err);
});
Test.once('open', function () {
callback();
});
}
Test:
var chai = require('chai'),
expect = chai.expect,
util = require('util');
var config = require('./config.json');
var proxyquire = require('proxyquire').noPreserveCache();
var sinon = require('sinon');
var mongoose = require('mongoose');
var mockgoose = require('mockgoose');
describe('test', () => {
describe('Connect', () => {
beforeEach((callback) => {
mockgoose(mongoose).then(() => {
callback();
});
});
it('Expect to connect to the database', (done) => {
var stub = {
mongoose: mongoose
},
test = proxyquire('./../test.js', {
'mongoose': stub.mongoose
});
test.connect((err) => {
try {
expect(err).to.not.be.ok;
done();
} catch(err) {
done(err);
}
});
it('Expect to throw error when failing to connect to the database', (done) => {
var stub = {
mongoose: mongoose
},
medical = proxyquire('./../medical.js', {
'mongoose': stub.mongoose
});
medical.connect((err) => {
try {
expect(err).to.be.ok;
done();
} catch(err) {
done(err);
}
});
});
});
});
Result in:
Connect
✓ Expect to connect to the database
1) Expect to throw error when failing to connect to the database
1 passing (234ms)
1 failing
1) Medical Connect Expect to throw error when failing to connect to the database:
AssertionError: expected undefined to be truthy
I ended up stubing mongoose.createConnection (so the generator) to return a object which call the error.
let stub = {
mongoose: {
createConnection: () => {
return {
on: (eventName, callback) => {
if(eventName === 'error') {
callback('Medical Error');
}
},
once: () => {}
}
}
}
},
medical = proxyquire('./../medical.js', {
'mongoose': stub.mongoose
});

How correct cover this test case in Hapi.js Lab?

I have the file upload POST point in my Hapi.Js server.
Here is code:
server.route([{
method: 'PUT',
path: '/upload/{id}',
config: {
handler: function(req,res) {
async.waterfall([
function checkEntityInDbExists(req.params.id,callback) {
...
callback(null, entityId);
},
function uploadPictureToAWS(entityId, callback) {
...
callback(null, imageLink);
},
function savePictureLinkInDbEntity(entityId, callback) {
...
callback(null, imageLink);
}
], function(err, result) {
if (err) {
return reply(err);
}
return reply(result);
});
}
}
}]);
How correctly cover the case "should return the uploaded image path" for this code/point without hitting DB and AWS?
I think you might need a package such as proxyquire, to help you mock out methods and make them return valid results so your logic can continue.
Example usage (from Async-Hapi-Test-Example):
var assert = require("assert");
var chai = require("chai");
var sinon = require("sinon");
var sinonChai = require("sinon-chai");
var proxyquire = require("proxyquire").noCallThru();
var expect = chai.expect;
chai.should();
chai.use(sinonChai);
describe("Testing route index", function() {
var sut;
var db;
var aws;
beforeEach(function() {
db = {
check: sinon.spy(),
savePic: sinon.spy(function(){ return "a link?"; })
}
aws = {
upload: sinon.spy()
}
sut = proxyquire('./index', {"./db": db, "./aws": aws});
});
describe("upload", function() {
it("should pass", function(done){
var request = {
params: {
id: 9001
}
}
var reply = function(results) {
results.should.equal('a link?');
db.check.should.been.calledOnce;
db.savePic.should.been.calledOnce;
aws.upload.should.been.calledOnce;
done();
}
sut[0].config.handler(request, reply);
});
});
});

Resources