8 logical cpu's should give *8 run time benefit? - node.js

the code i am running when using promise.all should run at about \8 the time when i am using regular "blocking" for, but the result is very similar not even half the time.
i have tried to run this 2 NODEJS programs in the same computer.
WHEN USING PROMISE.ALL
let user = { "name": "shay", "interests": ["category1", "category2", "category3", "category4", "category5", "category6", "category7", "category8", "category9", "category10"] }
let result = [];
let Companies = [];
for (i = 0; i < 3000000; i++) {
let company = { "name": "company" + i, "interest": "category" + (Math.floor(Math.random() * 10) + 1) };
Companies.push(company);
}
const successRate = () => {
return (Math.floor(Math.random() * 100) + 1) + "%"
}
const task = function (interest) {
return new Promise((resolve, reject) => {
Companies.forEach(company => {
if (company.interest == interest)
result.push({ "company": company.name, "interest": company.interest, "success": successRate() })
});
resolve('succeed!');
});
};
const run = async function () {
let tasks = user.interests.map(interest => {
return task(interest)
});
await Promise.all(tasks);
}
const start = Date.now();
run();
WHEN USING REGULAR FOR
let user = {"name":"shay","interests":["category1","category2","category3","category4","category5","category6","category7","category8","category9","category10"]}
let result =[];
let Companies = [];
for (i=0;i<3000000;i++)
{
let company = {"name":"company"+i,"interest":"category"+(Math.floor(Math.random() * 10) + 1)};
Companies.push(company);
}
const successRate=()=>{
return (Math.floor(Math.random() * 100) + 1)+"%"
}
const start = Date.now();
user.interests.forEach(interest => {
Companies.forEach(company => {
if (company.interest==interest)
result.push({"company":company.name,"interest":company.interest,"success":successRate()})
});
});
console.log("run time",Date.now()-start);
the promise.all total run time is 1314
the regular for total run time is 1430

Node.js follows a single threaded architecture, it doesn't create a new thread for a new promise - everything runs in the same thread. Asynchronous behaviour is achieved via event loop, underlying OS concurrency and libuv. If you're familiar with Nginx - it actually works pretty much the same way. So don't mix JavaScript's Promise<> with Java's Future<>.

Related

Using node.js for-loop index in a coinbase-api callback function

I am new to node.js and i am trying to make a simple script that will connect to the coinbase-api and get the current price of whatever markets are defined in the MARKET array.
The problem i am having is that the for-loop that iterates through the array is asynchronous and the callback function is not getting the correct index value for the array.
The two main solutions i have found are to use promises or force the loop to wait. I think i need to be using promises rather than forcing the for loop to wait but honestly i have failed to implement a solution either way. I have found may example of promises but i just cant seem to figure out how to implement them into my script. I would appreciate any help.
const coinbaseModule = require('coinbase-pro');
const COINBASE_URI = 'https://api-public.sandbox.pro.coinbase.com';
// const MARKET = ['BTC-USD'];
const MARKET = ['BTC-USD', 'ETH-BTC'];
let askPrice = [null, null];
let averagePrice = [null, null];
let tickerCount = null;
const getCallback = (error, response, data) =>
{
if (error)
return console.log(error);
if ((data!=null) && (data.ask!=null) && (data.time!=null))
{
askPrice[tickerCount] = parseFloat(data.ask);
if (averagePrice[tickerCount]===null)
{
averagePrice[tickerCount] = askPrice[tickerCount];
console.log(MARKET[tickerCount] + " ask price: " + askPrice[tickerCount].toFixed(6));
}
else
{
averagePrice[tickerCount] = (averagePrice[tickerCount] * 1000 + askPrice[tickerCount]) / 1001;
console.log(MARKET[tickerCount] + " ask price: " + askPrice[tickerCount].toFixed(6) + " average price: "+ averagePrice[tickerCount].toFixed(6));
}
}
}
setInterval(() =>
{
console.log('\n');
publicClient = new coinbaseModule.PublicClient(COINBASE_URI);
for (tickerCount = 0; tickerCount < MARKET.length; tickerCount++)
{
publicClient.getProductTicker(MARKET[tickerCount], getCallback);
}
}, 10000);
I was able to figure out how to use promises with trial and error from the helpful examples on the Mozilla Developer Network. I am sure i am making some mistakes but at least it is working now. Another little bonus is that i was able to remove a global.
const coinbaseModule = require('coinbase-pro');
const COINBASE_URI = 'https://api-public.sandbox.pro.coinbase.com';
// const MARKET = ['BTC-USD'];
const MARKET = ['BTC-USD', 'ETH-BTC'];
let askPrice = [null, null];
let averagePrice = [null, null];
function getProductTicker(tickerCount) {
return new Promise(resolve => {
publicClient.getProductTicker(MARKET[tickerCount],function callback(error, response, data){
if (error)
return console.log(error);
if ((data!=null) && (data.ask!=null) && (data.time!=null))
{
askPrice[tickerCount] = parseFloat(data.ask);
if (averagePrice[tickerCount]===null)
{
averagePrice[tickerCount] = askPrice[tickerCount];
console.log(MARKET[tickerCount] + " ask price: " + askPrice[tickerCount].toFixed(6));
}
else
{
averagePrice[tickerCount] = (averagePrice[tickerCount] * 1000 + askPrice[tickerCount]) / 1001;
console.log(MARKET[tickerCount] + " ask price: " + askPrice[tickerCount].toFixed(6) + " average price: "+ averagePrice[tickerCount].toFixed(6));
}
resolve();
}
});
});
}
setInterval( async () =>
{
console.log('\n');
publicClient = new coinbaseModule.PublicClient(COINBASE_URI);
for (var tickerCount = 0; tickerCount < MARKET.length; tickerCount++)
{
await getProductTicker(tickerCount);
}
}, 10000);

How to handle a long request with Express

I'm working on a simple function I have for a specific GET request triggered in the browser. The objective of this request is to make multiple queries to a mongodb (mongoose) database and then perform some calculation and structure formating on the results to send it back to the browser.
The only problem is that everything takes too long and it results in an error in the browser:
net::ERR_EMPTY_RESPONSE
to give an example of part of the function I'm trying to build here it goes:
async function getPriceByMake(makes, id) {
return new Promise(async (resolve, reject) => {
let pMakes = {};
const makesArr = Object.keys(makes);
for (let i = 0; i < makesArr.length; i++) {
console.log('Getting the docs ... ' + Math.round(i/makesArr.length*100) + '%')
const currMake = makesArr[i];
pMakes[currMake] = {};
const modelsArr = Object.keys(makes[currMake]);
for (let j = 0; j < modelsArr.length; j++) {
const currModel = modelsArr[j];
await Listing.find({ catFrom: id, model: currModel }, 'year asking', (err, docs) => {
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
});
}
}
resolve(pMakes);
});
}
In this function, if I leave the async / await out, I get an empty {} on the other end. Which is obviously not the objective.
I've been searching the web a little and was able to find an article pointing to this scheme:
Browser:
Initiates request
displays progress
Show result
WebServer:
Submit event
Checks for completion
Return result
BackEndApp:
Picks up event
Runs task
Returns results
My question is the following:
How can I do that with NodeJS and Express?
In this code:
for (let j = 0; j < modelsArr.length; j++) {
const currModel = modelsArr[j];
await Listing.find({ catFrom: id, model: currModel }, 'year asking', (err, docs) => {
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
});
}
Your await isn't working because you're passing a callback to Listing.find(). When you do that, it does NOT return a promise and therefore the await does nothing useful. You get the empty response because the await doesn't work and thus you call resolve() before there's any actual data there.
Change the code to this:
for (let j = 0; j < modelsArr.length; j++) {
const currModel = modelsArr[j];
let docs = await Listing.find({ catFrom: id, model: currModel }, 'year asking');
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
}
And, then the await will work properly.
You also should remove the return new Promise() wrapper. You don't want that. Just make the function async and use await and it will already return a promise.
Here's your function with the unnecessary promise wrapper removed:
async function getPriceByMake(makes, id) {
let pMakes = {};
const makesArr = Object.keys(makes);
for (let i = 0; i < makesArr.length; i++) {
console.log('Getting the docs ... ' + Math.round(i/makesArr.length*100) + '%')
const currMake = makesArr[i];
pMakes[currMake] = {};
const modelsArr = Object.keys(makes[currMake]);
for (let j = 0; j < modelsArr.length; j++) {
const currModel = modelsArr[j];
let docs = await Listing.find({ catFrom: id, model: currModel }, 'year asking');
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
}
}
return pMakes;
}
Then, keep in mind that whatever code sends your actual response needs to use .then() or await when calling this async function in order to get the final result.
Your best bet to speed up this code would be to refactor either your queries or your database structure or both to not have to do N * M separate queries to get your final result. That's likely where your slowness is coming from. The biggest performance gains will probably come from reducing the number of queries you have to run here to far fewer.
Depending upon your database configuration and capabilities, it might speed things up to run the inner loop queries in parallel as shown here:
async function getPriceByMake(makes, id) {
let pMakes = {};
const makesArr = Object.keys(makes);
for (let i = 0; i < makesArr.length; i++) {
console.log('Getting the docs ... ' + Math.round(i/makesArr.length*100) + '%')
const currMake = makesArr[i];
pMakes[currMake] = {};
const modelsArr = Object.keys(makes[currMake]);
await Promise.all(modelsArr.map(async currModel => {
let docs = await Listing.find({ catFrom: id, model: currModel }, 'year asking');
if (docs.length > 1) {
pMakes[currMake][currModel] = [docs];
} else {
pMakes[currMake][currModel] = {};
}
}));
}
return pMakes;
}

How to measure average execution time of a async function by running it multiple times in NodeJS?

I have a function that reads a huge file line by line and does something. I want to measure the execution time of the function in a loop to get an average execution time. I did something like below but it works only one time. It won't loop.
var start = new Date()
const preparePresets = () => {
let presets = []
fs.createReadStream(filename).pipe(jsonStream.input)
return new Promise((resolve, reject) => {
return jsonStream.on('data', ({ value }) => {
for (key in value) {
preset = value[key];
presets.push(preset)
}
resolve(presets.sort((a, b) => a.id - b.id))
})
})
}
const measureExecutionTime = async () => {
for (let index = 0; index < 10; index++) {
console.log(index)
await preparePresets()
var end = new Date() - start
executionTimes.push(end)
}
console.log(executionTimes)
}
mesureExecutionTime()
I think the problem in this:
fs.createReadStream(filename).pipe(jsonStream.input)
Try to fs.readFileSync first.
jsonStream.input - if this is same stream all the time I don't think you can pipe to it many streams simultaneously.

Proper way to make callbacks async by wrapping them using `co`?

It is 2016, Node has had nearly full ES6 support since v4, and Promises have been around since 0.12. It's time to leave callbacks in the dust IMO.
I'm working on a commander.js-based CLI util which leverages a lot of async operations - http requests and user input. I want to wrap the Commander actions in async functions so that they can be treated as promises, and also to support generators (useful for the co-prompt library I'm using for user input).
I've tried wrapping the CB with co in two ways:
1)
program.command('myCmd')
.action(program => co(function* (program) {...})
.catch(err => console.log(err.stack)) );
and
2) program.command('myCmd').action(co.wrap(function* (program) { .. }));
The problem with 1) is that the program parameter isn't passed
The problem with 2) is that errors are swallowed...
I'd really like to get this working as it yields much nicer code in my use case - involving a lot of http requests and also waiting for user input using the co-prompt library..
Is it a better option altogether perhaps to wrap program.Command.prototype.action somehow?
thanks!
I've used a bespoke version of something like co to get a db.exec function which uses yield to do database request. You can pass parameters into a generator function (I pass in a connection object - see the comment where I do it).
Here is by db.exec function that is very similar to what co does
exec(generator) {
var self = this;
var it;
debug('In db.exec iterator');
return new Promise((accept,reject) => {
debug('In db.exec Promise');
var myConnection;
var onResult = lastPromiseResult => {
debug('In db.exec onResult');
var obj = it.next(lastPromiseResult);
if (!obj.done) {
debug('db.exec Iterator NOT done yet');
obj.value.then(onResult,reject);
} else {
if (myConnection) {
myConnection.release();
debug('db.exec released connection');
}
accept(obj.value);
debug('db.exec Promise Resolved with value %d',obj.value);
}
};
self._connection().then(connection => {
debug('db.exec got a connection');
myConnection = connection;
it = generator(connection); //This passes it into the generator
onResult(); //starts the generator
}).catch(error => {
logger('database', 'Exec Function Error: ' + error.message);
reject(error);
});
});
}
the connection object also wraps by database connection object and provides a generator function ability to process the rows of the results from the database, but I won't post that here (although the example below is using it to process the rows).
Here is an example of using the exec function to run a sequence of sql
db.exec(function*(connection) {
if (params.name === ADMIN_USER) {
debug('Admin Logon');
user.name = ADMIN_DISPLAY;
user.keys = 'A';
user.uid = 0;
let sql = 'SELECT passwordsalt FROM Admin WHERE AdminID = 0';
connection.request(sql);
yield connection.execSql(function*() {
let row = yield;
if (row) {
user.nopass = (row[0].value === null);
} else {
user.nopass = false;
}
debug('Admin Password bypass ' + user.nopass.toString());
});
} else {
debug('Normal User Logon');
let sql = `SELECT u.UserID,PasswordSalt,DisplayName,AccessKey,l.LogID FROM Users u
LEFT JOIN UserLog l ON u.userID = l.userID AND DATEDIFF(D,l.LogDate,GETDATE()) = 0
WHERE u.UserName = #username`;
let request = connection.request(sql);
request.addParameter('username',db.TYPES.NVarChar,params.name);
let count = yield connection.execSql(function*() {
let row = yield;
if (row) {
user.uid = row[0].value;
user.name = row[2].value;
user.keys = (row[3].value === null) ? '' : row[3].value;
user.nopass = (row[1].value === null) ;
user.lid = (row[4].value === null) ? 0 : row[4].value;
debug('Found User with uid = %d and lid = %d, keys = %s',
user.uid, user.lid, user.keys);
}
});
if (count === 0) {
debug('Not Found User');
// couldn't find name in database
reply(false,false);
return;
}
}
if (!user.nopass) {
debug('Need a Password');
//user has a password so we must check it
passGood = false; //assume false as we go into this
let request = connection.request('CheckPassword');
request.addParameter('UserID',db.TYPES.Int,user.uid);
request.addParameter('password',db.TYPES.VarChar,params.password);
yield connection.callProcedure(function*() {
let row = yield;
if (row) {
//got a valid row means we have a valid password
passGood = true;
}
});
} else {
passGood = true;
}
if (!passGood) {
debug('Not a Good Pasword');
reply(false,true);
} else {
if (user.uid !== 0 && user.lid === 0) {
let sql = `INSERT INTO UserLog(UserID,LogDate,TimeOn,UserName) OUTPUT INSERTED.logID
VALUES(#uid,GETDATE(),GETDATE(),#username)`;
let request = connection.request(sql);
request.addParameter('uid',db.TYPES.Int,user.uid);
request.addParameter('username',db.TYPES.NVarChar,user.name);
yield connection.execSql(function*() {
let row = yield;
if (row) {
user.lid = row[0].value;
debug('Users Log Entry = %d',user.lid);
}
});
}
reply(true,user);
}
})
.catch((err) => {
logger('database','Error on logon: ' + err.message);
reply(false,false);
});
});
There is a quite simple way to do async function in Commander.js
async function run() {
/* code goes here */
}
program
.command('gettime')
.action(run);
program.parse(process.argv);

Synchronous http request in node js?

I'm looking for a simple way to perform synchronous http-requests in node.js, but it's still getting async responses ...
I've realised that node.js is recommended to async jobs, but in my case,
I need the synchronous response to call other functions that use this data, if it's null/undefined, I can't proceed with the flow...
What's the better way to do that?
Here's my code:
function callCellId(data) {
console.log("Data: " + data);
var towers = [],
rscp = [];
var request = require('sync-request');
for (var x = 0; x < data.length; x++) {
console.log("Request data: \n");
rscp[x] = data[x].rscp;
var res = request('POST', 'http://opencellid.org/cell/get?key=xxxxx&mcc=' + data[x].mcc + '&mnc=' + data[x].mnc + '&lac=' + data[x].LAC + '&cellid=' + data[x].cellID + '&format=json');
console.log("loop " + x);
data = res.getBody().toString();
console.log("rsp: " + data);
towers[x] = {
'latitude': data.lat,
'longitude': data.lon,
'rscp': rscp[x],
'signal': data.averageSignalStrength
};
}
console.log("Content for triangulation" + JSON.stringify(towers));
return towers;
}
Using async in a loop cloud be tricky.
I solved this without external libraries using generators:
LoopOperation: function() {
//define generator with the main loop
var loopIterator = function*() {
for (var index = 0; index < 10; index++) {
var result = yield asyncOperation( (res) => loopIterator.next(res)); //do something asyc and execute () => loopIterator.next() when done as callback
console.log(result);
}
}();
loopIterator.next(); //start loop
}
Since the nodejs nature is async, every time we need some sync call (like this nested request stack), we're able to use promises
"A Promise is an object representing the eventual completion or failure of an asynchronous operation"
reference
I.E:
const request = require('request-promise');
function callCellId(data) {
let towers = [];
let options = {
url: 'http://opencellid.org/cell/get',
method: 'POST',
json: true
};
data.forEach(location => {
options.body = {
key: 'YOUR_PRIVATE_KEY',
mcc: location.mcc,
mnc: location.mnc,
lac: location.lac,
cellId: location.cellID
}
request(options).then(cellInfo => {
towers.push({
latitude: cellInfo.lat,
longitude: cellInfo.lon,
rscp: location.rscp,
signal: cellInfo.averageSignalStrength
});
}).catch(err => {
console.log('Could not get cellId Info for',location);
console.log(err);
});
});
return towers;
}

Resources