Overhanding data to callbacks - node.js

Hej,
I'm experimenting with node.js and its callback mechanism. And now I wonder how to overhand data to such an anonymous callback:
var fs = require('fs');
for (var i = 0; i < 4; i++) {
console.log('Outer: ' + i);
fs.readFile('/etc/hosts', 'ascii', function(err, data) {
console.log('Inner: ' + i);
});
}
I do understand why the inner call to i returns 4 always. But how can I supply some variables to that specific readFile-callback function (so that the inner i will have the value of the outer i)? I could imagine some queue mechanism, where the callback function acts as a consumer, but as a node beginner I'd like to ask for the best practices. Thanks a lot,
mechko

With the pure Javascript (that is, not using libraries such as async):
var fs = require('fs');
var i,
reader = function (i) {
return function (err, data) {
console.log('Inner: ' + i);
}
};
for (i = 0; i < 4; i++) {
console.log('Outer: ' + i);
fs.readFile('/etc/hosts', 'ascii', reader(i));
}
Or, alternatively,
var fs = require('fs');
var i;
for (i = 0; i < 4; i++) {
console.log('Outer: ' + i);
fs.readFile('/etc/hosts', 'ascii', function (i, err, data) {
console.log('Inner: ' + i);
}.bind(null, i));
}
(moving the var i; declaration out of the for loop does not really changes anything in this specific example, it is just a code style to prevent the bugs related to Javascript, as opposed to the programming languages like Java, having only the function scope for variables - so that, in your original example, i is declared for the entire module, not just for for loop).
Alternative approach would be to use libraries such as async and underscore like this:
var _ = require('underscore'),
async = require('async'),
fs = require('fs');
async.parallel(_.map(_.range(0, 4), function (i) {
return async.waterfall.bind(null, [
fs.readFile.bind(null, '/etc/hosts', 'ascii'),
function (data, callback) {
console.log("Inner: " + i);
callback();
}
]);
}, function (err) {
if (err) {
console.log("Some read attempt failed");
} else {
console.log("Done reading");
}
});

Related

node js synchronous foreach easy example?

I want to handle nodejs asynchronous issue.
please help me with a sweet example - it ll be better for me if you able to do it by callback or like callback related thing.
Thanks
Sample examples
Foreach loop
var fs = require('fs')
var paths = ['/home' , '/root', '/var']
paths.forEach(function( path ) {
fs.lstat( path, function(err, stat) {
console.log( path, stat );
});
});
For loop
for (var i = 0, c = paths.length; i < c; i++)
{
// creating an Immiedately Invoked Function Expression
(function( path ) {
fs.lstat(path, function (error, stat) {
console.log(path, stat);
});
})( paths[i] );
// passing paths[i] in as "path" in the closure
}
Recursion
function iteratePath(paths, i, max){
if(i<max) {
return;
}else{
fs.lstat( path, function(err, stat) {
console.log( path, stat );
//Recursive call back
iteratePath(paths, i, max)
});
}
}
var paths = ['/home' , '/root', '/var']
iteratePath(paths,0, size)
You can use async
Using async/await with a forEach loop

Return value in function from a promise block

I'm trying to write a function (using WebdriverJS lib) that iterates through a list of elements, checks the names and build an xpath locator that corresponds to that name. I simplified xpath locators here, so don't pay attention.
The issues I'm facing here are:
1) Calling this function returns undefined. As far as I understand, this is because the return statement is not in its place, but:
2) Placing it in the correct place where a synchronous code would normally work, doesn't work for async promises, hence calling this function will return the same undefined, but because the return statement fires before the "driver.findElement" statement.
How should I use the return statement here, if I want to get createdTask variable as a result of calling this function?
var findCreatedTask = function() {
var createdTask;
driver.findElements(By.xpath("//div[#id='Tasks_Tab']")).then(function(tasks) {
for (var index = 1; index <= tasks.length; index++) {
driver.findElement(By.xpath("//div[#id='Tasks_Tab'][" + index + "]//div[#class='task-title']")).getText().then(function(taskTitle) {
if (taskTitle == "testName") {
createdTask = "//div[#id='Tasks_Tab'][" + index + "]";
return createdTask;
}
});
}
});
};
You could first get all the texts with promise.map and then get the position with indexOf :
var map = webdriver.promise.map;
var findCreatedTask = function() {
var elems = driver.findElements(By.xpath("//div[#id='Tasks_Tab']//div[#class='task-title']"));
return map(elems, elem => elem.getText()).then(titles => {
var position = titles.indexOf("testName") + 1;
return "//div[#id='Tasks_Tab'][" + position + "]";
});
}
Here you go, I cleaned it up a bit. This will actually return an error if one is experienced in the nested promises:
var findCreatedTask = function() {
var Promise = require('bluebird');
var createdTask;
return driver.findElements(By.xpath("//div[#id='Tasks_Tab']"))
.then(function(tasks) {
return Promise.map(tasks, function(task){
return driver.findElement(By.xpath("//div[#id='Tasks_Tab'][" + index + "]//div[#class='task-title']")).getText()
}).then(function(taskTitles){
for (let i = 0; i < taskTitles.length; i++){
if(taskTitles[i] === 'testName'){
createdTask = "//div[#id='Tasks_Tab'][" + i + "]";
return createdTask;
}
}
});
});
};
You call it using
findCreatedTask.then(function(res){
//do your thing
}).catch(function(err){
console.error(err.stack);
});
You will not be able to return the value that you want from this function because when this function returns, the value is not defined yet.
This is not a problem that you try to return the value in the wrong place, but that you try to access it at the wrong time.
You have two options: you can either return a promise from this function, or this function can take a callback that would be called when the value is available.
Examples
This is not tested but should give you an idea on how to think about it.
Promise
Version with promise:
var findCreatedTask = function (callback) {
var createdTask;
return new Promise(function (resolve, reject) {
driver.findElements(By.xpath("//div[#id='Tasks_Tab']")).then(function(tasks) {
for (let index = 1; index <= tasks.length && !createdTask; index++) {
driver.findElement(By.xpath("//div[#id='Tasks_Tab'][" + index + "]//div[#class='task-title']")).getText().then(function(taskTitle) {
if (taskTitle == "testName") {
createdTask = "//div[#id='Tasks_Tab'][" + index + "]";
resolve(createdTask);
}
});
}
});
});
};
and then you call it with:
findCreatedTask().then(function (createdTask) {
// you have your createdTask here
});
Callback
Version with callback:
var findCreatedTask = function (callback) {
var createdTask;
driver.findElements(By.xpath("//div[#id='Tasks_Tab']")).then(function(tasks) {
for (let index = 1; index <= tasks.length && !createdTask; index++) {
driver.findElement(By.xpath("//div[#id='Tasks_Tab'][" + index + "]//div[#class='task-title']")).getText().then(function(taskTitle) {
if (taskTitle == "testName") {
createdTask = "//div[#id='Tasks_Tab'][" + index + "]";
callback(null, createdTask);
}
});
}
});
};
and then you call it with:
findCreatedTask(function (err, createdTask) {
// you have your createdTask here
});
More info
You can read some other answers that explain how promises and callbacks work if you're interested to know ore about it:
A detailed explanation on how to use callbacks and promises
Explanation on how to use promises in complex request handlers
An explanation of what a promise really is, on the example of AJAX requests
An explanation of callbacks, promises and how to access data returned asynchronously

How node.js implement async-callback with single process

I don't know how node implement its amazing idea. And i have a question when use it.
I have to read four files file1.js file2.js file3.js file4.js and concat them into one big javascript file result.js. It's important to keep their order.
So it's normal for me to use readFileSync instead of readFile.
I know it's a bad solution. Anyone has a good idea to do that?
Q: Is it possible for node.js to read four files at the same time?
Hope someone can explain the principle of node.js and when process.nextTick will be fired.
A: yes it is possible for node to read 4 files at the same time.
My answer would be, it depends on your situation, for reading the files synchronously or asynchronously. If it's configuration data, or the files can be cached, I would suggest just doing it synchronously, it's easy, and it's only done once. So you won't be waiting around very much. Long operations on initialization are typical, and can make things in the long run more efficient. That being said, reading four files in order, asynchronously, so that your program can do other things in the background isn't that hard. I will work on sync and async examples of each and add an edit.
/* jshint node:true*/
var fs = require('fs');
function readFilesSync(fileNames) {
'use strict';
var results = '';
for (var i = 0; i < fileNames.length; i++) {
results += fs.readFileSync(fileNames[i]);
}
return results;
}
function readFiles(fileNames, callback) {
'use strict';
var results = '';
function readFile(index) {
if (index < fileNames.length) {
fs.readFile(fileNames[index], function (err, data) {
results += data;
readFile(index + 1);
});
} else {
callback(results);
}
}
readFile(0);
}
function readAllFilesAtOnce(fileNames, callback) {
'use strict';
var results = {};
var numFiles = fileNames.length;
function callBackWrapper() {
var resultsOrdered = '';
for (var i = 0; i < fileNames.length; i++) {
resultsOrdered += results[fileNames[i]];
}
callback(resultsOrdered);
}
function readFileAsync(fileName) {
fs.readFile(fileName, function (err, data) {
results[fileName] = data;
numFiles--;
if (numFiles === 0) {
callBackWrapper();
}
});
}
for (var i = 0; i < fileNames.length; i++) {
readFileAsync(fileNames[i]);
}
}
function doSomethingWithTheData(data) {
'use strict';
console.log('Results async: ' + data);
}
function doSomethingWithTheData2(data) {
'use strict';
console.log('Results async all at once: ' + data);
}
var fileNamesArray = ['blah.js', 'file.js', 'hello.txt'];
console.log('The results sync: ' + readFilesSync(fileNamesArray));
readFiles(fileNamesArray, doSomethingWithTheData);
readAllFilesAtOnce(fileNamesArray, doSomethingWithTheData2);
EDIT: There I added a method to read all of the files at once.
Process.nextTick does no more than process this function on the next time around the event loop. EX:
process.nextTick(function() {
console.log('never printed out');
});
while(true);
ex 2:
process.nextTick(function() {
console.log('printed last');
});
console.log('printed first');

[Node.js]loop insert 1000000 documents to mongodb via native driver, why node takes up a lot of memory?

I use the following code to loop insert 1000000 documents to mongodb,but i found node process takes up a lot of memory,my client are dead.
db.collection("batch_insert", function (err, collection) {
if (!err) {
var count = 0;
for (var i = 0; i < 1000000; i++) {
collection.insert({hello:'world', ok:'OKOKOK'}, {safe:true, serializeFunctions:false}, function (err, result) {
count++;
if (1000000 == count) {
db.close();
}
});
}
} else {
console.log(err);
}
});
Your for cycle blocks event loop. And it can't go to nextTick and handle query results until all queries sended to mongodb. You need to use asynchronous way to batch insert data.
Something like this:
var mongo = require('mongodb');
var Inserter = function (collection) {
this.collection = collection;
this.data = [];
this.maxThreads = 6;
this.currentThreads = 0;
this.batchSize = 5000;
this.queue = 0;
this.inserted = 0;
this.startTime = Date.now();
};
Inserter.prototype.add = function(data) {
this.data.push(data);
};
// Use force=true for last insert
Inserter.prototype.insert = function(force) {
var that = this;
if (this.data.length >= this.batchSize || force) {
if (this.currentThreads >= this.maxThreads) {
this.queue++;
return;
}
this.currentThreads++;
console.log('Threads: ' + this.currentThreads);
this.collection.insert(this.data.splice(0, this.batchSize), {safe:true}, function() {
that.inserted += that.batchSize;
var currentTime = Date.now();
var workTime = Math.round((currentTime - that.startTime) / 1000)
console.log('Speed: ' + that.inserted / workTime + ' per sec');
that.currentThreads--;
if (that.queue > 0) {
that.queue--;
that.insert();
}
});
}
};
var db = new mongo.Db('test', new mongo.Server('localhost', 27017, {}), {native_parser:false});
db.open(function(err, db) {
db.collection('test', function(err, collection) {
var inserter = new Inserter(collection);
setInterval(function() {
for (var i = 0; i < 5000; i++) {
inserter.add({test:'test'});
}
inserter.insert();
}, 0);
});
});
mongodb, just like any other database, takes some time to process requests. You're throwing a million requests at it, and since nothing in your code blocks, that means that at any time a whole bunch of them are going to be queued up somewhere (most likely in multiple places, with some of them inside the driver's code, others inside node's event loop). That takes more than a little bit of memory.
If the queuing didn't happen, you'd either block or drop some of the requests. There Ain't No Such Thing As A Free Lunch.

Using recursive pattern loop with node.js

ive been trying to use node.js to iterate through an array of cities and make an iterative request to google for directions on each (i then JSON.parse to abstract the drive times). I need to find a way to do this synchronously as otherwise i will just be requesting all the info from google on each city at once. I found a good pattern to use at http://tech.richardrodger.com/2011/04/21/node-js-%E2%80%93-how-to-write-a-for-loop-with-callbacks/ but cannot get the callback to work. As you can see, im using a 'show' function to test the same. My code is as follows:
var request = require('request');
var fs = require('fs');
var arr = ['glasgow','preston','blackpool','chorley','newcastle','bolton','paris','york','doncaster'];
//the function I want to call on each city from [arr]
function getTravelTime(a, b,callback){
request('https://maps.googleapis.com/maps/api/directions/json?origin='+a+'&destination='+b+'&region=en&sensor=false',function(err,res,data){
var foo = JSON.parse(data);
var duration = foo.routes[0].legs[0].duration.text;
console.log(duration);
});
};
function show(b){
fs.writeFile('testing.txt',b);
};
function uploader(i){
if( i < arr.length ){
show( arr[i],function(){
uploader(i+1);
});
}
}
uploader(0)
The problem I have is that only the first city from the array is output and the callback/iteration never proceeds. Any ideas where im going wrong please?
 
Thanks for the pointers, was clearly down to my poor understanding of callbacks in javascript. Just reading JavaScript patterns by O'Reilly and hit the 'Callback pattern' sections - doh!
For anyone who doesn't know, this is how the code will work:
var arr = ['glasgow','preston','blackpool','chorley','newcastle','bolton','paris','york','doncaster'];
function show(a,callback){
console.log(a);
callback();
}
function uploader(i){
if( i < arr.length ){
show(arr[i],
function(){
uploader(i+1)
});
};
}
uploader(0)
I was also facing issues like this, so I've written a recursive callback function which will act as a for loop but you can control when to increment. The following is that module, name as syncFor.js and include this in your program
module.exports = function syncFor(index, len, status, func) {
func(index, status, function (res) {
if (res == "next") {
index++;
if (index < len) {
syncFor(index, len, "r", func);
} else {
return func(index, "done", function () {
})
}
}
});
}
//this will be your program if u include this module
var request = require('request');
var fs = require('fs');
var arr = ['glasgow', 'preston', 'blackpool', 'chorley', 'newcastle', 'bolton', 'paris', 'york', 'doncaster'];
var syncFor = require('./syncFor'); //syncFor.js is stored in same directory
//the following is how u implement it
syncFor(0, arr.length, "start", function (i, status, call) {
if (status === "done")
console.log("array iteration is done")
else
getTravelTime(arr[i], "whatever", function () {
call('next') // this acts as increment (i++)
})
})
function getTravelTime(a, b, callback) {
request('https://maps.googleapis.com/maps/api/directions/json?origin=' + a + '&destination=' + b + '&region=en&sensor=false', function (err, res, data) {
var foo = JSON.parse(data);
var duration = foo.routes[0].legs[0].duration.text;
callback(); // call the callback when u get answer
console.log(duration);
});
};

Resources