I'm trying to follow exercises from [node school][1]. There is an exercise where one needs to collect three streams and only print the output when all three streams are done. Without using any 3rd party module.
Can somebody please point out why my approach is not working? It gets stuck in an infinite loop:
var http = require('http');
var concat = require('concat-stream');
var count = 3;
var str1, str2, str3;
http.get(process.argv[2], function (response) {
response.pipe(concat(function(data) {
str1 = data.toString();
--count;
}));
});
http.get(process.argv[3], function (response) {
response.pipe(concat(function(data) {
str2 = data.toString();
--count;
}));
});
http.get(process.argv[4], function (response) {
response.pipe(concat(function(data) {
str3 = data.toString();
--count;
}));
});
function foo() {
if (count > 0) {
process.nextTick(foo);
} else {
console.log(str1);
console.log(str2);
console.log(str3);
}
};
foo();
http.get() callbacks can't run until the next tick of the event loop or later. process.nextTick() puts something right at the front of the event loop, ahead of the callbacks that are already there.
Your recursive routine never stops recursing because it's waiting for those callbacks to decrement the counter but they never fire.
It might work if you swap out process.nextTick() for setImmediate(). (I didn't test that, and if you do, hey, let me know if it works or not.)
But I would say just get rid of the recursion altogether. It's not needed. You can (for example) do something like this instead:
var count = 0;
var httpGet = function (index) {
http.get(process.argv[2 + index], function (response) {
// Do stuff here
// This next bit will probably end up inside the callback provided to concat
count++;
if (count === 3) {
// Print results here
}
})
};
for (var i = 0; i < 3; i++) {
httpGet(i);
}
Related
Is there a way to make Node.js stream as coroutine.
Example
a Fibonacci numbers stream.
fibonacci.on('data', cb);
//The callback (cb) is like
function cb(data)
{
//something done with data here ...
}
Expectation
function* fibonacciGenerator()
{
fibonacci.on('data', cb);
//Don't know what has to be done further...
};
var fibGen = fibonacciGenerator();
fibGen.next().value(cb);
fibGen.next().value(cb);
fibGen.next().value(cb);
.
.
.
Take desired numbers from the generator. Here Fibonacci number series is just an example, in reality the stream could be of anything a file, mongodb query result, etc.
Maybe something like this
Make the 'stream.on' function as a generator.
Place yield inside the callback function.
Obtain generator object.
Call next and take the next value in stream.
Is it at-least possible if yes how and if not why? Maybe a dumb question :)
If you don't want to use a transpiler (e.g. Babel) or wait until async/await make it to Node.js, you can implement it yourself using generators and promises.
The downside is that your code must live inside a generator.
First, you can make a helper that receives a stream and returns a function that, when called, returns a promise for the next "event" of the stream (e.g. data).
function streamToPromises(stream) {
return function() {
if (stream.isPaused()) {
stream.resume();
}
return new Promise(function(resolve) {
stream.once('data', function() {
resolve.apply(stream, arguments);
stream.pause();
});
});
}
}
It pauses the stream when you're not using it, and resumes it when you ask it the next value.
Next, you have a helper that receives a generator as an argument, and every time it yields a promise, it resolves it and passes its result back to the generator.
function run(fn) {
var gen = fn();
var promise = gen.next().value;
var tick = function() {
promise.then(function() {
promise = gen.next.apply(gen, arguments).value;
}).catch(function(err) {
// TODO: Handle error.
}).then(function() {
tick();
});
}
tick();
}
Finally, you would do your own logic inside a generator, and run it with the run helper, like this:
run(function*() {
var nextFib = streamToPromises(fibonacci);
var n;
n = yield nextFib();
console.log(n);
n = yield nextFib();
console.log(n);
});
Your own generator will yield promises, pausing its execution and passing the control to the run function.
The run function will resolve the promise and pass its value back to your own generator.
That's the gist of it. You'd need to modify streamToPromises to check for other events as well (e.g. end or error).
class FibonacciGeneratorReader extends Readable {
_isDone = false;
_fibCount = null;
_gen = function *() {
let prev = 0, curr = 1, count = 1;
while (this._fibCount === -1 || count++ < this._fibCount) {
yield curr;
[prev, curr] = [curr, prev + curr];
}
return curr;
}.bind(this)();
constructor(fibCount) {
super({
objectMode: true,
read: size => {
if (this._isDone) {
this.push(null);
} else {
let fib = this._gen.next();
this._isDone = fib.done;
this.push(fib.value.toString() + '\n');
}
}
});
this._fibCount = fibCount || -1;
}
}
new FibonacciGeneratorReader(10).pipe(process.stdout);
Output should be:
1
1
2
3
5
8
13
21
34
55
This question already has answers here:
JavaScript closure inside loops – simple practical example
(44 answers)
Closed 7 years ago.
I am a beginner to node.js. I was trying out the examples from the 'learnyounode' tutorial. I am trying to write a program that takes three url parameters and fetches some data from those urls and displays the returned data in the order in which the urls were provided.
var http = require('http');
var bl = require('bl');
var url = [];
url[0] = process.argv[2];
url[1] = process.argv[3];
url[2] = process.argv[4];
var data = [];
var remaining = url.length;
for(var i = 0; i < url.length; i++){
http.get(url[i], function (response){
response.setEncoding('utf8');
response.pipe(bl(function (err, chunk){
if(err){
console.log(err);
}
else{
data[i] = chunk.toString();
console.log(data[i]);
remaining -= 1;
if(remaining == 0) {
for(var j = 0; j < url.length; j++){
console.log(data[j]);
}
}
}
}));
});
}
I have two console.log statements in the program. The output i get is as follows:
It'll be chunder where lets throw a ford. We're going durry where mad as a cooee
.
Shazza got us some apples with come a strides. Mad as a swag when get a dog up y
a roo. It'll be rapt piece of piss as cunning as a trackie dacks.
As cross as a bogged with watch out for the boardies. As cunning as a digger fla
min lets get some roo bar. As dry as a piker piece of piss he hasn't got a joey.
Lets throw a strides mate we're going digger.
undefined
undefined
undefined
It seems like the data is correctly fetched and stored in the 'data' array but it still displays undefined.
Any idea why this is happening?
Thanks in advance!
This is a very common issue in async programming in node.js or even in the browser. A main issue you have is that the loop variable i will not be what you want it to be some time later when the async callback is called. By then, the for loop will have run to the end of its loop and i will be at the end value for all response callbacks.
There are numerous ways to solve this. You can use a closure to close over the i value and make it uniquely available to each callback.
var http = require('http');
var bl = require('bl');
var url = [];
url[0] = process.argv[2];
url[1] = process.argv[3];
url[2] = process.argv[4];
var data = [];
var remaining = url.length;
for(var i = 0; i < url.length; i++){
// create closure here to uniquely capture the loop index
// for each separate http request
(function(index) {
http.get(url[index], function (response){
response.setEncoding('utf8');
response.pipe(bl(function (err, chunk){
if(err){
console.log(err);
}
else{
data[index] = chunk.toString();
console.log(data[index]);
remaining -= 1;
if(remaining == 0) {
for(var j = 0; j < url.length; j++){
console.log(data[j]);
}
}
}
}));
});
})(i);
}
If you do much node.js programming, you will find that you probably want to learn how to use promises because they are very, very handy for controlling the flow and sequence of async operations.
Hi I have a backbone web app using Jquery and NodeJs/mongo as the server side framework. I'm having problems with making a http get call with a foreah loop and the results of the get call being iteratively added to each row of the loop.
var eventid = this.model.get("_id");
var inPromise = $.get("/registrants/list?eventid="+eventid,null,null,"json").then(
function (result){
var temp;
var finalVal = '';
var tempfinalVal = "";
var loop = 0
percentage = 0;
$.each(result.registrants,function(index,registrant){
temp = JSON.parse(registrant.fields);
for (var key in temp) {
if(key =="Email"){
if(temp[key] != ""){
$.get("/stats/registrant?userid="+temp[key]+"&eventid="+eventid,null,null,"json").then(function(result2){
percentage = (result2.Stats.type ===undefined || result2.Stats.type ==null) ? "0": result2.Stats.type;
finalVal +=percentage+"\n";
}).fail(function(){
percentage = "0";
});
}
}else if(key =="eventid"){
loop++;
finalVal = finalVal.slice(0, - 1);
finalVal +='\n';
}
finalVal +=temp[key] + ',';
}
});
//promises.push(inPromise);
}
).done(function(finalVal){
$("#webcast-download-registrants-tn").attr("href",'data:text/csv;charset=utf-8;filename=registration.csv",'+encodeURIComponent(finalVal));
console.log("DONE");
}).fail(function(){
console.log("fail");
});
// promise.done(function () {
// console.log(" PROMISE DONE");
// });
So I have the loop through a collection and the last item of the docuemnt gets a content froma nother http call and when all is fone it will create a CSV file. The problem is that THE "DONE" text echos firts then the "CALL" text is displayed
Rick, your problem is not the simplest due to :
the need for nested asynchronous gets
the need to build each CSV data row partly synchronously, partly asynchronously.
the need for a mechanism to handle the fulfilment of multiple promises generated in the inner loop.
From what you've tried, I guess you already know that much.
One important thing to note is that you can't rely on for (var key in temp) to deliver properties in any particular order. Only arrays have order.
You might try something like this :
var url = "/stats/registrant",
data = { 'eventid': this.model.get('_id') },
rowTerminator = "\n",
fieldNames = ['firstname','lastname','email','company','score'];
function getScore(email) {
return $.get(url, $.extend({}, data, {'userid':email}), null, "json").then(function(res) {
return res.Stats ? res.Stats.type || 0 : 0;
}, function() {
//ajax failure - assume score == 0
return $.when(0);
});
}
$.get("/registrants/list", data, null, "json").then(function(result) {
var promises = [];//An array in which to accumulate promises of CSV rows
promises.push($.when(fieldNames)); //promise of CSV header row
if(result.registrants) {
$.each(result.registrants, function(index, registrant) {
if(registrant.fields) {
// Synchronously initialize row with firstname, lastname, email and company
// (omitting score for now).
var row = fieldNames.slice(0,-1).map(function(fieldName, i) {
return registrant.fields[fieldName] || '';
});
//`row` remains available to inner functions due to closure
var promise;
if(registrant.fields.Email) {
// Fetch the registrant's score ...
promise = getScore(registrant.fields.Email).then(function(score) {
//... and asynchronously push the score onto row
row.push(score);
return row;
});
} else {
//or synchronously push zero onto row ...
row.push(0);
//... and create a resolved promise
promise = $.when(row);
}
promises.push(promise);//Accumulate promises of CSV data rows (still in array form), in the correct order.
}
});
}
return $.when.apply(null, promises).then(function() {
//Join all the pieces, in nested arrays, together into one long string.
return [].slice.apply(arguments).map(function(row) {
return row.join(); //default glue is ','
}).join(rowTerminator);
});
}).done(function(str) {
$("#webcast-download-registrants-tn").attr("href",'data:text/csv;charset=utf-8;filename=registration.csv",'+encodeURIComponent(str));
console.log("DONE");
}).fail(function() {
console.log("fail");
});
partially tested
See comments in code for explanation and please ask if there's anything you don't follow.
I have seen Chaining an arbitrary number of promises in Q ; my question is different.
How can I make a variable number of calls, each of which returns asynchronously, in order?
The scenario is a set of HTTP requests, the number and type of which is determined by the results of the first HTTP request.
I'd like to do this simply.
I have also seen this answer which suggests something like this:
var q = require('q'),
itemsToProcess = ["one", "two", "three", "four", "five"];
function getDeferredResult(prevResult) {
return (function (someResult) {
var deferred = q.defer();
// any async function (setTimeout for now will do, $.ajax() later)
setTimeout(function () {
var nextResult = (someResult || "Initial_Blank_Value ") + ".." + itemsToProcess[0];
itemsToProcess = itemsToProcess.splice(1);
console.log("tick", nextResult, "Array:", itemsToProcess);
deferred.resolve(nextResult);
}, 600);
return deferred.promise;
}(prevResult));
}
var chain = q.resolve("start");
for (var i = itemsToProcess.length; i > 0; i--) {
chain = chain.then(getDeferredResult);
}
...but it seems awkward to loop through the itemsToProcess in that way. Or to define a new function called "loop" that abstracts the recursion. What's a better way?
There's a nice clean way to to this with [].reduce.
var chain = itemsToProcess.reduce(function (previous, item) {
return previous.then(function (previousValue) {
// do what you want with previous value
// return your async operation
return Q.delay(100);
})
}, Q.resolve(/* set the first "previousValue" here */));
chain.then(function (lastResult) {
// ...
});
reduce iterates through the array, passing in the returned value of the previous iteration. In this case you're returning promises, and so each time you are chaining a then. You provide an initial promise (as you did with q.resolve("start")) to kick things off.
At first it can take a while to wrap your head around what's going on here but if you take a moment to work through it then it's an easy pattern to use anywhere, without having to set up any machinery.
I like this way better:
var q = require('q'),
itemsToProcess = ["one", "two", "three", "four", "five"];
function getDeferredResult(a) {
return (function (items) {
var deferred;
// end
if (items.length === 0) {
return q.resolve(true);
}
deferred = q.defer();
// any async function (setTimeout for now will do, $.ajax() later)
setTimeout(function () {
var a = items[0];
console.log(a);
// pop one item off the array of workitems
deferred.resolve(items.splice(1));
}, 600);
return deferred.promise.then(getDeferredResult);
}(a));
}
q.resolve(itemsToProcess)
.then(getDeferredResult);
The key here is to call .then() on the deferred.promise with a spliced version of the array of workitems. This then gets run after the initial deferred promise resolves, which is in the fn for the setTimeout. In a more realistic scenario, the deferred promise would get resolved in the http client callback.
The initial q.resolve(itemsToProcess) kicks things off by passing in the work items to the first call of the work fn.
I added this in hopes it would help others.
Here is a concept of a state machine defined with Q.
Suppose you have the HTTP function defined, so it returns a Q promise object:
var Q_http = function (url, options) {
return Q.when($.ajax(url, options));
}
You can define a recursive function nextState as following:
var states = [...]; // an array of states in the system.
// this is a state machine to control what url to get data from
// at the current state
function nextState(current) {
if (is_terminal_state(current))
return Q(true);
return Q_http(current.url, current.data).then(function (result) {
var next = process(current, result);
return nextState(next);
});
}
Where function process(current, result) is a function to find out what the next step would be according to the current state and the result from the HTTP call.
When you use it, use it like:
nextState(initial).then(function () {
// all requests are successful.
}, function (reason) {
// for some unexpected reason the request sequence fails in the middle.
});
I propose another solutions, which looks easier to understand to me.
You do the same as you would when chaining promises directly:
promise.then(doSomethingFunction).then(doAnotherThingFunction);
If we put that into a loop, we get this:
var chain = Q.when();
for(...) {
chain = chain.then(functionToCall.bind(this, arg1, arg2));
};
chain.then(function() {
console.log("whole chain resolved");
});
var functionToCall = function(arg1, arg2, resultFromPreviousPromise) {
}
We use function currying to use multiple arguments. In our example
functionToCall.bind(this, arg1, arg2) will return a function with one argument: functionToCall(resultFromPreviousPromise)
You do not need to use the result from the previous promise.
In node.js, I know array.forEach is blocking.
I need to loop through an array and build a string like this:
var rarray = ['test', 'test2'];
var rstring = '';
rarray.forEach(function(e, i){
rstring = rstring + i + e;
});
return rstring;
how do i do it asynchronously?
In such a case, you probably don't need to transform this code into its async version.
Hypothetically, or for very very large arrays, heres's a way to transform your code into something asynchronous, or at least something that will get back to the main event loop periodically.
var rarray = ['test', 'test2'];
var rstring = '';
var max = rarray.length;
var current = 0;
process.nextTick(function next() {
rstring = rstring + rarray[current++];
if (current >= max) {
console.log("done", rstring);
} else {
process.nextTick(next);
}
});
In practice you'd wrap this in a function, and replace console.log by calling the completion callback.
I might add that what you are trying to do is actually called reduce.
you can write it this way (doc)
var res = array.reduce(function(prev, current, index) {
return prev + index + current ;
}, '');
doing it asynchronously could be done this way
var array = ['one', 'two'];
function reduceAsync(collection, initial, process, callback) {
var i = 0;
var res = initial;
function DO(err, result) {
if(err) return callback(err);
if(i > collection.length) return callback(null, res);
var index = i++;
var value = collection[index];
process(res, value, index, collection, DO);
}
DO(null, res);
}
reduceAsync(array, '', function(previous, current, index, collection, callback) {
setTimeout(function() {
callback(null, previous + index + current);
}, 10); // wait 10 ms
}, function finalResult(err, result) {
console.log(result);
})
or, you know, you could use async.reduce
Pascal's answer is essentially a form of cooperative multithreading (See Wikipedia: Thread).
It's hard to say without measuring, but I would guess that preemptive multithreading would have better throughput for things of this nature, because the compiler has the opportunity to do loop optimizations (not sure that would happen specifically with the code above), and the operating system is likely better at deciding exactly how often to task-switch. It looks like you can do Web Worker Threads in node.js.
Anybody have benchmarks?
If you want to use the async module with the code above, you would still end up with synchronous code. The async module allows you to avoid callback hell and manage your callbacks. It doesn't make something synchronous async. To achieve that you need to use process.nextTick like in Pascal's answer.
If you were doing some extra async processing on each item in your array and you want to aggregate the result from each operation while preserving order then you could use async.eachSeries like this:
var async = require('async');
var rarray = ['test', 'test2'];
var rstring = '', i = 0;
async.eachSeries(rarray, function(item, callback){
rstring += (i++) + item;
callback();
}, function(err){
console.log(rstring);
}
);
If you are not fussed about the order of things, then you could use async.each and it will execute your async processing function in parallel.