Can Node.js stream be made as coroutine? - node.js

Is there a way to make Node.js stream as coroutine.
Example
a Fibonacci numbers stream.
fibonacci.on('data', cb);
//The callback (cb) is like
function cb(data)
{
//something done with data here ...
}
Expectation
function* fibonacciGenerator()
{
fibonacci.on('data', cb);
//Don't know what has to be done further...
};
var fibGen = fibonacciGenerator();
fibGen.next().value(cb);
fibGen.next().value(cb);
fibGen.next().value(cb);
.
.
.
Take desired numbers from the generator. Here Fibonacci number series is just an example, in reality the stream could be of anything a file, mongodb query result, etc.
Maybe something like this
Make the 'stream.on' function as a generator.
Place yield inside the callback function.
Obtain generator object.
Call next and take the next value in stream.
Is it at-least possible if yes how and if not why? Maybe a dumb question :)

If you don't want to use a transpiler (e.g. Babel) or wait until async/await make it to Node.js, you can implement it yourself using generators and promises.
The downside is that your code must live inside a generator.
First, you can make a helper that receives a stream and returns a function that, when called, returns a promise for the next "event" of the stream (e.g. data).
function streamToPromises(stream) {
return function() {
if (stream.isPaused()) {
stream.resume();
}
return new Promise(function(resolve) {
stream.once('data', function() {
resolve.apply(stream, arguments);
stream.pause();
});
});
}
}
It pauses the stream when you're not using it, and resumes it when you ask it the next value.
Next, you have a helper that receives a generator as an argument, and every time it yields a promise, it resolves it and passes its result back to the generator.
function run(fn) {
var gen = fn();
var promise = gen.next().value;
var tick = function() {
promise.then(function() {
promise = gen.next.apply(gen, arguments).value;
}).catch(function(err) {
// TODO: Handle error.
}).then(function() {
tick();
});
}
tick();
}
Finally, you would do your own logic inside a generator, and run it with the run helper, like this:
run(function*() {
var nextFib = streamToPromises(fibonacci);
var n;
n = yield nextFib();
console.log(n);
n = yield nextFib();
console.log(n);
});
Your own generator will yield promises, pausing its execution and passing the control to the run function.
The run function will resolve the promise and pass its value back to your own generator.
That's the gist of it. You'd need to modify streamToPromises to check for other events as well (e.g. end or error).

class FibonacciGeneratorReader extends Readable {
_isDone = false;
_fibCount = null;
_gen = function *() {
let prev = 0, curr = 1, count = 1;
while (this._fibCount === -1 || count++ < this._fibCount) {
yield curr;
[prev, curr] = [curr, prev + curr];
}
return curr;
}.bind(this)();
constructor(fibCount) {
super({
objectMode: true,
read: size => {
if (this._isDone) {
this.push(null);
} else {
let fib = this._gen.next();
this._isDone = fib.done;
this.push(fib.value.toString() + '\n');
}
}
});
this._fibCount = fibCount || -1;
}
}
new FibonacciGeneratorReader(10).pipe(process.stdout);
Output should be:
1
1
2
3
5
8
13
21
34
55

Related

how to make node promise method sync?

I want to do some prepare work, and my other work should start after these is done, so I call these work by Q.all, but some work is async, this is what I want.
May be my code will make you understand me, in this simple example I want to do this:
call foo2 for item in array
in foo2, I wait 10 * a ms(assume this to be some completed work), and change res.
I want to foo2 is running over then console.log(res), this means all wait is over and all item is added to res. So in my example, res is change to 6.
Here is the code
var Q = require("q");
var res = 0;
function foo(a) {
res += a;
}
function foo2(a) {
// this is a simple simulation of my situation, this is not exactly what I am doing. In one word, change my method to sync is a little bit difficult
return Q.delay(10 * a).then(function() {
res += a;
});
}
// Q.all([1, 2, 3].map(foo)).done(); // yes, this is what I want, this log 6
// however, because of some situation, my work is async function such as foo2 instead of sync method.
Q.all([1, 2, 3].map(function(a) {
return foo2(a);
})).done();
console.log(res); // I want 6 instead of 0
You are mixing sync and async style of programming.
In this case, your console.log statement will be run before any promise had time to fulfill (before res was modified by them), as it it not inside a promise block.
See here how console.log will be run after promises have been resolved
var Q = require("q"),
res = 0;
function foo(a) { res += a; }
function foo2(a) {
return Q
.delay(10 * a)
.then(function() { res += a; });
}
Q.all( [1, 2, 3].map(function(a) { return foo2(a); }) )
.then(function(){ console.log(res) })
.done();

nodejs event loop, how to use nextTick correctly

I'm trying to follow exercises from [node school][1]. There is an exercise where one needs to collect three streams and only print the output when all three streams are done. Without using any 3rd party module.
Can somebody please point out why my approach is not working? It gets stuck in an infinite loop:
var http = require('http');
var concat = require('concat-stream');
var count = 3;
var str1, str2, str3;
http.get(process.argv[2], function (response) {
response.pipe(concat(function(data) {
str1 = data.toString();
--count;
}));
});
http.get(process.argv[3], function (response) {
response.pipe(concat(function(data) {
str2 = data.toString();
--count;
}));
});
http.get(process.argv[4], function (response) {
response.pipe(concat(function(data) {
str3 = data.toString();
--count;
}));
});
function foo() {
if (count > 0) {
process.nextTick(foo);
} else {
console.log(str1);
console.log(str2);
console.log(str3);
}
};
foo();
http.get() callbacks can't run until the next tick of the event loop or later. process.nextTick() puts something right at the front of the event loop, ahead of the callbacks that are already there.
Your recursive routine never stops recursing because it's waiting for those callbacks to decrement the counter but they never fire.
It might work if you swap out process.nextTick() for setImmediate(). (I didn't test that, and if you do, hey, let me know if it works or not.)
But I would say just get rid of the recursion altogether. It's not needed. You can (for example) do something like this instead:
var count = 0;
var httpGet = function (index) {
http.get(process.argv[2 + index], function (response) {
// Do stuff here
// This next bit will probably end up inside the callback provided to concat
count++;
if (count === 3) {
// Print results here
}
})
};
for (var i = 0; i < 3; i++) {
httpGet(i);
}

Wait for all query to finish and fill at the same time asynchronously

I want to fill each object of the result of a query, with other querys, and I want to do all in asynchronously way
Here is an example of the way how I do actually
var q = knex.select().from('sector');
q.then(function (sectores) {
var i = -1;
(function getDetalles(sectores) {
i++;
if(i < sectores.length){
knex.select().from('sector_detalle')
.where('sector_id', sectores[i].id)
.then(function (detalles) {
// this what i want to do asynchronously
sectores[i].sector_detalles = detalles;
console.log(sectores[i]);
getDetalles(sectores);
});
} else {
res.send({sucess: true, rows: sectores});
}
})(sectores);
});
I do some reserch and found this wait for all promises to finish in nodejs with bluebird
is close to what I want but don't know how to implement
I think you're looking for the map method that works on a promise for an array, and will invoke an asynchronous (promise-returning) callback for each of the items in it:
knex.select().from('sector').map(function(sector) {
return knex.select().from('sector_detalle')
.where('sector_id', sector.id)
.then(function(detalles) {
sector.sector_detalles = detalles;
// console.log(sector);
return sector;
});
}).then(function(sectores) {
res.send({sucess: true, rows: sectores});
});

How to chain a variable number of promises in Q, in order?

I have seen Chaining an arbitrary number of promises in Q ; my question is different.
How can I make a variable number of calls, each of which returns asynchronously, in order?
The scenario is a set of HTTP requests, the number and type of which is determined by the results of the first HTTP request.
I'd like to do this simply.
I have also seen this answer which suggests something like this:
var q = require('q'),
itemsToProcess = ["one", "two", "three", "four", "five"];
function getDeferredResult(prevResult) {
return (function (someResult) {
var deferred = q.defer();
// any async function (setTimeout for now will do, $.ajax() later)
setTimeout(function () {
var nextResult = (someResult || "Initial_Blank_Value ") + ".." + itemsToProcess[0];
itemsToProcess = itemsToProcess.splice(1);
console.log("tick", nextResult, "Array:", itemsToProcess);
deferred.resolve(nextResult);
}, 600);
return deferred.promise;
}(prevResult));
}
var chain = q.resolve("start");
for (var i = itemsToProcess.length; i > 0; i--) {
chain = chain.then(getDeferredResult);
}
...but it seems awkward to loop through the itemsToProcess in that way. Or to define a new function called "loop" that abstracts the recursion. What's a better way?
There's a nice clean way to to this with [].reduce.
var chain = itemsToProcess.reduce(function (previous, item) {
return previous.then(function (previousValue) {
// do what you want with previous value
// return your async operation
return Q.delay(100);
})
}, Q.resolve(/* set the first "previousValue" here */));
chain.then(function (lastResult) {
// ...
});
reduce iterates through the array, passing in the returned value of the previous iteration. In this case you're returning promises, and so each time you are chaining a then. You provide an initial promise (as you did with q.resolve("start")) to kick things off.
At first it can take a while to wrap your head around what's going on here but if you take a moment to work through it then it's an easy pattern to use anywhere, without having to set up any machinery.
I like this way better:
var q = require('q'),
itemsToProcess = ["one", "two", "three", "four", "five"];
function getDeferredResult(a) {
return (function (items) {
var deferred;
// end
if (items.length === 0) {
return q.resolve(true);
}
deferred = q.defer();
// any async function (setTimeout for now will do, $.ajax() later)
setTimeout(function () {
var a = items[0];
console.log(a);
// pop one item off the array of workitems
deferred.resolve(items.splice(1));
}, 600);
return deferred.promise.then(getDeferredResult);
}(a));
}
q.resolve(itemsToProcess)
.then(getDeferredResult);
The key here is to call .then() on the deferred.promise with a spliced version of the array of workitems. This then gets run after the initial deferred promise resolves, which is in the fn for the setTimeout. In a more realistic scenario, the deferred promise would get resolved in the http client callback.
The initial q.resolve(itemsToProcess) kicks things off by passing in the work items to the first call of the work fn.
I added this in hopes it would help others.
Here is a concept of a state machine defined with Q.
Suppose you have the HTTP function defined, so it returns a Q promise object:
var Q_http = function (url, options) {
return Q.when($.ajax(url, options));
}
You can define a recursive function nextState as following:
var states = [...]; // an array of states in the system.
// this is a state machine to control what url to get data from
// at the current state
function nextState(current) {
if (is_terminal_state(current))
return Q(true);
return Q_http(current.url, current.data).then(function (result) {
var next = process(current, result);
return nextState(next);
});
}
Where function process(current, result) is a function to find out what the next step would be according to the current state and the result from the HTTP call.
When you use it, use it like:
nextState(initial).then(function () {
// all requests are successful.
}, function (reason) {
// for some unexpected reason the request sequence fails in the middle.
});
I propose another solutions, which looks easier to understand to me.
You do the same as you would when chaining promises directly:
promise.then(doSomethingFunction).then(doAnotherThingFunction);
If we put that into a loop, we get this:
var chain = Q.when();
for(...) {
chain = chain.then(functionToCall.bind(this, arg1, arg2));
};
chain.then(function() {
console.log("whole chain resolved");
});
var functionToCall = function(arg1, arg2, resultFromPreviousPromise) {
}
We use function currying to use multiple arguments. In our example
functionToCall.bind(this, arg1, arg2) will return a function with one argument: functionToCall(resultFromPreviousPromise)
You do not need to use the result from the previous promise.

Does node.js support yield?

Is there any way to get generators into node.js?
I'm currently faking them with callbacks, but I have to remember to check the response of the callback inside of my generator function which creates a lot of if (callback(arg) === false) return;
I want something like in python:
for p in primes():
if p > 100: break
do_something(p)
which I'm doing in node like this:
primes(function(p) {
if (p > 100) return false;
do_something(p)
});
Maybe something like coffeescript could help?
Yes, since version 0.11. Enjoy!
http://wingolog.org/archives/2013/05/08/generators-in-v8
http://jlongster.com/A-Study-on-Solving-Callbacks-with-JavaScript-Generators
The answer is "not currently" but Marcel seems to be my hero. Lets hope this goes somewhere:
https://groups.google.com/forum/#!msg/nodejs/BNs3OsDYsYw/oCsWBw9AWC0J
https://github.com/laverdet/node-fibers
You can use generators in Node.js, but only in 0.11+. Node.js 0.12 (stable) is now available. Add --harmony_generators or --harmony to the command line parameters of node to enable it.
With Traceur, you can compile advanced JavaScript to vanilla JavaScript. You could make a loader for node.js that does this on-the-fly. Since it runs on, and compiles to vanilla JavaScript, it runs in node.js < 0.11 as well as in the browser.
Facebook has developed a lighter version that only supports generators, called Regenerator. It works similarly to Traceur.
Apparently not in the current stable version. You can however achieve the same using node-fibers + promises.
Here is my implementation:
var fiber = require('fibers');
module.exports.yield = function (promise) {
var currentFiber = fiber.current;
promise
.then(function (value) {
currentFiber.run(value);
})
.otherwise(function (reason) {
currentFiber.throwInto(reason);
});
return fiber.yield();
};
module.exports.spawn = function (makeGenerator) {
fiber(function () {
makeGenerator.apply(this, Array.prototype.slice.call(arguments, 1));
}).run();
};
And a sample code on how it works: (query.find returns a promise)
var generators = require('./utils/generators');
var query = require('./utils/query');
generators.spawn(function () {
try {
var field1 = generators.yield(query.find('user', { _id : '1' }));
var field2 = generators.yield(query.find('user', { _id : '2' }));
console.log('success', field1[0]._id, field2[0]._id);
}
catch (e) {
console.error('error', e);
}
});
You might check out wu.js at http://fitzgen.github.com/wu.js/ It has lots of interesting iterator functions.
Yes and no.
var myGen = (function () {
var i = 0;
return function () {
i++; return i; }
})();
var i;
while ((i = myGen()) < 100 ) {
do something; }
As you see, you can implement something like one using closures, but it does not have native generators.
The issue proposing generatiors in v8 has recently been accepted by v8 project member.
Please vote there to make yield come true.
Update 2014: Node does support callbacks now. The following is a post from 2010.
You should use callbacks. If the function does something asynchronously, you may also want a continuation callback (continuation is a bad word, since it also means something else, but you get my point.)
primes(function(p) {
if (p > 100) return false // i assume this stops the yielding
do_something(p)
return true // it's also better to be consistent
}, function(err) { // fire when the yield callback returns false
if (err) throw err // error from whatever asynch thing you did
// continue...
})
Updated with example code
I flipped it, so that it returns true on complete (since null, false and undefined all evaluate to false anyways).
function primes(callback) {
var n = 1, a = true;
search: while (a) {
n += 1;
for (var i = 2; i <= Math.sqrt(n); i += 1)
if (n % i == 0)
continue search;
if (callback(n)) return
}
}
primes(function(p) {
console.log(p)
if (p > 100) return true
})
We are using gnode for generators in node < 0.11.3 - https://github.com/TooTallNate/gnode
Yes Node.js and JavaScript now have both synchronous iterators (as of atleast Node v6) and asynchronous iterators (as of Node v10):
An example generator/iterator with synchronous output:
// semi-pythonic like range
function* range(begin=0, end, step=1) {
if(typeof end === "undefined") {
end = begin;
begin = 0;
}
for(let i = begin; i < end; i += step) {
yield i;
}
}
for(const number of range(1,30)) {
console.log(number);
}
A similar async generator/iterator.
const timeout = (ms=1000) => new Promise((resolve, reject) => setTimeout(resolve, ms));
async function* countSeconds(begin=0, end, step=1) {
if(typeof end === "undefined") {
end = begin;
begin = 0;
}
for(let i = begin; i < end; i += step) {
yield i;
await timeout(1000);
}
}
(async () => {
for await (const second of countSeconds(10)) {
console.log(second);
}
})();
There is a lot to explore here are some good links. I will probably update this answer with more information later:
Generators
Generator functions
Iterable Protocol
Iterator Protocol
Async Generators
Jake Archibald's Article on Async Generators
Async Iterators
for await ... of

Resources