I have the following code with http module:
function PostCode(stream) {
// Build the post string from an object
console.log("Start upload: " + stream);
console.log("A");
var post_data = JSON.stringify({
'stream' : stream,
});
console.log("B");
// An object of options to indicate where to post to
var post_options = {
'host': 'myserver',
'port': '5000',
'path': '/upload',
'method': 'POST',
'headers': {
'Content-Type': 'application/json'
}
};
// Set up the request
console.log("C");
var post_req = http.request(post_options, function(res) {
console.log("D");
res.setEncoding('utf8');
console.log(post_options);
res.on('data', function (chunk) {
console.log('Response: ' + chunk);
});
});
// post the data
post_req.write(post_data,'utf8');
post_req.end();
}
I execute the postCode function 1000~ times in order to backup my filesystem.
The probelm is that the callback isn't executed, I see a sequence output of:
A
B
C
A
B
C
and so on.. without D.
Just when all the postCode were executed, the callback is starting to run.
How I exceute the callback parallel? So that also D will be printed?
EDIT:
this is the new question, hope it clear. I still don't understand how to fix this issue:
The problem is that I have a loop which call to function A.
In this function, there is a codeblock with execute a function with callback, let say that the callback is B.
Now, B call to other function C.
Meaning:
`function main(){
for (int i = 0; i<5; i++){
console.log("main" + i);
A();
}
}
function A(){
// do some stuff
var u = http.request(o, function (res){
console.log("A" + i);
B();
})
}
function B(){
//do some stuff
var u = http.request(o, function (res){
console.log("B" + i);
C();
})
}
function C(){
console.log("C" + i);
}
I see that C() callback is waiting until all A loop is finish, and just then execute.
In this case, that what I would see:
main 0
A 0
B 0
main 1
A 1
B 1
main 2
A 2
B 2
main 3
A 3
B 3
main 4
A 4
B 4
C 0
C 1
C 2
C 3
C 4
How can I fix it, so C would be printed after main, a, and b?
Your code misses the part where PostCode() is called, but it's probably something like a for-loop. As you already found out, that only fills the event queue which only gets executed when no other code is running.
You might try something like this:
function postNextStream() {
if (streams.length > 0)
PostCode(streams.shift());
}
var streams = [];
for (*however you find the stuff you'd like to post*)
streams.push(stream);
postNextStream();
Then you set an listener for the "end" event on the ClientRequset returned from http.request() and call postNextStream() from there.
This makes all the requests sequentially. If you want some kind of parallelism, you have to implement a more sophisticated queue management in postNextStream().
Related
Is there a way to make Node.js stream as coroutine.
Example
a Fibonacci numbers stream.
fibonacci.on('data', cb);
//The callback (cb) is like
function cb(data)
{
//something done with data here ...
}
Expectation
function* fibonacciGenerator()
{
fibonacci.on('data', cb);
//Don't know what has to be done further...
};
var fibGen = fibonacciGenerator();
fibGen.next().value(cb);
fibGen.next().value(cb);
fibGen.next().value(cb);
.
.
.
Take desired numbers from the generator. Here Fibonacci number series is just an example, in reality the stream could be of anything a file, mongodb query result, etc.
Maybe something like this
Make the 'stream.on' function as a generator.
Place yield inside the callback function.
Obtain generator object.
Call next and take the next value in stream.
Is it at-least possible if yes how and if not why? Maybe a dumb question :)
If you don't want to use a transpiler (e.g. Babel) or wait until async/await make it to Node.js, you can implement it yourself using generators and promises.
The downside is that your code must live inside a generator.
First, you can make a helper that receives a stream and returns a function that, when called, returns a promise for the next "event" of the stream (e.g. data).
function streamToPromises(stream) {
return function() {
if (stream.isPaused()) {
stream.resume();
}
return new Promise(function(resolve) {
stream.once('data', function() {
resolve.apply(stream, arguments);
stream.pause();
});
});
}
}
It pauses the stream when you're not using it, and resumes it when you ask it the next value.
Next, you have a helper that receives a generator as an argument, and every time it yields a promise, it resolves it and passes its result back to the generator.
function run(fn) {
var gen = fn();
var promise = gen.next().value;
var tick = function() {
promise.then(function() {
promise = gen.next.apply(gen, arguments).value;
}).catch(function(err) {
// TODO: Handle error.
}).then(function() {
tick();
});
}
tick();
}
Finally, you would do your own logic inside a generator, and run it with the run helper, like this:
run(function*() {
var nextFib = streamToPromises(fibonacci);
var n;
n = yield nextFib();
console.log(n);
n = yield nextFib();
console.log(n);
});
Your own generator will yield promises, pausing its execution and passing the control to the run function.
The run function will resolve the promise and pass its value back to your own generator.
That's the gist of it. You'd need to modify streamToPromises to check for other events as well (e.g. end or error).
class FibonacciGeneratorReader extends Readable {
_isDone = false;
_fibCount = null;
_gen = function *() {
let prev = 0, curr = 1, count = 1;
while (this._fibCount === -1 || count++ < this._fibCount) {
yield curr;
[prev, curr] = [curr, prev + curr];
}
return curr;
}.bind(this)();
constructor(fibCount) {
super({
objectMode: true,
read: size => {
if (this._isDone) {
this.push(null);
} else {
let fib = this._gen.next();
this._isDone = fib.done;
this.push(fib.value.toString() + '\n');
}
}
});
this._fibCount = fibCount || -1;
}
}
new FibonacciGeneratorReader(10).pipe(process.stdout);
Output should be:
1
1
2
3
5
8
13
21
34
55
I have multiple HTTP requests in a nodejs app that each returns a word of a sentence. The replies will come at different times, so I'm saving them in a dictionary, with the key being the original sentence's word index. Problem is, when I access the request object, I only get the last one.
var completed_requests = 0;
sentence = req.query.sentence;
sentence = "sentence to be translated"
responses=[];
words = sentence.split(" ");
for(j=0;j<words.length;j++){
var word = words[j];
var data={
word:word
};
var options = {
url: 'example.com',
form:data,
index:j
};
request.post(options, function(err,httpResponse,body){
options = options;
if(!err){
responses.push({options.index: body});
completed_requests+=1;
if(completed_requests==words.length){
var a="";
for(var k=0;k<words.length;k++){
a+=responses[k]+" ";
}
res.render('pages/index', { something: a });
}
}
else{
//err
}
});
}
Basically, when I access the object.index object, the object returned isn't the one used for the original request, but the last one (for some reason). How should I resolve this?
When we take a look at how the code is evaluated by JavaScript due to it's async nature in node.js the problem becomes obvious:
For the first word the loop for(j=0;j<words.length;j++){ is executed.
The value of j is assigned to options.index. For the loop run this options.index has now the value 0.
request.post(options, function(err,httpResponse,body){ is executed but the callback handler will be invoked later.
For the first word the loop for(j=0;j<words.length;j++){ is executed.
The value of j is assigned to options.index. options.index has now the value 1.
request.post(options, function(err,httpResponse,body){ is executed but the callback handler will be invoked later.
The problem becomes obvious now since no new options objects are created but the value of j is assigned to options.index in every loop run. When the first callback handler is invoked options.index has the value words.length - 1.
To fix the problem we will wrap creating the options object in a function executeRequest
var completed_requests = 0;
sentence = req.query.sentence;
sentence = "sentence to be translated"
responses=[];
words = sentence.split(" ");
for(j=0;j<words.length;j++){
var word = words[j];
var data={
word:word
};
function executeRequest(url, form, index) {
var options = {
url: url,
form: form,
index: index
};
request.post(options, function(err,httpResponse,body){
// options = options; Superfluous
if(!err){
responses.push({ [index]: body});
completed_requests+=1;
if(completed_requests==words.length){
var a="";
for(var k=0;k<words.length;k++){
a+=responses[k]+" ";
}
res.render('pages/index', { something: a });
}
}
else{
//err
}
});
}
executeRequest('example.com', data, j);
}
A good read about scoping and hoisting in JavaScript can be found here http://www.adequatelygood.com/JavaScript-Scoping-and-Hoisting.html
You need to use an async routine such as forEach or map, also I suggest you read up on the async nature of node to help understand how to handle callbacks for io.
Is there a less-nested way to achieve the following with request-promise:
r = require('request-promise');
r(url1).then(function(resp1) {
// Process resp 1
r(url2 + 'some data from resp1').then(function(resp2) {
// Process resp 2
// .....
});
});
Each request is dependent on the result of the last, and so they need to be sequential. However, some of my logic requires up to five sequential requests and it causes quite the nested nightmare.
Am I going about this wrong?
You can return a Promise in the onFulfilled function provided to Promise.then:
r = require('request-promise');
r(url1).then(function(resp1) {
// Process resp 1
return r(url2 + 'some data from resp1');
}).then(function(resp2) {
// resp2 is the resolved value from your second/inner promise
// Process resp 2
// .....
});
This lets you handle multiple calls without ending up in a nested nightmare ;-)
Additionally, this makes error handling a lot easier, if you don't care which exact Promise failed:
r = require('request-promise');
r(url1).then(function(resp1) {
// Process resp 1
return r(url2 + 'some data from resp1');
}).then(function(resp2) {
// resp2 is the resolved value from your second/inner promise
// Process resp 2
// ...
return r(urlN + 'some data from resp2');
}).then(function(respN) {
// do something with final response
// ...
}).catch(function(err) {
// handle error from any unresolved promise in the above chain
// ...
});
I'm trying to follow exercises from [node school][1]. There is an exercise where one needs to collect three streams and only print the output when all three streams are done. Without using any 3rd party module.
Can somebody please point out why my approach is not working? It gets stuck in an infinite loop:
var http = require('http');
var concat = require('concat-stream');
var count = 3;
var str1, str2, str3;
http.get(process.argv[2], function (response) {
response.pipe(concat(function(data) {
str1 = data.toString();
--count;
}));
});
http.get(process.argv[3], function (response) {
response.pipe(concat(function(data) {
str2 = data.toString();
--count;
}));
});
http.get(process.argv[4], function (response) {
response.pipe(concat(function(data) {
str3 = data.toString();
--count;
}));
});
function foo() {
if (count > 0) {
process.nextTick(foo);
} else {
console.log(str1);
console.log(str2);
console.log(str3);
}
};
foo();
http.get() callbacks can't run until the next tick of the event loop or later. process.nextTick() puts something right at the front of the event loop, ahead of the callbacks that are already there.
Your recursive routine never stops recursing because it's waiting for those callbacks to decrement the counter but they never fire.
It might work if you swap out process.nextTick() for setImmediate(). (I didn't test that, and if you do, hey, let me know if it works or not.)
But I would say just get rid of the recursion altogether. It's not needed. You can (for example) do something like this instead:
var count = 0;
var httpGet = function (index) {
http.get(process.argv[2 + index], function (response) {
// Do stuff here
// This next bit will probably end up inside the callback provided to concat
count++;
if (count === 3) {
// Print results here
}
})
};
for (var i = 0; i < 3; i++) {
httpGet(i);
}
I have the follow NODE.JS code:
var a = [1,2,3,4,5,6]
function test(){
var v = a.pop()
if (!v) return
function uno(){
due(v, function(){
console.log(v)
})
console.log("Start:",v)
return test()
}
function due(v, cb){
setTimeout(function(){
console.log(v);
cb();
}, 5000);
}
uno();
}
test()
This is the output:
Start: 6
Start: 5
Start: 4
Start: 3
Start: 2
Start: 1
6
6
5
5
4
4
3
3
2
2
1
1
as you can see inside uno() function i call due() function with a timeout.
I have two: console.log(v) (inside uno() and due())
could somone explain me WHY when i call the callback (cb()) the v value is the same?
doing:
due(v, function(){
console.log(v)
})
the console.log will keep the v value i passed in the due() call?
Why it does not get the "global" v value on the test() function?
The callback cb() is the following function: function(){ console.log(v) } and the v is taken from the local environment that is in effect when you define the function, because it is not a parameter to the callback function (upvalue). That means, the first time you call test(), it has the value 6, the second time the value 5 etc.
You should give the parameters different name than the global variables, for example:
function due(param_v, cb){
setTimeout(function(){
console.log(param_v);
cb();
}, 500);
}
Then you might spot the difference.
Edit: this is not related to node at all, more to JavaScript (and many programming languages behave exactly the same). You should play around with it and put the callbacks etc. aside for a while.
var a
function print_a () {
// this function sees the variable named a in the "upper" scope, because
// none is defined here.
console.log(a)
}
function print_b () {
// there is no variable named "b" in the upper scope and none defined here,
// so this gives an error
console.log(b)
}
a = 1
print_a() // prints 1
// print_b() // error - b is not defined
var c = 1
function dummy () {
var c = 99
function print_c () {
// the definition of c where c is 99 hides the def where c is 1
console.log(c)
}
print_c()
}
dummy() // prints 99