I have to make multiple serialise call on server side in NodeJs using request module , and RxJs Observable
I have array of Url Address like ['www.google.com','facebook.com']
When I make request to call then server get busy in event loop . I want to know when my last url call gets response so that I can send response to front end
here is my code so far
const foo = Observable.create(function (observer) {
res.writeHeader(200, { "Content-Type": "text/html" });
var str = '<html><head></head><body><h1> Following are the responses: </h1><ul>'
res.write(str);
let count = 0;
addressesArray.forEach(function (element, i) {
observer.next(element);
if (some Condition) {
console.log('here i want to call complete when I get last response')
// observer.complete();
}
})
})
and this is my subscribe function
const subscription = foo.subscribe({
next : addres =>
{
request(addres , function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(response)
}
if (typeof response === "undefined" || typeof body === "undefined") {
console.log('No resp')
}
})
},
complete:()=>{console.log('done')}
})
You can start off your Observable from addressesArray with from operator, it'll complete itself when it reach the end of the array
from(addressesArray).pipe(
mergeMap(address => {
return new Observable(obs => {
request(address, function (error, response, body) {
if (!error && response.statusCode == 200)
obs.next(response)
else
obs.next('')
obs.complete()
})
})
}),
reduce((allResponses, response) => {
return allResponse += '<li>' + response + '<li/>'
}, ''),
tap(allResponses => {
res.writeHeader(200, { "Content-Type": "text/html" });
var str = '<html><head></head><body><h1> Following are the responses: </h1><ul>'
res.write(allResponses + '</ul')
res.end()
})
).subscribe()
Related
I am calling network requests in a forEach loop, each request returns a new object which I would like to push to an array that is outside the forEach. When it is called after awaiting for the network requests to complete, it returns as empty "[]".
return this.get(urlOne, function (error, response) {
if (response.statusCode === 200) {
let array = [];
stores.forEach((store) => {
this.get(urlTwo, function (error, response) {
if (response.statusCode === 200) {
array.push(response.body);
}
});
});
console.log(array);
}
});
This is just a guess but, stores.forEach contents are async thus console.log(array) is called right after stores.forEach() is calle d(not completed) meaning array is blank.
console.log(array); probably needs to be called when the requests are actually done not right after they are made.
Simplest way to test if this is true is to see the array for every push if you see the array growing with your data there is the problem.
return this.get(urlOne, function (error, response) {
if (response.statusCode === 200) {
let array = [];
stores.forEach((store) => {
this.get(urlTwo, function (error, response) {
if (response.statusCode === 200) {
array.push(response.body);
console.log(array);
}
});
});
}
});
(EDIT)
if (response.statusCode === 200)
array.push(response.body);
else if(response.statusCode === 204)
console.log(array);
Status code 204 means No Content, if this isn't being sent automatically or an equivalent, you may need to send it manual some how.
I am new in nodejs and I am using express js, I am calling REST API and want to show the response, but my problem is that the response is showing in console.log but I want to pass the body(response) in assistant.ask, where I am wrong here is my code:
var request = require('request');
let rawInput = function (assistant) {
let rawInput = assistant.getRawInput();
request.post(
'http://xxxxxxx.ngrok.io/api/v1/240/respond',
{ json: { query: rawInput } },
function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body);
assistant.ask(body);
}
else{
console.log(error);
}
}
);
I have this koa route /landing which is resulting in 404.
function* landing() {
//this.body = "response"; //1
var request = require('request');
request.post('http://url.com/resource',
{ json: { key: "post data"} },
function (error, response, body) {
if (!error && response.statusCode == 200) {
var token = body.data;
getListByToken(token, function(list){
this.body = list; //2
});
}
});
}
See comment in the top //1 - that is how you define body of a response in koa in a route. Instead of //1 I want to send response from //2 i.e. from within that request.get.
When a user is routed to /landing a post request has to fetch some data from a url. That fetched data is to be used by getListByToken to bring some other data, list, which should be sent to the user. The above code should have worked but it results in 404 Not Found response by koa.
We can use promises, simple and clean way of managing asynchronous codes.
var request = require('request-promise');
. . .
function* landing() {
try {
let response = yield request( {
method: 'POST',,
url: 'http://url.com/resource',
headers: { 'content-type': 'application/json' },
body: JSON.stringify( { json: { key: "post data"} } )
} );
this.body = yield new Promise( function( resolve, reject ) {
if ( response.statusCode == 200 ) {
getListByToken( response.body.token, function( list ) {
resolve( list );
} );
}
} );
} catch ( err ) {
/* do something with your errors */
}
}
Solved it with q. It makes koa hold response until yield happens.
var q = require('q');
function* landing() {
var deferred = q.defer();
var request = require('request');
request.post('http://url.com/resource',
{ json: { key: "post data"} },
function (error, response, body) {
if (!error && response.statusCode == 200) {
var token = body.data;
getListByToken(token, function(list){
deferred.resolve(repolist);
});
}
});
this.body = yield deferred.promise;
}
Thanks to https://stackoverflow.com/a/22159513/1128379
I'm trying a pretty complex computation in my code below. I'm trying to get the list of bugs from github in the given project using the api https://api.github.com/repos/marklogic/java-client-api/issues?page=1&per_page=10. From the list of bugs I'm trying to get each issues' corresponding events and comments from their corresponding endpoints ex: https://api.github.com/repos/marklogic/java-client-api/issues/291/events and https://api.github.com/repos/marklogic/java-client-api/issues/291/comments.
I'm using async library. I'm using waterfall function and parallel function to return a consolidated JSON for each bug such that each issue will have comment, & events in the same response for each issue. The problem is its throwing Can't set headers after they are sent error & its pointing to line 2 lines, I understand what the error is saying but I can't figure out how to fix it, because commenting out either of the offending lines results in request hang because the server is not sending the response. Please help! Thanks in advance
exports.listGitHubBugs = function(req, res) {
var _page = req.query.page || 1;
var _per_page = req.query.per_page || 25;
var finalResult = []
//console.log('url:', 'https://api.github.com/repos/marklogic/' + req.query.project + '/issues?page=' + _page + '&per_page=' + _per_page);
var options = {
url: 'https://api.github.com/repos/marklogic/' + req.query.project + '/issues?page=' + _page + '&per_page=' + _per_page,
headers: {
'User-Agent': req.query.project
},
auth: githubAuth
};
request(options, function(error, response, body) {
if (error) {
console.log(error);
res.send(error);
}
if (!error && response.statusCode === 200) {
var issues = JSON.parse(response.body)
async.waterfall([
// get comments & events for all bugs and then send the response
function(callback) {
issues.forEach(function(issue) {
// for each bug, get comments and events
async.parallel([
function(parallelCallback) {
var options = {
url: issue.events_url,
headers: {
'User-Agent': getProjectNameFromURL(issue.events_url)
},
auth: githubAuth
};
request(options, function(error, response, body) {
if (error) {
console.log('ERROR', error);
parallelCallback(error)
}
if (!error && response.statusCode === 200) {
// console.log('events:', body);
parallelCallback(null, body)
}
})
},
function(parallelCallback) {
var options = {
url: issue.comments_url,
headers: {
'User-Agent': getProjectNameFromURL(issue.comments_url)
},
auth: githubAuth
};
request(options, function(error, response, body) {
if (error) {
console.log('ERROR', error);
parallelCallback(error)
}
if (!error && response.statusCode === 200) {
// console.log('comments:', body);
parallelCallback(null, body)
}
})
}
], function(err, result) {
if (err) {
console.log('ERROR:', err);
callback(err);
}
console.log('parallel process done');
issue.events = JSON.parse(result[0]);
issue.comments = JSON.parse(result[1]);
finalResult.push(issue)
callback(null, finalResult) // offending line#1
})
}) // forEach end
}
], function(err, result) {
if (err) {
res.send(err);
}
console.log('waterfall done');
console.log(result);
res.send(result); // offending line#2
})
} // if end
}) // reqest end
}
Error
UncaughtException: Can't set headers after they are sent.
ERROR Error: Can't set headers after they are sent.
at ServerResponse.OutgoingMessage.setHeader (http.js:691:11)
at ServerResponse.res.set.res.header (/Users/sreddy/space/angularjs/BugTrack/node_modules/express/lib/response.js:524:10)
at ServerResponse.res.send (/Users/sreddy/space/angularjs/BugTrack/node_modules/express/lib/response.js:125:10)
at ServerResponse.res.json (/Users/sreddy/space/angularjs/BugTrack/node_modules/express/lib/response.js:191:15)
at /Users/sreddy/space/angularjs/BugTrack/server/api/common/common.controller.js:163:33
at /Users/sreddy/space/angularjs/BugTrack/server/api/common/common.controller.js:153:29
at /Users/sreddy/space/angularjs/BugTrack/node_modules/async/lib/async.js:254:17
at done (/Users/sreddy/space/angularjs/BugTrack/node_modules/async/lib/async.js:135:19)
at /Users/sreddy/space/angularjs/BugTrack/node_modules/async/lib/async.js:32:16
at /Users/sreddy/space/angularjs/BugTrack/node_modules/async/lib/async.js:251:21
Final working code
exports.listGitHubBugs = function(req, res) {
var _page = req.query.page || 1;
var _per_page = req.query.per_page || 25;
var finalResult = []
//console.log('url:', 'https://api.github.com/repos/marklogic/' + req.query.project + '/issues?page=' + _page + '&per_page=' + _per_page);
var options = {
url: 'https://api.github.com/repos/marklogic/' + req.query.project + '/issues?page=' + _page + '&per_page=' + _per_page,
headers: {
'User-Agent': req.query.project
},
auth: githubAuth
};
request(options, function(error, response, body) {
if (error) {
console.log(error);
return res.send(error);
}
if (!error && response.statusCode === 200) {
var issues = JSON.parse(response.body)
async.waterfall([
// get events and comments for all bugs and return the final processes list of bugs
function getEventsAndCommentsForAllBugs(callback) {
issues.forEach(function getEventsAndComments(issue, index) {
// for each bug, get comments and events
async.parallel([
function getEvents(parallelCallback) {
var options = {
url: issue.events_url,
headers: {
'User-Agent': getProjectNameFromURL(issue.events_url)
},
auth: githubAuth
};
request(options, function(error, response, body) {
if (error) {
console.log('ERROR', error);
parallelCallback(error)
}
if (response.statusCode === 200) {
// console.log('events:', body);
parallelCallback(null, body)
}
})
},
function getComments(parallelCallback) {
var options = {
url: issue.comments_url,
headers: {
'User-Agent': getProjectNameFromURL(issue.comments_url)
},
auth: githubAuth
};
request(options, function(error, response, body) {
if (error) {
console.log('ERROR', error);
parallelCallback(error)
}
if (response.statusCode === 200) {
// console.log('comments:', body);
parallelCallback(null, body)
}
})
}
], function attachEventsAndComments(err, result) {
if (err) {
console.log('ERROR:', err);
callback(err);
}
console.log('parallel process done');
issue.eventList = JSON.parse(result[0]);
issue.commentList= JSON.parse(result[1]);
finalResult.push(issue)
if (index === (issues.length - 1)) {
callback(null, finalResult)
}
//
})
}) // forEach end
}
], function processedBugs(err, result) {
if (err) {
res.send(err);
}
console.log('waterfall done');
console.log(result);
res.send(result);
})
} // if end
}) // reqest end
}
could you provide a complete working example of the code, something we can try.
this said, there are several errors in this source code.
Onthe first request, if an error occurs, you write it in app.response,, but you don t stop execution. Thus, if an error occurs, you ll write twice the response object.
You should do
if (error) {
console.log(error);
return res.send(error);
}
instead of
if (error) {
console.log(error);
res.send(error);
}
Then, this can be changed
if (!error && response.statusCode === 200) {
to
if (response.statusCode === 200) {
Same mistake occurs while fetching issues events and comments, please consider to fix it.
And also in the final callback of async.//
And in the final callback of async.waterfall.
finally, i suggest you to make use of named functions. That would help you to debug by providing more meaningfull error stack trace.
For example instead of doing,
async.prallel([function(){/* code here*/}]);
You would write
async.parallel([function nameOfTheTask(){/* code here*/}]);
consider also to use a linter such as eslint, several missing ; could break your code, see http://eslint.org/
I figured it out. I was calling the parellelCallback() for every iteration instead of calling it at the end of the loop. Simple if condition was all that was required.
if (index === (issues.length-1)) {
callback(null, finalResult)
}
Can somebody see in my code why the variable oauthToken is defined on the server but not defined when returned to the client in the result of Meteor.call
I make a call to initiate a post request on the server
The body is parsed and I store a value into the variable oauthToken
This prints out on the server but does not print out on the client in my 'result'
Is this because the client is running a simulation? Can we do a 'return' in an asynchronous function?
Server.js
Meteor.methods({
getGoodreads: function () {
request.post('http://www.goodreads.com/oauth/request_token', {oauth:{
consumer_key: '89hdg8pEoMzRdg',
consumer_secret: 'dfgdfgHthtdtjtt' }}, function (error, response, body) {
if (!error && response.statusCode == 200) {
var a = querystring.parse(body)
oauthToken = a.oauth_token
console.log(oauthToken); //prints value i need
return oauthToken
}else{
console.log('there is an error ' + error);
}
});
}
});
client.js
Template.profile.events({
'click #goodreads': function (event) {
event.preventDefault();
Meteor.call('getGoodreads', function(error, result) {
if (error) {
console.log('this is an error ');
} else {
console.log(result); //THIS IS UNDEFINED...possibilities?
}
});
}
});
Use futures to return values from async functions:
var Future = Npm.require("fibers/future")
Meteor.methods({
getGoodreads: function () {
var f = new Future();
request.post('http://www.goodreads.com/oauth/request_token', {oauth:{
consumer_key: '89hdg8pEoMzRdg',
consumer_secret: 'dfgdfgHthtdtjtt' }}, function (error, response, body) {
if (!error && response.statusCode == 200) {
var a = querystring.parse(body)
oauthToken = a.oauth_token
console.log(oauthToken); //prints value i need
return f.return(oauthToken)
}else{
f.throw(error);
console.log('there is an error ' + error);
}
});
return f.wait()
}
});
It's because you are starting an asynchronous request in the method body. The method returns immediately, and the callback from the request doesn't really have anywhere to return the result.
I don't really know how to do this with methods, but you could do it with publish/subscribe. Something like this:
Server.js
var result = "";
Meteor.setInterval(function() {
request.post('http://www.goodreads.com/oauth/request_token', {oauth:{
consumer_key: '89hdg8pEoMzRdg',
consumer_secret: 'dfgdfgHthtdtjtt' }},
function(error, response, body) {
if (!error && response.statusCode == 200) {
var a = querystring.parse(body);
oauthToken = a.oauth_token;
result = oauthtoken;
}
});
});
}, 10*1000);
Meteor.publish("goodReads", function() {
this.added("goodReads", 1, {reads: result});
Meteor.setInterval(function() {
this.changed("goodReads", 1, {reads: result});
}, 10*1000);
this.ready();
});
Client.js
goodReads = new Meteor.Collection("goodReads");
Meteor.subscribe("goodReads"); // May be done in waitOn in the router
This was taken off the top of my head, and is not tested. It may or may not work, but is at least a pointer to how you could do it