Can't set headers after they are sent in get request - node.js

router.get('/getpostcode', function(req, res){
console.log(req.query.suburb);
var options = {
url: 'http://apiurl?suburb=' + req.query.suburb + "&state=" + req.query.state ,
headers: {
'auth-key': key
}
}
request(options, callback);
function callback(error, response, body){
if(!error && response.statusCode == 200){
info = JSON.parse(body);
info = info.localities.locality;
if( Object.prototype.toString.call( info ) == '[object Array]' ){
for ( var x = 0 ; x < info.length ; x++ ){
var locx = info[x].location;
var qsuburb = req.query.suburb;
if( locx == qsuburb.toUpperCase() ){
res.send( { 'postcode': info[x].postcode } );
}
}
} else if (Object.prototype.toString.call( info ) != '[object Array]') {
var locx = info.location;
var qsuburb = req.query.suburb;
if ( locx == qsuburb.toUpperCase() ){
res.send( { 'postcode': info.postcode } );
}
}
}
}
});
So, I am trying to request data from an API and then based on that data, I am then sending some data back. My code is as above.
Unfortunately, in the callback function, when I am running a loop to find a specific element that I will then send back to the client, when sending that element to client, I am getting the error as shown in the title.
This does not occur when there is no loop and I simply send one element of the data back.
Any ideas on what this could be?

You are sending response more than one for one request, write res.send or JSON outside the loop.
for (var x = 0; x < info.length; x++) {
var locx = info[x].location;
var qsuburb = req.query.suburb;
var postcodes = [];
if (locx == qsuburb.toUpperCase()) {
postcodes.push({
'postcode': info[x].postcode
});
}
res.json(postcodes);
}

You are reaching the res.send function call at least twice in your flow and this is not allowed. That function must be called once to send the response to the client.

Yeah, As #Rahul said, It is not allowed to send response more than one time for the same request - You must be getting same post code, so you either need to store your postcode in a variable and send it after loop overs or you can use a break. Though it is not advisable to use a break in complex loops in the pretty simple loop you can leverage its usage.
for (var x = 0; x < info.length; x++) {
var locx = info[x].location;
var qsuburb = req.query.suburb;
if (locx == qsuburb.toUpperCase()) {
res.send({
'postcode': info[x].postcode
});
break;
}
}
Or as following:
for (var x = 0; x < info.length; x++) {
var locx = info[x].location;
var qsuburb = req.query.suburb;
vat postcode = = null;
if (locx == qsuburb.toUpperCase()) {
postcode = info[x].postcode
}
}
res.send({
postcode
});

Related

How can I restructure my loop to paginate faster right now it takes too long?

Per_page limit is 100 & I needed the function to be able to find results for any date range. So I added a do while loop and this is what I ended up with:
async function foo(repoOwner, repository, startingDate, endingDate){
const listOfUserObjects = [];
let pages, page = 1;
do{
await axios({
method: 'get',
url : `https://api.github.com/repos/${repoOwner}/${repository}/pulls?state=all&per_page=100&page=${page}`,
}).then(response => {
const users = response.data, startAt = new Date(startingDate), endAt = new Date(endingDate.valueOf());
endAt.setDate(endAt.getDate() + 1);
if(page === 1) pages = (Math.floor((users[0].number)/100))/2, console.log("collecting data, please wait...");
for(let i = 0; i < users.length; i++) {
const createdDate = new Date(users[i].created_at), updatedDate = new Date(users[i].updated_at), closedDate = new Date(users[i].closed_at), mergedDate = new Date(users[i].merged_at);
if(((createdDate || updatedDate || closedDate || mergedDate) >= startAt) && ((createdDate || updatedDate || closedDate || mergedDate) <= endAt)) {
const userObj = {};
userObj.id = users[i].id;
userObj.user = users[i].user.login;
userObj.title = users[i].title;
userObj.state = users[i].state;
userObj.created_at = users[i].created_at.slice(0,10);
listOfUserObjects.push(userObj);
} else {
continue;
};
}
page++
}).catch(error => {
throw error.response.status === 404 ? Error("Error 404 User or Repo Not Found") : Error(error);
});
} while(page < pages);
console.log(listOfUserObjects);
}
For paging through the data I added a do while loop. I expected it to loop through all the pages and retrieve the relevant data.
The code works, but the do while loop is too slow and it loops through all the pages; instead I would prefer it to stop when the data from the desired date range has been retrieved.
This takes ages to retrieve data. Is there a more efficient way to do this?

my sync function doesn't wait for all the subfunctions to finish

i'm trying to make an api that can see if i made a post on imgur and i'm really close to finish but my function returns before the end
exports.handler = (event, context, callback) => {
ddb.scan(params1, function(err, data) {
if (err) callback(null, err);
else {
data.Items.forEach(function(item) {
for (var o = 0; item.links && item.links.L && o < item.links.L.length; o++) {
var taction = "";
var treaction = "";
var action = item.links.L[o].S.substring(0, item.links.L[o].S.indexOf(' '));
var saction = action.substring(0, action.indexOf(':'));
var reaction = item.links.L[o].S.substring(item.links.L[o].S.indexOf('=')+2);
var sreaction = reaction.substring(0, reaction.indexOf(':'));
for (var z = 0; item.accsplus && item.accsplus.L && z < item.accsplus.L.length; z++) {
if (item.accsplus.L[z].S.substring(0, item.accsplus.L[z].S.indexOf(':')) == saction)
taction = item.accsplus.L[z].S.substring(item.accsplus.L[z].S.indexOf('token:')+6);
if (item.accsplus.L[z].S.substring(0, item.accsplus.L[z].S.indexOf(':')) == sreaction)
treaction = item.accsplus.L[z].S.substring(item.accsplus.L[z].S.indexOf('token:')+6);
}
if (taction == "" || treaction == "") log += "no token for this service #" +action+reaction + " ";
else{
console.log("testing " +action+reaction)
if ((saction == "imgur" || saction == "reddit") && (sreaction == "imgur" || sreaction == "reddit")) {
if (saction == "imgur")
imgur(action.substring(action.indexOf(':')+1), item, taction, reaction.substring(reaction.indexOf(':')+1));
}
}
}
})
callback(null, "ok");
}
});
async function imgur(action, whom, token, reaction) {
if (action =="") return;
var requestdone = false;
var toret;
var old = "";
var needed = action;
if (action == "onpost" || action == "onrem") needed = "postnbr";
if (action == "onlike" || action == "ondis") needed = "likenbr";
console.log("imgur " + needed);
var myHeaders = new Headers();
myHeaders.append("Authorization", "Bearer " + token);
var requestOptions = {
method: 'GET',
headers: myHeaders,
redirect: 'follow'
};
async function request(success) {
const response = await fetch("https://api.imgur.com/3/account/me/images", requestOptions)
const json = await response.json();
return await success(json);;
}
function success(json) {
var worked;
if (needed == "postnbr") actual = (json.data.length);
if (needed == "likenbr"){
if (json.data[0].vote == null || json.data[0].vote == "null") actual = (0);
actual = (json.data[0].vote);
}
if (needed == "postnbr") console.log("postnbr = " + actual);
if (needed == "likenbr") console.log("likenbr = " + actual);
if (whom.old && whom.old.L && whom.old.L.length > 0){
for (var p = 0; old == "" && p < whom.old.L.length; p++){
if (whom.old.L[p].S.substring(0,whom.old.L[p].S.indexOf(':')) == "imgur"+needed)
{
old = whom.old.L[p].S.substring(whom.old.L[p].S.indexOf(':')+1);
if (action == "onpost" && old < actual) worked = true;
if (action == "onrem" && old > actual) worked = true;
if (action == "onlike" && old < actual) worked = true;
if (action == "ondis" && old > actual) worked = true;
}
}
}
if (worked)
{
return imgur(reaction, whom, token, "");
}
upold("imgur", needed, whom, actual)
}
await request(success)
}
var getactual = function(service, token, needed) {
return new Promise(function(resolve, reject){
if (service == "imgur"){
}
if (service == "reddit"){
console.log("do reddit mofo");
}
})
};
function upold(service, needed, whom, actual) {
var toret = [];
if (whom.old && whom.old.L.length > 0){
for (var m = 0; m < whom.old.L.length ; m++) {
if (whom.old.L[m].S.substring(0,whom.old.L[m].S.indexOf(':')) != service+needed)toret.push(whom.old.L[m].S);
}
}
toret.push(service+needed + ":" +actual);
param = {
TableName:"biipboop",
Key:{
"email": whom.email.S
},
UpdateExpression: "set old=:r",
ExpressionAttributeValues:{
":r":toret
},
ReturnValues:"UPDATED_NEW"
};
docClient.update(param, function() {});
}
};
so far it seems like the function works and the request subfunction is called but the main doesn't wait for an answer and returns Promise {} then the log does happen once the success completes itself but i don't have my variable at the end of the imgur function, it just logs itself when finished
EDITED :
since you asked for the actual code i had, i copy pasted it brutally
i have my scan working well, gets all my user and then it doesn't wait for function "imgur" to give a proper answer and just leaves triggers it and iterate through the foreach.
i'm trying to use lambda and ddb through aws
the logs do arrive in this order :
testing imgur:onpostimgur:onlike
imgur postnbr
testing reddit:postreddit:onpost
postnbr = 3
The way you are returning data from imgur looks really off to me.. This would be a lot easier, in my opinion, if you turned imgur into an async function so that you can await for the data that request returns.
async function imgur(action, whom, token) {
var toret;
var old = "";
var needed = "postnbr";
var myHeaders = new Headers();
myHeaders.append("Authorization", "Bearer " + token);
var requestOptions = {
method: 'GET',
headers: myHeaders,
redirect: 'follow'
};
async function request(success) {
const response = await fetch("https://api.imgur.com/3/account/me/images", requestOptions)
const json = await response.json();
return /* await */ success(json); // <-- shouldn't have to await this
}
function success(json) {
old = "2";
if (needed == "postnbr") actual = (json.data.length);
if (needed == "postnbr") console.log("postnbr = " + actual);
if (action == "onpost" && old < actual)
return("you made a post recently");
updateold("imgur", needed, whom, actual)
}
return await request(success); // .then(function(msg) {return msg;});
}
Then you can use it like:
async function useImgur() {
const imgurData = await imgur('some-action', 'some-whom', 'some-token');
console.log(imgurData);
}
useImgur();
...or like:
imgur('some-action', 'some-whom', 'some-token')
.then(imgurData => console.log(imgurData))
.catch(err => console.error(err));
Well imgur() returns a promise. So,
console.log(imgur("onpost", accounts[i], accounts[i].token))
will always log an unresolved promise, every time. Even if that promise is eventually going to get a resolved value from
return("you made a post recently");
it won't have it yet since promises are never resolved synchronously. The soonest the promise that imgur() returns will be resolved is on the next tick of the event loop which is after your console.log().
You can change the code to this to log the value:
async function main(accounts) {
for (var i = accounts.length - 1; i >= 0; i--) {
let val = await imgur("onpost", accounts[i], accounts[i].token);
console.log(val);
}
}
Keep in mind that some code paths through imgur() do not return a value which is probably not a desired design, thus sometimes the resolved value from the imgur() promise can be undefined and sometimes not.

Node.js passing variable to parent function

So i ran into a problem. I don't know how to pass single string to the parental function from a child function and then pass that string as a response to the client side.
This whole thing gets five recent matches from API and then checks for a win or a loss depending on the player name.
Question 1: as i said before i don't know how to pass string from a child function to the parental function and then send it as a response to client side.
Question 2: the output of this should be WWWLW and how i think it should be ordered like that. But every time it outputs in different order like LWWWW WLWWW and so on... it has good arguments but different order and i am missing something here.
code:
var request = require('request');
app.get('/history',getmatches, getwins);
function getmatches(req, res, next){
var match = {};
request({
url: "https://eun1.api.riotgames.com/lol/match/v3/matchlists/by-account/"+ID+"/recent?api_key=" + key,
json: true
}, function (error, res) {
if (!error && res.statusCode === 200) {
for(var i=0; i < 5; i++){ //getting ID's of five last matches
match[i] = res.body.matches[i].gameId;
}
req.somevariable = match;
next();
}
}
);
};
function getwins(req, res, callback){
var match = req.somevariable;
var streak = '';
var pending = 0;
for( i = 0; i < 5; i++){ // passing ID's to another api link to get single match data
request({
url: "https://eun1.api.riotgames.com/lol/match/v3/matches/"+match[i]+"?api_key=" + key,
json: true
}, function(req,res, body){
for(var j = 0; j < 10; j++){ //looping through 10 players in a match to find specific one
if(body.participantIdentities[j].player.summonerName == nickname){
if( body.participants[j].stats.win == true){
streak += 'W';
}else{
streak += 'L';
}
}
}
if(pending == 4){
console.log(streak); // need this to pass to parent function
return callback(null, streak); // is this something i need ?
}
pending++
});
}
// res streak string to client.js
};
There is solution to process all results when it done. The result variable have all results use any appropriate key instead of url;
function getwins(req, res, callback){
var match = req.somevariable;
var streak = '';
var pending = 0;
var results = {};
var total = 5;
for( i = 0; i < total; i++){ // passing ID's to another api link to get single match data
var url = "https://eun1.api.riotgames.com/lol/match/v3/matches/"+match[i]+"?api_key=" + key;
request({
url: url,
json: true
}, function(req,res, body){
for(var j = 0; j < 10; j++){ //looping through 10 players in a match to find specific one
if(body.participantIdentities[j].player.summonerName == nickname){
if( body.participants[j].stats.win == true){
streak += 'W';
}else{
streak += 'L';
}
}
}
console.log(streak); // need this to pass to parent function
results[url] = streak;
if( total == Object.keys(results).length ) {
// here all requests are done - do with all result what you need
console.log( results );
}
return callback(null, streak); // is this something i need ?
}
});
}
// res streak string to client.js
};

NodeJS Queue performing delayed HTTP Request are weird

Dear helpful people in the internet,
i started to write an HTTP Queue. I have a Request Class, which works if I use it without context, but with my queue active, my http requests wont work. I can't figure out why.
var t = new Request('s','s', function(){});
t.perform();
When i perform the request like this in any file. It works. But when i use it with my queue (index.js,L19 to L22) no request is performed. The function Request.perform() were executed, but the HTTP-Request isn't there.
Sorry for my english, I'm not native ^^
index.js
const http = require('http');
const https = require('https');
const {Request} = require('./classes/Request');
const queue = require('./queue.js');
queue.setCallbackFunction(performRequest);
function performRequest(request){
console.log("2");
request.perform();
}
var req = new Request('','', function(response,body){
console.log(JSON.stringify(response) + " :: " + body);
});
queue.add(req);
queue.js
var queue = [];
var ratelimits = [];
module.exports.add = function(request){
queue.push(request);
run_queue();
}
module.exports.setCallbackFunction = function(cb){
call = cb;
}
module.exports.setRateLimits = function(ratelimitings){
ratelimits = [];
for(var z in ratelimitings){
var x = ratelimitings[z];
var data = {};
data.max = x[0];
data.time = x[1];
data.count = 0;
ratelimits[x[1]] = data;
}
}
function run_queue(){
var q;
if(queue.length > 0){
q = run_request(queue[0]);
while (q == true) {
queue.shift();
if(queue.length > 0)
q = run_request(queue[0]);
}
}
}
function run_request(request){
for(var z in ratelimits){
var x = ratelimits[z];
if(x.max <= x.count){
return false;
}
}
for(var z in ratelimits){
var x = ratelimits[z];
if(x.count === 0){
setTimeout(function(z){
console.log(JSON.stringify(x));
ratelimits[z].count = 0;
run_queue();
},z,z);
}
x.count++;
//console.log(JSON.stringify(x));
}
//DO REQUEST
console.log("1")
call(request);
return true;
}
Request.js
exports.Request = class{
constructor(host,path,cb){
this.path = path;
this.cb = cb;
this.host = host
}
perform(){
console.log("3");
var https = require('https');
var options = {
host: 'www.example.com',
path: '/'
};
var callback = function(response) {
//HERE THIS GETS NEVER CALLED BECAUSE OF WHATEVER WHILE ITS IN THE QUEUE
var str = '';
//another chunk of data has been recieved, so append it to `str`
response.on('data', function (chunk) {
str += chunk;
});
//the whole response has been recieved, so we just print it out here
response.on('end', function () {
console.log(str);
});
}
https.request(options, callback).end();
}
}
All 3 console.logs get printed, but the Request Callback Never gets called.
The problem is caused by function run_queue:
function run_queue(){
var q;
if(queue.length > 0){
q = run_request(queue[0]);
while (q == true) {
queue.shift();
if(queue.length > 0)
q = run_request(queue[0]);
}
}
}
After run_request() is executed successfully (HTTP request is sent), queue.shift() is called immediately, which means the req object you just added into the queue is removed from queue array and qualified for Garbage Collection. As one HTTP request/response will normally cost a few milliseconds, it is very likely that before response is retrieved, the req object is GCed (destroyed). Thus, no callback will be invoked, as the HTTP connection does not exist any more.
To change queue but keep req object from GC, you need to save it somewhere else, such as a temporary array (the infinite loop bug is also fix in following code):
var tmp = [];
function run_queue(){
var q;
if(queue.length > 0){
q = run_request(queue[0]);
while (q == true) {
tmp.push(queue.shift());
if(queue.length > 0) {
q = run_request(queue[0]);
} else {
q = false;
}
}
}
}
Please note the above code is only for demo. In production code, you need to manage the tmp array -- when one request is done, it needs to be removed from tmp. Otherwise, tmp array will keep growing...

How to wait a request response and return the value?

When I use a request module in node.js server, I have some problem such as wait and return.
I would like to receive a "responseObject" value at requestController.
To solve this problem, I have search the best way but I still do not find it.
How can solve this problem?
Thank in advance!! :)
=========================================================================
var requestToServer = require('request');
function getRequest(requestObject) {
var urlInformation = requestObject['urlInformation'];
var headerInformation = requestObject['headerInformation'];
var jsonObject = new Object( );
// Creating the dynamic body set
for(var i = 0; i < headerInformation.length; i++)
jsonObject[headerInformation[i]['headerName']] = headerInformation[i]['headerValue'];
requestToServer({
url : urlInformation,
method : 'GET',
headers : jsonObject
}, function(error, response ,body) {
// todo response controlling
var responseObject = response.headers;
responseObject.body = body;
});
}
// Controlling the submitted request
exports.requestController = function(requestObject) {
var method = requestObject['methodInformation'];
var resultObject = null;
// Selecting the method
if(method == "GET")
resultObject = getRequest(requestObject);
else if(method =="POST")
resultObject = postRequest(requestObject);
else if(method == "PUT")
resultObject = putRequest(requestObject);
else if(method == "DELETE")
resultObject = deleteRequest(requestObject);
console.log(JSON.stringify(resultObject));
}
You can use callbacks in the following way.
function getRequest(requestObject, callback) {
// some code
requestToServer({
url : urlInformation,
method : 'GET',
headers : jsonObject
}, function(error, response ,body) {
// todo response controlling
var responseObject = response.headers;
responseObject.body = body;
callback(responseObject);
});
}
And
// Controlling the submitted request
exports.requestController = function(requestObject) {
var method = requestObject['methodInformation'];
// Selecting the method
if(method == "GET")
getRequest(requestObject, function(resultObject){
console.log(JSON.stringify(resultObject));
});
//some code
}
Hope, it helps.

Resources