I have the following code:
app.get('/pull-requests', function (request) {
fetchRepos(fetchPullRequests);
app.on('pull-requests:fetched', function (pullRequestsByRepo) {
var html = "";
_.each(pullRequestsByRepo, function (pullRequests) {
html += 'There is <strong>'+ pullRequests.length +'</strong> pending pull request(s) for <strong>'+ pullRequests[0].title +'</strong>:';
html += '<ul>';
_.each(pullRequests, function (pullRequest) {
html += '<li><em>'+ pullRequest.title +'</em> ('+ pullRequest.url +')</li>';
});
html += '</ul>';
});
response.send(html);
});
});
It works fine once. Every second request ends raising an error Can't set headers after they are sent..
EDIT: More code to explicit the logic
function fetchRepos (callback) {
_options.path = '/orgs/'+ app.get('org') +'/repos?client_id='+ app.get('client_id') +'&client_secret='+ app.get('client_secret');
// Fetch the list of repos for a given organisation
var request = https.get(_options, function (res) {
data = "";
res.on('data', function (chunk) {
data += chunk;
});
res.on('end', function () {
var repos = JSON.parse(data);
return callback(repos);
});
});
request.on('error', function (error) {
console.log('Problem with request: '+ e);
});
}
function fetchPullRequests (repos) {
var pullRequests = [];
_.each(repos, function (repo, index) {
_options.path = '/repos/'+ app.get('org') +'/'+ repo.name +'/pulls?client_id='+ app.get('client_id') +'&client_secret='+ app.get('client_secret');
var request = https.get(_options, function (res) {
(function () {
var data = "";
res.on('data', function (chunk) {
data += chunk;
});
res.on('end', function () {
data = JSON.parse(data);
if (data.length > 0) {
pullRequests.push(data);
}
if (index == (repos.length - 1)) {
app.emit('pull-requests:fetched', pullRequests);
}
});
})();
});
});
}
Your problem is that whenever you call app.on('pull-requests:fetched', …), you add a new listener, meaning that when the second request arrives, it will trigger the first one again.
node then complains because you try reply twice to the first request.
You could fix your immediate issue by calling app.once, which would ensure that the only fires once, but you would still have an issue if 2 requests arrive at the same time.
The correct pattern in this case is to pass a callback to fetchRepos.
Related
I am trying to write a AWS Lambda function with Nodejs.
Inside the code I have to call a API, wait for the response and do other things with that data (I haven't still write anything about this)
NOTE: sorry about declaring customerTAX as global, but I prefer to get the lambda function to work with and after, try to return the value from the function isself.
This is the code:
'use strict';
var customerTAX;
const https = require('https');
const options = {
host: 'xxxxxxx.com',
port: 443,
path: '/yyyyyyy.json',
method: 'POST',
headers: {
'Content-Type': 'application/graphql',
}
};
exports.handler = async (event) => {
const body = JSON.parse(event.body);
const orderId = body.id;
const customerId = body.customer.id;
console.log('ORDER ID: ' + orderId);
console.log('CUST ID: ' + customerId);
const query = `xxxxxxxxxxxxxxxxxxxx`;
//I CAN SEE ALL LOGS OF THIS FUNCTION IN CLOUDWATCH
await getCustomerTAX(query);
//I CAN'T SEE NOTHING BELOW THIS LINE IN AWS CLOUDWATCH
console.log('CUST TAX: ' + customerTAX);
if (customerTAX != null) {
console.log('LETs GO TO SAVE IT')
} else {
console.log('NOTAX: No customerTAX');
}
const response = {
statusCode: 200,
body: JSON.stringify(event.body),
};
return response;
};
var getCustomerTAX = function(query) {
return new Promise(function(resolve, reject) {
var req = https.request(options, function(res) {
res.setEncoding('utf8');
var bodyRaw = '';
res.on('readable', function () {
var chunk = this.read() || '';
bodyRaw += chunk;
console.log('getTAX CHUNK (' + Buffer.byteLength(chunk) + ' bytes): ' + chunk);
});
res.on('end', function () {
const body = JSON.parse(bodyRaw);
if (body.TAX.value != null) {
customerTAX = body.TAX.value;
} else {
customerTAX = null;
}
console.log("getTAX END: " + customerTAX);
resolve;
//console.log('body: ' + Buffer.byteLength(body) + ' bytes');
});
});
//handle the possible errors
req.on('error', function(e) {
console.log("ERROR: " + e);
reject(e);
});
//do the request
req.write(query);
//finish the request
req.end();
});
};
Function getCustomerTAX works perfectly, but I don't know why my lambda function is "finished" in this line, and I can't see more console-logs in cloudwatch.
Hope your answer and thanks a lot.
For starters in getCustomerTax(), resolve; needs to be resolve();.
It's a function. You need to call it.
Without calling resolve(), the promise is never resolved and thus await getCustomerTax() never finishes and the lines after it are never executed.
FYI, the request-promise module will automatically do a lot of what you have in getCustomerTax() (make an http request, get the response, handle all possible errors and return a promise that represents the result).
EDIT Jan, 2020 - request() module in maintenance mode
FYI, the request module and its derivatives like request-promise are now in maintenance mode and will not be actively developed to add new features. You can read more about the reasoning here. There is a list of alternatives in this table with some discussion of each one. I have been using got() myself and it's built from the beginning to use promises and is simple to use.
I am trying to pass on an array of strings from the 'LaunchRequest' to the 'TellAJoke' intent.
My problem is that this.attributes['joke'] is always undefined in the 'TellAJoke' function.
'LaunchRequest': function() {
var url = "https://www.reddit.com/r/jokes/top.json?limit=1";
var self = this;
var request = https.get(url, function(res) {
var body = '';
res.on('data', function(chunk){
body += chunk;
});
res.on('end', function(){
var redditResponse = JSON.parse(body);
redditResponse['data'].children.forEach(function(child) {
jokeArray.push({
title: child.data.title,
text: child.data.selftext
});
});
// Assinging the filled jokeArray to the session variable
self.attributes['joke'] = jokeArray;
self.emit(':saveState', true);
}).on('error', function(e){
self.emit(':tell', "Error");
console.log("Got an error: ", e);
});
this.emit('TellAJoke');
});
},
'TellAJoke': function() {
this.attributes['randomNumber'] = Math.floor(Math.random() * 8) +0;
//This prints undefined
console.log(this.attributes['joke']);
},
Now, I have read the similar question here, but I does not have sufficient reponses to explain whats going on there. Basically, I was hoping that due to this.emit(':saveState', true) the variable will persist, but it is undefined. Any ideas?
Use emit inside the end block.
res.on('end', () => {
this.emit('TellAJoke');
})
For error handling pass it on to the Unhandled or SessionEndedRequest Intents.
use emitWithState('') instead of
emit('') to move on to another Intent.
I'm doing an introduction to node.js using learnyounode. I wonder if you could help realize this thing: asynchronism.
So, here is the problem:
This problem is the same as the previous problem (HTTP COLLECT) in
that you need to use http.get(). However, this time you will be
provided with three URLs as the first three command-line
arguments.
You must collect the complete content provided to you by each of the URLs and print it to the console (stdout). You don't need to
print out the length, just the data as a String; one line per URL.
The catch is that you must print them out in the same order as the
URLs are provided to you as command-line arguments.
and here is my bad solution who, in fact, don't work.
var http = require('http');
var message = [];
for (var i = 2; i < 5; i++)
http.get(process.argv[i], function (res) {
res.setEncoding('utf8');
res.on('data', function(line) {
message[i] += line.toString();
});
res.on('end', function(line) {
for (var i = 0; i < 3; i++)
console.log(message[i]);
});
});
UPDATE
So I tried a similar approach to your solution.
Here goes:
var http = require('http');
var count = 0;
var message = ["","",""];
for (var i = 2; i < 5; i++)
{
http.get(process.argv[i], function (res) {
res.setEncoding('utf8');
res.on('data', function( line ) {
message[count] += line.toString();
});
res.on('end', function(line) {
count++;
if(count !== 3)
return;
else
printOutput();
});
});
}
function printOutput(){
for (var i = 0; i < 3; i++)
console.log(message[i]);
}
But the output is lagged: / (not in the right order)
CURRENT: "He has not got the skite and watch out for the bogged Trent from punchy blue with the dry to the Vinnie's It'll be flanno
where flat out like the slabs..."
EXPECTED: "He's got a massive coldie my watch out for the smoko We're jackaroo going on she'll be right servo dramas.."
CURRENT ". He has not got a banana bender piece of piss the dry as a budgie smugglers Come a flamin clacker you little bog standard
ripper The cross them to his blood's worth bottling flamin the cunning
of a rip snorter.."
EXPECTED: "He has not got the skite and watch out for the bogged Trent from punchy blue with the dry to the Vinnie's It'll be flanno
where flat out like the slabs..."
CURRENT: "He's got a massive coldie my watch out for the smoko We're jackaroo going on she'll be right servo dramas.."
EXPECTED: "He has not got a banana bender piece of piss the dry as a budgie smugglers Come a flamin clacker you little bog standard
ripper The cross them to his blood's worth bottling flamin the cunning
of a rip snorter..."
CURRENT: ""
EXPECTED ""
a more cleaner way to do it asynchronously is by having all Promises in an array and calling Promise.all() on that array
var http = require('http');
promises = [
promiseLoad(process.argv[2]),
promiseLoad(process.argv[3]),
promiseLoad(process.argv[4])
];
Promise.all(promises).then(function(res){
console.log(res);
});
function promiseLoad(url) {
var body = '';
return new Promise(function(resolve, reject) {
http.get(url, function(res) {
res.on('data', function(d) {
body += d;
});
res.on('end', function() {
resolve(body);
});
});
});
}
You have to wait for the prior request to reach the 'end' event before processing the next request, hence the asynchronous challenge. This can be accomplished via callbacks, or promises.
Promise implementation:
var http = require('http');
promiseLoad(process.argv[2])
.then(promiseLoad(process.argv[3])
.then(promiseLoad(process.argv[4]);
function promiseLoad(url) {
var body = '';
return new Promise(function(resolve, reject) {
http.get(url, function(res) {
res.on('data', function(d) {
body += d;
});
res.on('end', function() {
console.log(body);
resolve();
});
});
});
}
I will leave the callback implementation to you as an exercise. As a starting point, the next request will have to be fired only once the end event if fired.
UPDATE:
To load these truly asynchronously and at the same time, your code will work with minor modifications. You need to simply wait for end to be called 3 times and only log at that point indicating that all loading is complete:
var http = require('http');
var count = 0;
var message = [];
for (var i = 2; i < 5; i++)
http.get(process.argv[i], function (res) {
res.setEncoding('utf8');
var correctIndex = i;
res.on('data', function(line) {
message[correctIndex] += line.toString();
});
res.on('end', function(line) {
count++;
if(count !== 3) return;
for (var i = 0; i < 3; i++)
console.log(message[i]);
});
});
First, I want to say that the answer already here that says to use Promise.all() is the way that I would suggest. However, I want to point out a particular scenario where it might not cover your needs.
Consider that you have 3 requests:
"Service" | "Time to complete"
----------------------------
A | 3
B | 1
C | 5
D | 4
And you're going to use a load handler similar to what has already been mentioned:
// Url loader
function load(url) {
var message = "";
return new Promise(function (resolve, reject) {
http.get(url, function (res) {
// Add message piece
res.on("data", function (data) {
message += data;
});
// Resolve whole message
res.on("end", function (data) {
resolve(message);
});
});
});
}
Printing After Everything Finishes
If you use the Promise.all(), you are going to have to wait for all of the requests to finish before you see any output. So if we output a timestamp with our data, we will get the following:
Code
/*
Wait for all promises to complete and then
print out all of their collected data
*/
Promise.all(promises).then(function (res) {
res.forEach(function (data) {
timestamp(data);
});
});
Output
[14:9:4.106] Start
[14:9:10.335] aaaa
[14:9:10.336] bbbb
[14:9:10.336] cccc
[14:9:10.336] dddd
Where it takes 6 seconds after we start to see any output from the result of our services.
Printing As Soon As Possible
Comparatively, if we wanted to print output while we are getting results from our service calls, we need to print the result as the service finishes, but not until all "prior" services are done. With that in mind, we could do could do something like this:
Code
promises[0].then(function (dataA) {
timestamp(dataA);
promises[1].then(function (dataB) {
timestamp(dataB);
promises[2].then(function (dataC) {
timestamp(dataC);
promises[3].then(function (dataD) {
timestamp(dataD);
});
});
});
});
Output
[14:16:19.245] Start
[14:16:22.974] aaaa
[14:16:22.975] bbbb
[14:16:25.474] cccc
[14:16:25.474] dddd
Here, we see the start, then only 3 seconds later we print out both Service A and Service B. We see A because its service just resolved and B because its service was already done, but we didn't want to print until A was finished. Similarly, C and D show up about 2 seconds after B.
Now, that code is somewhat verbose, so we could write a recursive function to handle all that nesting for us.
// Function to print an array of promises in order
function cascadeInOrder(promiseArr) {
var curr = 0;
// This closure is going to recursively print out our promises
function nexter(data) {
if (data) {
timestamp(data);
}
// Have the next promise print its data whenever it is done
promiseArr[curr += 1].then(nexter);
}
// Wait for our first promise to finish and have it kick off the next
promiseArr[curr].then(nexter);
}
I haven't really run into many uses cases where we need to make "synchronous" usage of asynchronous data, but I'm sure that there is a potential need for it somewhere.
Test Code Used:
Change the method variable if you want to use the other methods.
/*global Promise*/
"use strict";
// Provide response times for fake services
function getUrlTiming(url) {
var timing = 0;
switch (url) {
case "a":
timing = 3000;
break;
case "b":
timing = 1000;
break;
case "c":
timing = 5000;
break;
case "d":
timing = 4000;
break;
default:
timing = 0;
break;
}
return timing;
}
// Service to wrap events
function Service() {
this.listeners = [];
}
Service.prototype = {
on: function (event, cb) {
if (!this.listeners[event]) {
this.listeners[event] = [];
}
this.listeners[event].push(cb);
},
emit: function (event, details) {
if (this.listeners[event]) {
this.listeners[event].forEach(function (cb) {
cb(details);
});
}
}
};
// Make a fake http module
var http = {
get: function (url, cb) {
// Make an event emiiter
var req = new Service();
// If we got a callback
if (cb && (typeof cb === "function")) {
// Call it to set up listeners
cb(req);
}
// Make a promise to resolve after the service finishes
return new Promise(function (resolve, reject) {
var network,
message = "",
part = 0,
maxParts = 4;
/*
Create a network simulation to send a massage in parts
until the request finishes
*/
network = setInterval(function () {
// If the message isn't complete
if (part < 4) {
// Add to the whole message tracker
message += url;
// Emit that we got data
req.emit("data", url);
// Increment how far in the message we are
part += 1;
} else {
// Stop transmitting
clearInterval(network);
// Emit the end of the request
req.emit("end", message);
// Resolve the request
resolve(url);
}
}, (getUrlTiming(url) / maxParts));
});
}
};
// Url loader
function load(url) {
var message = "";
return new Promise(function (resolve, reject) {
http.get(url, function (res) {
// Add message piece
res.on("data", function (data) {
message += data;
});
// Resolve whole message
res.on("end", function (data) {
resolve(message);
});
});
});
}
// Get a readable time
function getTime() {
var now = new Date();
return (now.getHours() + ":" + now.getMinutes() + ":" + now.getSeconds() + "." + now.getMilliseconds());
}
// Print a timestamped message
function timestamp(message) {
console.log("[%s] %s", getTime(), message);
}
// Function to print an array of promises in order
function cascadeInOrder(promiseArr) {
var curr = 0;
// This closure is going to recursively print out our promises
function nexter(data) {
if (data) {
timestamp(data);
}
// Have the next promise print its data whenever it is done
promiseArr[curr += 1].then(nexter);
}
// Wait for our first promise to finish and have it kick off the next
promiseArr[curr].then(nexter);
}
/*
No matter what, we want all of our requests to
start right now, and effectively at the same time.
We don't want to start one after another finishes
*/
var promises = [
load("a"),
load("b"),
load("c"),
load("d")
];
/*
Which method we want to use to test our stuff
Change between [1, 2, 3] for each method listed
below. 1 for Promise.all(), 2 for ASAP printing,
and 3 for the verbose version of 2.
*/
var method = 3;
// Note when we started
timestamp("Start");
if (method === 1) {
/*
Wait for all promises to complete and then
print out all of their collected data
*/
Promise.all(promises).then(function (res) {
res.forEach(function (data) {
timestamp(data);
});
});
} else if (method === 2) {
/*
Print each ones data as soon as it is
available; but make sure to do it in order
*/
cascadeInOrder(promises);
} else if (method === 3) {
/*
This is the same as the "cascadeInOrder" function,
except written without recursion and more verbosely.
*/
promises[0].then(function (dataA) {
timestamp(dataA);
promises[1].then(function (dataB) {
timestamp(dataB);
promises[2].then(function (dataC) {
timestamp(dataC);
promises[3].then(function (dataD) {
timestamp(dataD);
});
});
});
});
}
#Luís Melo
Here's my solution after going through this thread:
var http = require('http');
var bl = require('bl')
promises = [
promiseLoad(process.argv[2]),
promiseLoad(process.argv[3]),
promiseLoad(process.argv[4])
];
Promise.all(promises).then(function(res) {
for(i=0; i<promises.length; i++) {
console.log(res[i]);
}
});
function promiseLoad(url) {
var body = '';
return new Promise(function(resolve, reject) {
http.get(url, function (response) {
response.setEncoding('utf8');
response.pipe(bl(function (err, data) {
resolve(data.toString())
}))
})
});
}
Here's the official solution in case you want to compare notes:
var http = require('http')
var bl = require('bl')
var results = []
var count = 0
function printResults () {
for (var i = 0; i < 3; i++) {
console.log(results[i])
}
}
function httpGet (index) {
http.get(process.argv[2 + index], function (response) {
response.pipe(bl(function (err, data) {
if (err) {
return console.error(err)
}
results[index] = data.toString()
count++
if (count === 3) {
printResults()
}
}))
})
}
for (var i = 0; i < 3; i++) {
httpGet(i)
}
I'm rather new in express framework. I have call flickr api to get albums list and for each album need to get its thumbnail. On the end need to build covers array with list of objects like {title, thumb}. I would like to pass fully created covers array to template and render. I have problem with it because of the way that node.js callbacks works and for loop ends quickly before requests ends. How to do that properly ?
http.get(base_url+'&user_id='+flickr.user_id+'&method=flickr.photosets.getList', function(resp){
var body = '';
resp.on('data', function(chunk) {
body += chunk;
});
resp.on('end', function() {
var json = JSON.parse(body);
var ps = json.photosets.photoset;
// final answer
var covers = {};
for(var i=0; i<ps.length; i++) {
var p = ps[i];
var getSizesUrl = base_url+'&user_id='+flickr.user_id+'&method=flickr.photos.getSizes&photo_id='+p.primary;
http.get(getSizesUrl, function(resp){
var body1 = '';
resp.on('data', function(chunk) {
body1 += chunk;
});
resp.on('end', function() {
var json1 = JSON.parse(body1);
covers += {title: p.title._content, thumb: json1.sizes.size[1].source};
if(i + 1 == ps.length) {
// last call
console.log(covers);
res.render('photosets', {covers: covers});
}
});
});
}
});
});
Update using async and request as #sgwilly said, but something wrong...
request(base_url+'&user_id='+flickr.user_id+'&method=flickr.photosets.getList', function (error, response, body) {
var json = JSON.parse(body);
var ps = json.photosets.photoset;
// functions list to call by `async`
var funcs = {};
for(var i = 0; i < ps.length; i++) {
var p = ps[i];
funcs += function(callback) {
request(base_url+'&user_id='+flickr.user_id+'&method=flickr.photos.getSizes&photo_id='+p.primary, function (error, response, body1){
var json1 = JSON.parse(body1);
var cover = {title: p.title._content, thumb: json1.sizes.size[1].source};
callback(null, cover);
});
};
}
// run requests and produce covers
async.series(funcs,
function(covers){
console.log(covers);
res.render('photosets', {covers: covers});
}
);
});
Node.js callbacks are a little tricky, but they'll make sense after a while.
You want to use request as your outside loop, then async.concat as a callback inside, and have each iterator be a request call of a URL.
Thanks for #dankohn. I got this and work :)
request(base_url+'&user_id='+flickr.user_id+'&method=flickr.photosets.getList', function (error, response, body) {
var json = JSON.parse(body);
var ps = json.photosets.photoset;
async.concat(ps, function(p, callback){
request(base_url+'&user_id='+flickr.user_id+'&method=flickr.photos.getSizes&photo_id='+p.primary, function (error, response, body1){
var json1 = JSON.parse(body1);
var cover = {title: p.title._content, thumb: json1.sizes.size[1].source};
callback(null, cover);
});
},
function(err, covers){
console.log(covers);
res.render('photosets', {covers: covers});
});
});
I'm relatively new to callbacks and have been unsuccessful in getting the following code to work. I have used the async.map function to return the data from each web call to its respective page. However, my console.log(return) is returning [ , undefined] even though the console.log(data) prior to the callback(data) is returning the web page's html. Here's my code:
var http = require("http"),
fs = require("fs"),
page, chap, ot,
async = require("async");
ot = fs.open('ot.txt', 'w');
page = "test";
chap = 2;
function getData(url, callback) {
var data = "";
var options = {
host: "rs.au.com",
port: 80
}
options.path = url;
console.log("request sent to: http://" + options.host + options.path);
var req = http.request(options, function(res) {
console.log("Response received " + res.statusCode);
res.on('data', function(chunk) {
data += chunk;
});
res.on('end', function(e) {
console.log(data);
callback(e, data);
});
}).end();
}
function main() {
var pathArr = [];
for ( var i = 1; i <= chap; i++ ) {
pathArr[i] = "/".concat(page, "/", i, ".html");
}
async.map(pathArr, getData, function(err, result) {
console.log("The result is :" + result);
});
}
main();
Could anyone please point out why my code is not working and how I can correct it?
Much appreciated!
EDIT: After Brandon Tilley's response I amended the callback function from callback(data) to callback(e, data), however I'm getting no response now from the last console.log output.
The Async library assumes your callbacks adhere to the standard Node.js callback signature, which is callback(err, others...). Since you are passing data as the first argument, Async assumes it's an error. You should use callback(e, data) instead (since e will be null in the case of no errors).
[Update]
The other problem is that your array is not correct. Since i starts at 1 and goes up to chap, pathArr[0] is undefined. Change:
pathArr[i] = "/".concat(page, "/", i, ".html");
to
pathArr[i-1] = "/".concat(page, "/", i, ".html");