Well, i'm visiting an array of urls making a request for each one, when one request ends the method executes the next. The array is something like this: [link1,link2,link3]. If i try to open first the link3 in browser i'll get an error (error 404) but opening the link1 and link2 first i'll have the desired response. In the browser works without problems, but isn't working in my code because i got "status:200" using the first two links, but a 404 with the third.
(If i open link2 and link2 in the browser the problem ends, but i want to do that not using the browser)
The code:
function websiteOpener(links_array, index, final) {
var methodStr = className + '::websiteOpener';
try {
log.info(methodStr + '>> Open the link: ' + links_array[index]);
var protocol;
var _host;
var rawhost;
if (links_array[index].match(/https:\/\/[^\/]+/)) {
rawhost = links_array[index].match(/https:\/\/[^\/]+/);
_host = rawhost[0].replace(/https:\/\//, '');
protocol = 'https:'
_path = links_array[index].replace(rawhost, '');
incremental = index + 1;
var options = {
host: _host,
path: _path,
method: 'GET',
headers: { 'Content-type': 'text/html' },
protocol: protocol,
agent: new https.Agent({
rejectUnauthorized: false,
})
}
} else {
incremental = index + 1;
var options =links_array[index];
}
if (incremental < final) {
if (links_array[index].match(/https:\/\/[^\/]+/)) {
var request = https.request(options, function (response) {
console.log(response.statusCode);
//if (response.statusCode === 200) {
var data;
response.on('data', (chunk) => {
data += chunk;
});
response.on('end', function () {
websiteOpener(links_array, incremental, final);
});
//}
});
request.end();
} else {
var request = http.request(options, function (response) {
//if (response.statusCode === 200) {
var data;
response.on('data', (chunk) => {
data += chunk;
});
response.on('end', function () {
websiteOpener(links_array, incremental, final);
});
//}
});
request.end();
}
} else {
options.headers = { 'Content-type': 'applcation/pdf' };
var request = https.request(options, function (response) {
console.log(response.statusCode);
//if (response.statusCode === 200) {
var data;
response.on('data', (chunk) => {
data += chunk;
});
response.on('end', function () {
log.info(methodStr + '>>link found ' + links_array[index]);
});
//}
});
request.end();
}
} catch (e) {
log.error(methodStr + ">> Server error: ", e);
reject({ statusCode: 500, flag: 'ERR_PROCESS' });
}
}
Related
I have an azure function with this line of code.
var myReq = https.request(options, function(res) {
context.log('STATUS: ' + res.statusCode);
context.log('HEADERS: ' + JSON.stringify(res.headers));
body += res.statusCode
res.on('data', function (chunk) {
context.log('BODY: ' + chunk);
});
});
myReq.on('error', function(e) {
context.log('problem with request: ' + e.message);
});
myReq.write(postData);
myReq.end();
But my code seems to just skip this part of code, with no errors. I am new to Azure and node.js so I might have missed some basic parts in setting this up.
Any ideas?
Edit:
Here is my full code
const https = require('https');
const querystring = require('querystring');
module.exports = async function (context, req) {
if (req.query.accessCode || (req.body && req.body.accessCode)) {
context.log('JavaScript HTTP trigger function processed a request.');
var options = {
host: 'httpbin.org',
port: 80,
path: '/post',
method: 'POST'
};
var postData = querystring.stringify({
client_id : '1234',
client_secret: 'xyz',
code: req.query.accessCode
});
var body = "";
var myReq = https.request(options, function(res) {
context.log('STATUS: ' + res.statusCode);
context.log('HEADERS: ' + JSON.stringify(res.headers));
body += res.statusCode
res.on('data', function (chunk) {
context.log('BODY: ' + chunk);
});
});
myReq.on('error', function(e) {
context.log('problem with request: ' + e.message);
});
myReq.write(postData);
myReq.end();
context.log("help");
context.res = {
status: 200,
body: "Hello " + (body)
};
} else {
context.res = {
status: 400,
body: "Please pass a name on the query string or in the request body"
};
}
};
Ideally it should work. You can also try using request module like below
const request = require('request');
request('http://www.google.com', function (error, response, body) {
console.error('error:', error); // Print the error if one occurred
console.log('statusCode:', response && response.statusCode); // Print the response status code if a response was received
console.log('body:', body); // Print the HTML for the Google homepage.
});
Try and see if it helps.
Solved by doing await properly. Used this as guide.
var https = require('https');
var util = require('util');
const querystring = require('querystring');
var request = require('request')
module.exports = async function (context, req) {
context.log('JavaScript HTTP trigger function processed a request.');
/*if (req.query.name || (req.body && req.body.name)) {*/
var getOptions = {
contentType: 'application/json',
headers: {
'Authorization': <bearer_token>
},
};
var postData = {
"key": "value"
};
var postOptions = {
method: 'post',
body: postData,
json: true,
url: <post_url>,
headers: {
'Authorization': <bearer_token>
},
};
try{
var httpPost = await HttpPostFunction(context, postOptions);
var httpGet = await HttpGetFunction(context, <get_url>, getOptions);
return {
res: httpPost
};
}catch(err){
//handle errr
console.log(err);
};
};
async function HttpPostFunction(context, options) {
context.log("Starting HTTP Post Call");
return new Promise((resolve, reject) => {
var data = '';
request(options, function (err, res, body) {
if (err) {
console.error('error posting json: ', err)
reject(err)
}
var headers = res.headers;
var statusCode = res.statusCode;
//context.log('headers: ', headers);
//context.log('statusCode: ', statusCode);
//context.log('body: ', body);
resolve(body);
})
});
};
async function HttpGetFunction(context, url, options) {
context.log("Starting HTTP Get Call");
return new Promise((resolve, reject) => {
var data = '';
https.get(url, options, (resp) => {
// A chunk of data has been recieved.
resp.on('data', (chunk) => {
data += chunk;
})
// The whole response has been received. Print out the result.
resp.on('end', () => {
resolve(JSON.parse(data));
});
}).on("error", (err) => {
console.log("Error: " + err.message);
reject(err.message);
});
});
};
I am running the drill query using node.js.It is taking more than 500ms for retrieving only 10 records.I am using http request method of nodejs.
function executeService(params) {
return new Promise((resolve, reject)=> {
try {
var serverOptions = {
hostname: "127.0.0.1",
port: 8047,
path: "/query.json",
method: "POST",
headers: {
'Content-Type': 'application/json',
}
};
var http = require("http");
var req = http.request(serverOptions, function (res) {
if (params && params.response) {
res.setEncoding('binary');
} else {
res.setEncoding('utf8');
}
var body = '';
res.on('data', function (chunk) {
body += chunk;
});
res.on('end', function () {
resolve(body);
});
});
req.on('error', function (err) {
reject(err);
});
req.write(params);
req.end();
} catch (err) {
reject(err);
}
})
}
var params = '{"query": "select * from mongo.school.student limit 10", "queryType": "SQL"}';
executeService(params).then(function (res) {
console.log("res>>>>>>>>>>>" + res);
})
Is it right way of running drill query from nodejs?if yes how can i decrease the query time?
There is probably not much you can do about this on your end. If this is still an issue for you get in touch with the Drill team on the mailing lists https://drill.apache.org/mailinglists/ so that they can create a ticket and work on improving it.
I am trying to perform a GET request to an API and return the data from the API response to the client. I think the client receives a response before the GET request to the API finishes. How can I change the code to ensure that the response from the API is passed on to the client?
if (request.method == 'POST' && request.url == '/locationdata') {
var body = '';
request.on('data', function (data) {
body += data;
});
request.on('end', function () {
var formattedLocation = body.replace(/[\[\]']+/g, '');
var urlAPI = 'https://api.darksky.net/forecast/166731d8eab28d33a26c5a51023eff4c/' + formattedLocation;
response.writeHead(200, { 'Content-Type': 'application/json' });
var apiData = '';
var apirequest = function () {
https.get(urlAPI, function (response) {
response.on('data', function (data) {
apiData += data;
});
response.on('end', function () {
console.log(apiData);
return apiData;
});
});
}
response.end(apirequest);
});
return;
}
You are ending the response to the client before you get all the data from the api. Moving the response.end() call up to the end of the api response should fix it:
if (request.method == 'POST' && request.url == '/locationdata') {
var body = '';
request.on('data', function (data) {
body += data;
});
request.on('end', function () {
var formattedLocation = body.replace(/[\[\]']+/g, '');
var urlAPI = 'https://api.darksky.net/forecast/166731d8eab28d33a26c5a51023eff4c/' + formattedLocation;
response.writeHead(200, { 'Content-Type': 'application/json' });
var apiData = '';
https.get(urlAPI, function (apiResponse) {
apiResponse.on('data', function (data) {
apiData += data;
});
apiResponse.on('end', function () {
console.log(apiData);
// send response to browser after we get all the data from the api
response.end(apiData);
});
});
// remove this because we moved it up
//response.end(apirequest);
});
return;
}
I have a larger process running through a large collection of locations/devices in an API response and I'm trying to get to individual devices and turn that into a response my target system will understand. However it seems my inline HTTP request is not firing.
I've tried moving the callback out of the 'end' event, but I'm not even getting the logging for earlier up in the function. The only logging output I get is the "getting status for zone xyz"
async.each(tempSystem.zones, function(zone, zoneCallback) {
var applianceDiscovered = {};
console.log("getting status for zone", zone);
var options = {
host: host,
path: '/webapi/' + zone.zoneId + '/status',
headers: {'Authorization' : 'Bearer ' + accessToken}
};
var req = https.get(options, function(res) {
if (res.statusCode != 200) {
console.log("error ", res.statusCode);
}
console.log(res);
var bodyChunks = [];
res.on('data', function(chunk) {
bodyChunks.push(chunk);
});
res.on('end', function() {
if (res.statusCode === 200) {
console.log("get zone status: ", res.statusCode);
var body = Buffer.concat(bodyChunks);
var zoneStatus = JSON.parse(body);
console.log(zoneStatus);
zoneCallback();
} else {
console.log(res.statusCode);
}
});
res.on('error', function(error) {
console.log(error);
});
});
req.on('error', function(e) {
console.log("error: ", e);
});
}, function(err){
console.log("finished with zones");
});
I'm trying to limit my use of an external API in my node.js code.
I've set up node rate limiter, but it doesn't seem to be working. I still hit 429's. What else should I be doing that I'm not?
var RateLimiter = require('limiter').RateLimiter; // Rate limits
var limiter = new RateLimiter(1, 2000); // one call every two seconds
self.riotAPI = function(options, cb){
limiter.removeTokens(1, function() {
https.request(options, function(response) {
// Error handling
response.on('error', function (e) {
console.log(e);
});
var str = '';
// Another chunk of data has been recieved, so append it to `str`
response.on('data', function (chunk) {
str += chunk;
});
// Parse and return the object
response.on('end', function () {
if(response.statusCode >= 400) {
var err = "HTTP response "+response.statusCode;
console.log(err);
cb(new Error("err"), null);
}
else {
cb(null, JSON.parse(str));
}
});
}).end();
});
}
I switched to Bottleneck and got everything functioning as desired.
self.riotAPI = function(options, cb){
limiter.submit( function(lcb) {
https.request(options, function(response) {
// Error handling
response.on('error', function (e) {
console.log(e);
});
var str = '';
// Another chunk of data has been recieved, so append it to `str`
response.on('data', function (chunk) {
str += chunk;
});
// Parse and return the object
response.on('end', function () {
if(response.statusCode >= 400) {
var err = "HTTP response "+response.statusCode;
console.log(err);
// If it's a 429, retry
if(response.statusCode == 429) {
console.log("retrying...");
self.riotAPI(options, cb);
}
// If not, fail
else {
cb(new Error("err"), null);
lcb();
}
}
else {
cb(null, JSON.parse(str));
lcb();
}
});
}).end();
}, null);
}