NodeJS eventloop execution order - node.js

When sending several requests serially, it seems like
callbacks are executed when all requests are sent.
requests seems to get added into a queue but not actually getting executed before the loop is done.
Example
var http = require('http');
var i=0;
var postData = [];
while (i++ < 12) {
var largeObject = [
'this' + i,
'that' + i,
'then' + i
];
postData.push(largeObject);
if (postData.length >= 2) {
console.log('request');
var options = {
hostname: 'httpbin.org',
port: 80,
path: '/post',
method: 'POST',
headers: {
'Content-Type': 'application/json',
}
};
var req = http.request(options, function(res) {
console.log(res.statusCode);
res.on('data', function(chunk) {
console.log(chunk.toString());
});
res.on('end', function() {
console.log('finished reading response');
});
});
req.end(JSON.stringify(postData), function() {
console.log('request stream finished');
});
postData = [];
}
}
The output here would look like
request
request
request
request
request
request
request stream finished
request stream finished
request stream finished
request stream finished
request stream finished
request stream finished
200
{
"args": {},
"data": "[[\"this7\",\"that7\",\"then7\"],[\"this8\",\"that8\",\"then8\"]]",
"files": {},
"form": {},
"headers": {
"Content-Length": "53",
"Content-Type": "application/json",
"Host": "httpbin.org"
},
"json": [
[
"this7",
"that7",
"then7"
],
[
"this8",
"that8",
"then8"
]
],
"origin": "xxxxx",
"url": "http://httpbin.org/post"
}
finished reading response
200
{
"args": {},
"data": "[[\"this1\",\"that1\",\"then1\"],[\"this2\",\"that2\",\"then2\"]]",
"files": {},
"form": {},
"headers": {
"Content-Length": "53",
"Content-Type": "application/json",
"Host": "httpbin.org"
},
"json": [
[
"this1",
"that1",
"then1"
],
[
"this2",
"that2",
"then2"
]
],
"origin": "xxxx",
"url": "http://httpbin.org/post"
}
...
Is there any way to finish one request before the next one is getting executed?
I don't really care about the order, it's more about memory - I'd like to get rid of large objects I am posting
e.g.
request1
response1
request2
response2
request3
response3
request4
request5
response5
response4
would be absolutely fine.
Any suggestion?

Of course, just use some control-flow modules like async or bluebird.
As you don't care about order, I'd advise you to use either async#parallel or bluebird#map. The bluebird#map has a concurrency argument, which can be nice when you need some more control over loops and their amount (so does async#parallelLimit).
If this doesn't seem straightforward, please comment and I'll add an example.

Related

Azure Cognitve Search upload document via browser cors issue

I want to use the cognitive search to store usernames and levels of a browser game. There should be an option to create a new username in the web frontend. The idea is, that the user enters a username and submits via button. Then these username should be saved in a cognitive search document.
MyiIssue: I am getting a cors error in browser console:
Access to XMLHttpRequest at 'https://XXXXXXX.search.windows.net/indexes/users/docs/index' from origin 'http://localhost:3000' has been blocked by CORS policy: Response to preflight request doesn't pass access control check: No 'Access-Control-Allow-Origin' header is present on the requested resource.
The same issue occurs when I deploy it to a azure static web app.
The CORS settings at azure allow all origins.
Regarding to Azure Search and CORS issue with PUT, it is not possible to modify the data in this way. I am new to programming, how can it be done in another way (serverless function?)?
My code:
OnSubmitCreate(){
console.log('Create started')
let data = JSON.stringify({
"value": [
{
"#search.action": "mergeOrUpload",
"id": this.createName,
"username": this.createName,
"level": 1
}
]
});
let config = {
method: 'post',
url: 'https://XXXXXXXXXXX.search.windows.net/indexes/users/docs/index?api-version=2020-06-30',
headers: {
'api-key': 'XXXXXXXXXXXXXXXXXxx',
'Content-Type': 'application/json'
},
data : data
};
Thanks in advance!
I`ve solved it with creating a azure function using javascript. When I call the URL, the write to cognitive search will be done.
So I technically am doing a http rest call (push).
index.js
var https = require('https');
module.exports = async function (context, req) {
context.log('JavaScript HTTP trigger function processed a request.');
var username = context.bindingData.username;
var level = context.bindingData.level;
let data = JSON.stringify({
"value": [
{
"#search.action": "mergeOrUpload",
"id": username,
"username": username,
"level": level
}
]
})
var options = {
host: 'XXXXXXXXXXX.search.windows.net',
path: '/indexes/users/docs/index?api-version=2020-06-30',
port: 443,
method: 'POST',
headers: {
'api-key': 'XXXXXXXXXXXX',
'Content-Type': 'application/json'
},
body: data
};
var req = https.request(options, function(res) {
console.log('STATUS: ' + res.statusCode);
console.log('HEADERS: ' + JSON.stringify(res.headers));
res.setEncoding('utf8');
res.on('data', function (chunk) {
console.log('BODY: ' + chunk);
});
});
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
});
// write data to request body
req.write(data);
req.end();
context.res = {
// status: 200, /* Defaults to 200 */
body: req
};
}
function.json
{
"bindings": [
{
"authLevel": "function",
"type": "httpTrigger",
"direction": "in",
"name": "req",
"methods": [
"get",
"post"
],
"route": "data/{username:alpha}/{level:int?}"
},
{
"type": "http",
"direction": "out",
"name": "res"
}
]
}

Is there a "standard" way to construct an HTTP GET request for an API?

I'm trying to work with a new API provided by a messaging service.
Their documentation says their API can be queried with a "standard" HTTPS "GET" request.
Now to me, GET request URLS look like this:
https://api.domain.com/path/subpath/?params&otherparams
I should be able to construct the above using their "documentation", which is an example given in node.js:
var http = require("https");
var options = {
"method": "GET",
"hostname": "api-cpaas.dotdigital.com",
"auth": "apiuser:password",
"port": null,
"path": "/messages/{Message Id}",
"headers": {
"content-type": "application/json",
"cache-control": "no-cache"
}
};
var req = http.request(options, function (res) {
var chunks = [];
res.on("data", function (chunk) {
chunks.push(chunk);
});
res.on("end", function () {
var body = Buffer.concat(chunks);
console.log(body.toString());
});
});
req.end();
Example message ID: 6df8b42b-7165-4099-bd9b-356b2f41f1a9
Published here:
https://developer-cpaas.dotdigital.com/reference/getting-message-statuses
But no matter what I try I always get a "404" error.
I don't know node.js so I can't interpret their example, and they don't offer any other examples. What should the actual GET request for this service should look like?

Post Request from NodeJS results in empty package

I am trying to make a http post request towards an external API from my NodeJS application. I tried two different approaches I found on the web, however both failed with different issues. The working curl command that results in the correct request looks like this:
curl -d ' { "auth_token":"XXXXXXXXX",
"hrows": [ {"cols": [ {"value":"Name 0"}, {"value":"Value 0"} ] } ],
"rows": [ {"cols": [ {"value":"Name 1"}, {"value":"Value 1"} ] },
{"cols": [ {"value":"Name 2"}, {"value":"Value 2"} ] },
{"cols": [ {"value":"Name 3"}, {"value":"Value 3"} ] },
{"cols": [ {"value":"Name 4"}, {"value":"Value 4"} ] } ]
}' http://example.com:3030/widgets/alarms
1) Trying to make a request with the request library. This doesn't throw an error in the application, but I get an empty request on the API Server (Yes, the content I want to send is a string)
var request = require('request');
var test = "{\"auth_token\":\"XXXXXXXXXX\", \"hrows\": [ {\"cols\": [{\"value\":\"Loc Nr.\"},{\"value\":\"Address\"},{\"value\":\"Status\"}] } ], \"rows\": [ {\"cols\": [ {\"value\":\"Name 1\"}, {\"value\":\"Value 1\"} ] }, {\"cols\": [ {\"value\":\"Name 2\"}, {\"value\":\"Value 2\"} ] } ]}";
var wid = "alarms";
postRequest(wid,test);
function postRequest(widget, content) {
var headers = {
'User-Agent': 'Super Agent/0.0.1',
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': Buffer.byteLength(content)
}
var options = {
traditional: true,
url: 'http://example.com:3030/widgets/'+widget,
method: 'POST',
headers: headers,
data: content,
contentType : "application/x-www-form-urlencoded"
}
console.log(options);
request.post(options, function (error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body)
}
})
}
When I do a tcpdump on the API server, I see that the packet received looks similar to the one received when executing the curl command, except for the field http.content_length_header being set to 0 (And the conrent itself is missing). When I check the debug output of the options variable, it sill looks ok:
{ traditional: true,
url: 'http://example.com:3030/widgets/alarms',
method: 'POST',
headers:
{ 'User-Agent': 'Super Agent/0.0.1',
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': 243 },
data:
'{"auth_token":"XXXXXXXXX", "hrows": [ {"cols": [{"value":"Loc Nr."},{"value":"Address"},{"value":"Status"}] } ], "rows": [ {"cols": [ {"value":"Name 1"}, {"value":"Value 1"} ] },\t{"cols": [ {"value":"Name 2"}, {"value":"Value 2"} ] } ]}',
contentType: 'application/x-www-form-urlencoded' }
As you can see content length as well as data are there but not in the packet received (See attached Screenshots, one from the curl command and one from the NodeJS request, both captured on the API server)
Any idea why this is happening?
2) The second way I tried was from a Stackoverflow Post I found on how to post a string. However that one fails with
Error: connect ECONNREFUSED 127.0.0.1:80
though I don't understand why it wants to bind to port 80 on my localhost, however method #1 would anyway be the preferred one, this is just for completeness (And maybe someone knows a smart answer to this as well
var querystring = require('querystring');
var http = require('http');
var fs = require('fs');
var test = "{\"auth_token\":\"XXXXXXXX\", \"hrows\": [ {\"cols\": [{\"value\":\"Loc Nr.\"},{\"value\":\"Address\"},{\"value\":\"Status\"}] } ], \"rows\": [ {\"cols\": [ {\"value\":\"Name 1\"}, {\"value\":\"Value 1\"} ] }, {\"cols\": [ {\"value\":\"Name 2\"}, {\"value\":\"Value 2\"} ] } ]}";
var wid = "alarms";
postRequest(wid,test);
function postRequest(widget, content) {
var headers = {
'User-Agent': 'Super Agent/0.0.1',
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': Buffer.byteLength(content)
}
var options = {
url: 'http://example.com:3030/widgets/'+widget,
method: 'POST',
headers: headers,
}
var pdata = querystring.stringify(content);
console.log(pdata)
console.log(Object.prototype.toString.call(pdata));
var post_req = http.request(options, function() {
});
// post the data
post_req.write(pdata);
post_req.end();
}
Thanks for any useful hints (Or a hint how to trigger a dashing job using the dashing API but as far as I see that's not possible so I have to stick with these API calls :) )
It goes wrong in
var options = {
traditional: true,
url: 'http://example.com:3030/widgets/'+widget,
method: 'POST',
headers: headers,
data: content,
contentType : "application/x-www-form-urlencoded"
}
data isn't a valid key on request options, it's either body or form. That would
explain why no body data is being sent in the request.
In your case you want form
form - when passed an object or a querystring, this sets body to a querystring representation of value, and adds Content-type:
application/x-www-form-urlencoded header. When passed no options, a
FormData instance is returned (and is piped to request). See "Forms"
section above.
var options = {
traditional: true,
url: 'http://example.com:3030/widgets/'+widget,
method: 'POST',
headers: headers,
form: content,
contentType : "application/x-www-form-urlencoded"
}
see https://www.npmjs.com/package/request#requestoptions-callback for more details.

HTTP Post request not going through in AWS Lambda

I am trying to add an item to my Todoist project through an Alexa skill in AWS Lambda. I am very new to all of these technologies so forgive me if the fix is incredibly obvious. When I ask Alexa to invoke my addZipcode skill, it fails. This is what I have (excluding some stuff that is in all Alexa Lambda functions):
Alexa stuff
...
const handlers = {
'LaunchRequest': function() {
this.emit('AMAZON.HelpIntent');
},
'addZipcode': function() {
const newZip = this.event.request.intent.slots.zipcode.value;
const speechOutput = newZip;
var http = require("https");
function postZip(newZip) {
var options = {
"method": "POST",
"hostname": [
"beta",
"todoist",
"com"
],
"path": [
"API",
"v8",
"tasks"
],
"headers": {
"Content-Type": "application/json",
"Authorization": "Bearer " + token
}
};
var req = http.request(options, function(res) {
var chunks = [];
res.on("data", function(chunk) {
chunks.push(chunk);
});
res.on("end", function() {
var body = Buffer.concat(chunks);
console.log(body.toString());
});
});
req.write(JSON.stringify({ content: newZip, project_id: XXXXXXXXXX }));
req.end();
}
postZip(newZip);
this.response.cardRenderer(SKILL_NAME, newZip);
this.response.speak(speechOutput);
this.emit(':responseReady');
},
.... cont
I get the resulting error when I try to run the skill with Alexa:
Response:
{
"errorMessage": "hostHeader.replace is not a function",
"errorType": "TypeError",
"stackTrace": [
"Agent.addRequest (_http_agent.js:130:39)",
"new ClientRequest (_http_client.js:159:16)",
"Object.exports.request (http.js:31:10)",
"Object.exports.request (https.js:199:15)",
"postZip (/var/task/index.js:72:28)",
"Object.addZipcode (/var/task/index.js:88:9)",
"emitNone (events.js:86:13)",
"AlexaRequestEmitter.emit (events.js:185:7)",
"AlexaRequestEmitter.EmitEvent (/var/task/node_modules/alexa-sdk/lib/alexa.js:216:10)",
"AlexaRequestEmitter.ValidateRequest (/var/task/node_modules/alexa-sdk/lib/alexa.js:181:23)"
]
}
I tried searching for more information about hostHeader.replace or even just hostHeader but to no avail. When I surround my postZip function with
exports.handler = function(event, context, callback) {}
the skill actually works, but the Post request does not go through (as in, the new zipcode is not added as a new task on my Todoist). I'm pretty sure the Post request code itself is correct because I ran it through Postman and the zipcode was added.
Please help me understand why it doesn't work.
It's hard to tell what causes that error. But the node docs say, that hostname as well as path are supposed to be nothing but strings and not arrays as it's the case in your code.
So what I'd do first is to change your code to this:
var options = {
"method": "POST",
"hostname": "beta.todoist.com",
"path": "/API/v8/tasks",
"headers": {
"Content-Type": "application/json",
"Authorization": "Bearer " + token
}

Node HTTP Delete request no longer works after upgrading from 0.10.40

I've been trying to upgrade most of my node stuff to 4.x but am running into some problems with executing delete queries to Elasticsearch. The following worked on 0.10.40 and prior, but does not work on 4.x.x or 5.7.0. I'm out of ideas, it appears that node doesn't send the body of my request as the error I get back from Elasticsearch is {"error":"ActionRequestValidationException[Validation Failed: 1: source is missing;]","status":400}.
var http = require('http');
var request = http.request({
host: 'localhost',
port: 9200,
path: 'test/col/_query',
method: 'DELETE'
});
request.on('response', function(res) {
res.setEncoding('utf8');
res.on('data', function(response) {
if(response.indexOf('"failed":0') === -1) {
console.log('Failed. Response: ', response);
process.exit(1);
}
});
res.on('end', function() {
console.log('completed successfully');
process.exit(0);
});
});
request.on('error', function(err) { cb(err); });
var q = {
query: {
"bool": {
"must_not": [
{"ids": {"values": ['1','2','3'] } }
]
}
}
};
request.write(JSON.stringify(q));
request.end();
HTTP DELETE requests are not supposed to contain a payload. However, in 0.10 it was supported due to the fact that the Transfer-Encoding: Chunked HTTP header was being sent along the request even if no payload was sent to the server. This was fixed in 0.11.14 by issue 6164.
From now on, if you really need to send a body with a DELETE request, you also need to add a Content-Length header specifying the length of what you're sending, otherwise the server will disregard any payload.
So if you construct your request like this, it will work:
var q = {
query: {
"bool": {
"must_not": [
{"ids": {"values": ['1','2','3'] } }
]
}
}
};
var payload = JSON.stringify(q);
var request = http.request({
host: 'localhost',
port: 9200,
path: 'test/col/_query',
method: 'DELETE',
headers: { <--- add this
'Content-Type': 'application/json',
'Content-Length': Buffer.byteLength(payload)
}
});
...
request.write(payload);
request.end();

Resources