I have a node.js server that making POST requests to an external API, each time I have to make ~10k requests (don't worry I'm not abusing the API) and I need that it will take around 2-3 minutes.
I'm using request-promise library in order to make the requests along with Promise.all() to wait for all the requests to resolve.
My problem is that the requests seems stuck and not running in parallel, I know that the promise executes as soon it's created but it seems that the resolve event can only listen to about 10 events at one time.
I tried updating the maxListeners and also using es6-promise-pool (with pool of 500) but no luck.
My next solution will probably be to use child-process with fork, will this solution seems the best for my problem?
Thanks!
code:
async function send_msg(msg) {
return new Promise(function (resolve, reject) {
request.post(options, function (err, res, body) {
if (err) {
logger.error('error sending msg ' + err);
resolve(null);
} else {
resolve(body);
}
})
});
}
}
async function send_msgs() {
let msgs = await OutgoingMessage.findAll();
for (let i = 0; i < msgs.length; i++) {
promises.push(send_msg(msgs[i]).then(async (result) => {
if (result != null) {
try {
let sid = result['MessageSid'];
let status = result['Status'];
msgs[i].update({sid: sid, status: status});
} catch (e) {
logger.error(e + JSON.stringify(result));
msgs[i].update({status: 'failed'});
}
}
}));
}
return Promise.all(promises);
}
Related
Here is my scenario.
I'm using NodeJS to make API requests.
I need to make an API request to fetch a set of data. I receive a limited set of records with a field called counter, which will let me know if I need to make another request to fetch the remaining data (pagination basically).
So I need to make multiple requests to the same API to fetch the total data.
Here is what I have tried. The resolve function is not returning data.
What should be the right syntax to achieve this?
app.post('/api/fetchData', async function(req, res) {
try {
var totalData = await getDataFromLoop(token, '');
} catch (e) {
console.log('Exception', e);
}
});
var loopArray = [];
function getDataFromLoop(accessToken, counter){
request({
url: 'API URL',
auth: {
'bearer': accessToken
}
}, function(err, response) {
if(err)
{
}
else
{
var res_data = JSON.parse(response.body);
if(res_data.hasOwnProperty('hasMore'))
{
loopArray.push(res_data.data);
console.log('One More Loop to go', res_data.offset);
getDataFromLoop(accessToken, res_data.offset);
}
else
{
console.log('Looping Done');
loopArray.push(res_data.data);
return new Promise(function(resolve, reject) {
resolve(loopArray);
});
}
}
});
}
You need to move the Promise outside of the request function call in getDataFromLoop
function getDataFromLoop(accessToken, counter){
return new Promise((resolve, reject) => {
request(...,
...
if (res_data.hasOwnProperty('hasMore')) {
...
getDataFromLoop(accessToken, res_data.offset).then(resolve)
} else {
...
resolve(loopArray)
}
)
})
}
Since you're using async...await, I guess you can achieve that with a simple while loop, like so:
let shouldFetchAgain = true;
let finalData = [];
while (shouldFetchAgain) {
const data = await fetch('<some_url>');
shouldFetchAgain = data.currentCount < data.total;
finalData.push(data.content);
}
// Process `finalData` here.
The code snippet above demonstrates how you can leverage while loop to decide whether you need to fetch the data again. This looks like a polling technique you're using to fetch paginated data.
Alternatively, you can check out PollingObserver.
Hope this helps you.
I'm new to NodeJS and I'm currently working on node-soap module that can be found from https://github.com/vpulim/node-soap.
I'm calling a Web Service by using the referenced module. I need to return the SOAP client's response to user's web browser, however, the problem is that the response gets returned before it is fetched to an array. This is related to asynchronous way of working.
The second problem is that I need to call the Web Service again until I get specific amount of results. Each result will be pushed to the same array. How to do this? I need to return this array to user but as described before, it is always empty.
How would you use soap.createClientAsync and client.methodAsync in this case?
I have already tried writing a while-loop that continues until I get specific amount of results. I tried wrapping soap.createClient to a promise as well as soap.method. Those promises are in different functions and I tried to call them in async function which returns the array.
function createSoapClient() {
return new Promise(function(resolve, reject) {
var url = '...';
soap.createClient(url, function(err, client) {
if (err) {
reject(err);
}
resolve(client);
});
});
}
function fetchServiceCustomers(client) {
return new Promise(function(resolve, reject) {
var args = {...};
client.method(args, function(error, result, rawResponse, soapHeader, rawRequest) {
if (error) {
reject(error);
}
resolve(result);
}, {timeout: 60 * 1000});
});
}
exports.getServiceCustomers = async function() {
let client = await createSoapClient();
var results = 0,
completeResult = [];
while (results <= 0 || results >= 10000) {
completeResult.push(await fetchServiceCustomers(client);
results = completeResult[completeResult.length - 1];
console.log(results);
}
return completeResult;
}
I am hitting an API that takes in addresses and gives me back GPS coordinates. The API only accepts a single address, but it can handle 50 live connections at any given time. I am trying to build a function that will send 50 requests, wait until they all return and send 50 more. Or send 50 request and send the next one as a previous is returned. Below is the code I have been working with, but I am stuck.
One issue is in batchFunct. The for loop sends all the API calls, doesn’t wait for them to come back, then runs the if statement before updating returned. This makes since considering the asynchronicity of Node. I tried to put an await on the API call, but that seemingly stops all the async process (anyone have clarification on this) and effectively makes it send the requests one at a time.
Any advice on adapting this code or on finding a better way of batching and queuing API requests?
const array = ['address1', 'address2', 'address3', 'address4', '...', 'addressN']
function batchFunc(array) {
return new Promise(function (resolve, reject) {
var returned = 1
for (let ele of array) {
apiCall(ele).then(resp => { //if but an await here it will send one at a time
console.log(resp)
returned++
})
};
if (returned == array.length) {
resolve(returned);
}
})
}
async function batchCall(array) {
while (array.length > 0) {
let batchArray = []
if (array.length > 50) {
for (let i = 0; i < 50; i++) {
batchArray.push(array[0])
array.splice(0, 1)
}
} else {
batchArray = array
array = []
}
let result = await batchFunc(batchArray);
console.log(result);
}
}
batchCall(array)
I ended up using the async.queue, but I am still very interested in any other solutions.
const array = ['address1', 'address2', 'address3', 'address4', 'address5', 'address6']
function asyncTime(value) {
return new Promise(function (resolve, reject) {
apiCall(ele).then(resp => {
resolve(resp)
})
})
}
function test(array) {
var q = async.queue(async function(task, callback) {
console.log(await asyncTime(task))
if(callback) callback()
}, 3);
q.push(array, function(err) {
if (err) {
console.log(err)
return
}
console.log('finished processing item');
});
}
I have the following code that I cannot figure out what I need to do to make this entire process finish before sending a response back to the client. Basically I am looking for the client to receive a response once everything in the method completes or errors out, not before.
I have tried several things, moving things around and adding some .then() and .catch() blocks that may not even be needed. I'm still fairly new to NodeJs so while I understand how things kind of work, I still have not got used to asynchronous coding.
let markStepsComplete = async function() {
let stepsToProcess = new Multimap();//multimap JS npm
let results = await dbUtils.getCompletedSteps();
results.forEach(result => {
stepsToProcess.set(result.ticket_id, result.step_id);
});
return new Promise((resolve,reject)=>{
stepsToProcess.forEachEntry(async function(entry, key) {
let payload = {
ticketId: key,
stepIds: entry
}
let response = await updateStep(payload)//returns promise
if (response.statusCode === 200) {
await Promise.all(payload.stepIds.map(async (stepId) => {
try {
await dbUtils.markStepsCompleted(stepId, payload.ticketId);
} catch(err) {
reject(err);
}
}));
}
else {
await Promise.all(payload.stepIds.map(async (stepId) => {
try {
await dbUtils.markStepsProcessed(stepId, payload.ticketId, response.statusCode);
} catch(err) {
console.log(err);
reject(err);
}
}));
}
});
resolve('Steps Marked Complete');
});//promise end
}
This is the path that i am testing right now that's making the call to the above method.
The updateStep() method is the method that actually calls a HTTP request to an outside REST API. That seems to be working returning a promise.
app.use('/api/posts',async (req,res,next)=>{
let data = await markStepsComplete.markStepsComplete()
.then((data)=>{
res.status(200).json({message: data})
})
.catch((e)=>{
console.log('error from catch:',e);
})
})
The code above runs and does run some stuff in our database as part of this process, but I get back the "Steps Marked Complete" resolve message before the process finishes.
Since the resolve is hit before the process ends, I can never get any of the reject() errors to come back to the client since resolve is called already.
Thanks in advance.
I noticed this snippet and wanted to call it out because it wont work:
payload.stepIds.forEach(async (stepId) => {
try {
await dbUtils.markStepsProcessed(stepId, payload.ticketId, response.statusCode);
} catch(err) {
console.log(err);
reject(err);
}
});
when looping through an array of items that require an async fetch, you can do something like:
await Promise.all(payload.stepIds.map(async (stepId) => {
try {
await dbUtils.markStepsProcessed(stepId, payload.ticketId, response.statusCode);
} catch(err) {
console.log(err);
reject(err);
}
}));
in React-Native I have a bunch of IPs that I try to fetch simultaneously. The first one to answer with a specific status code is the one I am looking for. This part happens at the app startup, so it needs to be as fast as possible. Using the async library, my code is like this:
// Here an array with a bunch of IPs
// ...
async.detect(uris, function(uri, callback) {
// Fetching a specific URL associated with the IP
fetch(`http://${uri}/productionservice/DataService.svc/`)
.then((response) => {
// If the URL answers with a 401 status code I know it's the one I'm looking for
if(response.status == '401') {
callback(null, true);
// Otherwise It's not
} else {
callback(null, false)
}
})
.catch((error) => {
callback(null, false)
});
}, function(err, result) {
if(typeof(result)=='undefined') {
console.log('No result found');
}
console.log(result);
});
}
However, when one of the tests succeeds, I do get a result, but when none succeeds, the detect method hangs indefinitely, never letting me know that none of the IPs is returning the answer I expect.
My question is: How, using async.detect and RN's fetch, can I fetch multiple links, get a result if my test succeeds, or a false statement if none of them succeeds.
Thank you.
Using async await you can do something along these lines:
async function detect(uris) {
const promises = [];
uris.forEach((uri) => promises.push(fetch(`http://${uri}/productionservice/DataService.svc/`)));
const responses = await Promise.all(promises);
for (let i = 0; i < responses.length; i++) {
if (responses[i] && responses[i].status === '401') {
return true;
}
}
return false;
}