React-native Detecting the first successful fetch - node.js

in React-Native I have a bunch of IPs that I try to fetch simultaneously. The first one to answer with a specific status code is the one I am looking for. This part happens at the app startup, so it needs to be as fast as possible. Using the async library, my code is like this:
// Here an array with a bunch of IPs
// ...
async.detect(uris, function(uri, callback) {
// Fetching a specific URL associated with the IP
fetch(`http://${uri}/productionservice/DataService.svc/`)
.then((response) => {
// If the URL answers with a 401 status code I know it's the one I'm looking for
if(response.status == '401') {
callback(null, true);
// Otherwise It's not
} else {
callback(null, false)
}
})
.catch((error) => {
callback(null, false)
});
}, function(err, result) {
if(typeof(result)=='undefined') {
console.log('No result found');
}
console.log(result);
});
}
However, when one of the tests succeeds, I do get a result, but when none succeeds, the detect method hangs indefinitely, never letting me know that none of the IPs is returning the answer I expect.
My question is: How, using async.detect and RN's fetch, can I fetch multiple links, get a result if my test succeeds, or a false statement if none of them succeeds.
Thank you.

Using async await you can do something along these lines:
async function detect(uris) {
const promises = [];
uris.forEach((uri) => promises.push(fetch(`http://${uri}/productionservice/DataService.svc/`)));
const responses = await Promise.all(promises);
for (let i = 0; i < responses.length; i++) {
if (responses[i] && responses[i].status === '401') {
return true;
}
}
return false;
}

Related

nodejs express search in request parallel single response

i need to query multiple pages from another api and return a object from the page if a specified value matches.
i guess the problem is that the loop is done asychron because i always get "not found" and later i get "Cannot set headers after they are sent to the client" if the loop found the object.
solved this by calling it recursive but i need more speed because there are many pages with many entries. if possible requests should run parallel but not "found should" be called after all loops finished
router.post('/search', function (req, res) {
var foundObj = false;
for (var page = 1; page < req.body.cubesize; page++) {
request({
method: 'GET',
uri: 'http://localhost:8080/api/v1/getpage/json/' + page
},
function (error, response, body) {
if (!error) {
var result = JSON.parse(body);
for (var obj in result) {
console.log(result[obj]);
if (result[obj].hasOwnProperty(req.body.field)) {
if (result[obj][req.body.field] == req.body.value) {
foundObj = true;
return res.status(200).send(result[obj]);
}
}
}
}
});
}
if(!foundObj){
return res.status(404).send("not found");
}
});
anyone got an idea how to fast loop all pages with all entries but wait for calling not found?
As long as you have a res.send() inside a for loop and at least two matches occurs, two (at least) res.send() calls will be executed and an error will rise.
How to run in parallel ?
router.post('/search', function (req, res) {
const callApi = (page) => new Promise( (resolve, reject) => {
request({
method: 'GET',
uri: `http://localhost:8080/api/v1/getpage/json/${page}`,
},
function (error, response, body) {
if (error) reject(null)
let result = JSON.parse(body);
for (var obj in result) {
console.log(result[obj]);
if (result[obj].hasOwnProperty(req.body.field)) {
if (result[obj][req.body.field] == req.body.value)
return resolve(result[obj]);
}
}
return reject(null);
}
});
});
const promisesArr = [];
for ( let page = 1; page < req.body.cubesize; page++) {
promisesArr.push(callApi(page))
}
Promise.allSettled(promisesArr).then((resArr)=>{
const resolvedArray = resArr.filter(val => !!val);
if (resolvedArray.length === 0) return res.status(404).send("not found");
if (resolvedArray.length === 1)
return res.status(200).send(resolvedArr[0][obj])
if (resolvedArray.length > 1)
return res.status(500).send("Too many matches")
// It is not clear to me in your code what you need to do in case more than one resolves
});
});
Some explanation about the code.
The idea is to promisify request and run in parallel
To run in parallel, Promise object allows four methods:
Promise.all, Promise.race and Promise.allSettled and Promise.any. The two last ones, Promise.allSettled and Promise.any are not fully compatible, so keep this in mind.
Once you have the array and run in parallel, Promise.all and Promise.allSettled returns an array of results. The array is filtered and if some value matchs, it response that, otherwise, response 404.
Further information about promises will be required to select the right one for your specific case. You can found about it here[https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise]
Unfortunately my code is not tested, so please review it and refactor to adapt to your specific case.

How to use async await when the function in await needs to be looped dynamically

Here is my scenario.
I'm using NodeJS to make API requests.
I need to make an API request to fetch a set of data. I receive a limited set of records with a field called counter, which will let me know if I need to make another request to fetch the remaining data (pagination basically).
So I need to make multiple requests to the same API to fetch the total data.
Here is what I have tried. The resolve function is not returning data.
What should be the right syntax to achieve this?
app.post('/api/fetchData', async function(req, res) {
try {
var totalData = await getDataFromLoop(token, '');
} catch (e) {
console.log('Exception', e);
}
});
var loopArray = [];
function getDataFromLoop(accessToken, counter){
request({
url: 'API URL',
auth: {
'bearer': accessToken
}
}, function(err, response) {
if(err)
{
}
else
{
var res_data = JSON.parse(response.body);
if(res_data.hasOwnProperty('hasMore'))
{
loopArray.push(res_data.data);
console.log('One More Loop to go', res_data.offset);
getDataFromLoop(accessToken, res_data.offset);
}
else
{
console.log('Looping Done');
loopArray.push(res_data.data);
return new Promise(function(resolve, reject) {
resolve(loopArray);
});
}
}
});
}
You need to move the Promise outside of the request function call in getDataFromLoop
function getDataFromLoop(accessToken, counter){
return new Promise((resolve, reject) => {
request(...,
...
if (res_data.hasOwnProperty('hasMore')) {
...
getDataFromLoop(accessToken, res_data.offset).then(resolve)
} else {
...
resolve(loopArray)
}
)
})
}
Since you're using async...await, I guess you can achieve that with a simple while loop, like so:
let shouldFetchAgain = true;
let finalData = [];
while (shouldFetchAgain) {
const data = await fetch('<some_url>');
shouldFetchAgain = data.currentCount < data.total;
finalData.push(data.content);
}
// Process `finalData` here.
The code snippet above demonstrates how you can leverage while loop to decide whether you need to fetch the data again. This looks like a polling technique you're using to fetch paginated data.
Alternatively, you can check out PollingObserver.
Hope this helps you.

Synchronous while-loop and array push

I'm new to NodeJS and I'm currently working on node-soap module that can be found from https://github.com/vpulim/node-soap.
I'm calling a Web Service by using the referenced module. I need to return the SOAP client's response to user's web browser, however, the problem is that the response gets returned before it is fetched to an array. This is related to asynchronous way of working.
The second problem is that I need to call the Web Service again until I get specific amount of results. Each result will be pushed to the same array. How to do this? I need to return this array to user but as described before, it is always empty.
How would you use soap.createClientAsync and client.methodAsync in this case?
I have already tried writing a while-loop that continues until I get specific amount of results. I tried wrapping soap.createClient to a promise as well as soap.method. Those promises are in different functions and I tried to call them in async function which returns the array.
function createSoapClient() {
return new Promise(function(resolve, reject) {
var url = '...';
soap.createClient(url, function(err, client) {
if (err) {
reject(err);
}
resolve(client);
});
});
}
function fetchServiceCustomers(client) {
return new Promise(function(resolve, reject) {
var args = {...};
client.method(args, function(error, result, rawResponse, soapHeader, rawRequest) {
if (error) {
reject(error);
}
resolve(result);
}, {timeout: 60 * 1000});
});
}
exports.getServiceCustomers = async function() {
let client = await createSoapClient();
var results = 0,
completeResult = [];
while (results <= 0 || results >= 10000) {
completeResult.push(await fetchServiceCustomers(client);
results = completeResult[completeResult.length - 1];
console.log(results);
}
return completeResult;
}

Multiple and fast external POST requests

I have a node.js server that making POST requests to an external API, each time I have to make ~10k requests (don't worry I'm not abusing the API) and I need that it will take around 2-3 minutes.
I'm using request-promise library in order to make the requests along with Promise.all() to wait for all the requests to resolve.
My problem is that the requests seems stuck and not running in parallel, I know that the promise executes as soon it's created but it seems that the resolve event can only listen to about 10 events at one time.
I tried updating the maxListeners and also using es6-promise-pool (with pool of 500) but no luck.
My next solution will probably be to use child-process with fork, will this solution seems the best for my problem?
Thanks!
code:
async function send_msg(msg) {
return new Promise(function (resolve, reject) {
request.post(options, function (err, res, body) {
if (err) {
logger.error('error sending msg ' + err);
resolve(null);
} else {
resolve(body);
}
})
});
}
}
async function send_msgs() {
let msgs = await OutgoingMessage.findAll();
for (let i = 0; i < msgs.length; i++) {
promises.push(send_msg(msgs[i]).then(async (result) => {
if (result != null) {
try {
let sid = result['MessageSid'];
let status = result['Status'];
msgs[i].update({sid: sid, status: status});
} catch (e) {
logger.error(e + JSON.stringify(result));
msgs[i].update({status: 'failed'});
}
}
}));
}
return Promise.all(promises);
}

Wait for nodeJs async/promise process to end before sending response to client

I have the following code that I cannot figure out what I need to do to make this entire process finish before sending a response back to the client. Basically I am looking for the client to receive a response once everything in the method completes or errors out, not before.
I have tried several things, moving things around and adding some .then() and .catch() blocks that may not even be needed. I'm still fairly new to NodeJs so while I understand how things kind of work, I still have not got used to asynchronous coding.
let markStepsComplete = async function() {
let stepsToProcess = new Multimap();//multimap JS npm
let results = await dbUtils.getCompletedSteps();
results.forEach(result => {
stepsToProcess.set(result.ticket_id, result.step_id);
});
return new Promise((resolve,reject)=>{
stepsToProcess.forEachEntry(async function(entry, key) {
let payload = {
ticketId: key,
stepIds: entry
}
let response = await updateStep(payload)//returns promise
if (response.statusCode === 200) {
await Promise.all(payload.stepIds.map(async (stepId) => {
try {
await dbUtils.markStepsCompleted(stepId, payload.ticketId);
} catch(err) {
reject(err);
}
}));
}
else {
await Promise.all(payload.stepIds.map(async (stepId) => {
try {
await dbUtils.markStepsProcessed(stepId, payload.ticketId, response.statusCode);
} catch(err) {
console.log(err);
reject(err);
}
}));
}
});
resolve('Steps Marked Complete');
});//promise end
}
This is the path that i am testing right now that's making the call to the above method.
The updateStep() method is the method that actually calls a HTTP request to an outside REST API. That seems to be working returning a promise.
app.use('/api/posts',async (req,res,next)=>{
let data = await markStepsComplete.markStepsComplete()
.then((data)=>{
res.status(200).json({message: data})
})
.catch((e)=>{
console.log('error from catch:',e);
})
})
The code above runs and does run some stuff in our database as part of this process, but I get back the "Steps Marked Complete" resolve message before the process finishes.
Since the resolve is hit before the process ends, I can never get any of the reject() errors to come back to the client since resolve is called already.
Thanks in advance.
I noticed this snippet and wanted to call it out because it wont work:
payload.stepIds.forEach(async (stepId) => {
try {
await dbUtils.markStepsProcessed(stepId, payload.ticketId, response.statusCode);
} catch(err) {
console.log(err);
reject(err);
}
});
when looping through an array of items that require an async fetch, you can do something like:
await Promise.all(payload.stepIds.map(async (stepId) => {
try {
await dbUtils.markStepsProcessed(stepId, payload.ticketId, response.statusCode);
} catch(err) {
console.log(err);
reject(err);
}
}));

Resources