Watch an API for Updates - node.js

I am trying to make an API watch to https://api.exchangeratesapi.io/latest
and detect if any changes so I can update my clients just like a web hook notification. But what am doing is
while(true){
fetch(https://api.exchangeratesapi.io/latest)
.then(res => ......
}
I am caching the result and always check if there is any changes, If there is, I will send a request to the client.
I am looking for a better way to pull the data instead of making a while loops

If you use a while loop, you would potentially send many requests before one returns. And even when one returns, it's not going to necessarily be in order. Here is a quick example of what might happen if there is a network spike for example:
const sleep = ms => new Promise(res => setTimeout(res, ms));
async function mockNetworkCall(num) {
const timings = [50, 150, 50]
console.log(`sending request ${num}`);
await sleep(timings[num]);
console.log(`request ${num} finished`)
}
for(let i = 0; i < 3; i++) {
mockNetworkCall(i);
}
You could avoid the while loop if you instead take the approach to only do a new request when the last one finishes. In that case, you will only have a single request active at any one time and you know that you're getting the result in order.
You can wrap the logic for that in a simple function to watch a URL and only re-initiate a request when the previous one is finished. This is the skeleton for such a function - it might need tweaking according to your needs:
function watch({ url, retryCount, lastResponse = null}) {
fetch(url)
.then(res => {
/* do something */
return res;
})
.then(res => watch({url, retryCount, lastResponse: res})) //launch again
.catch(err => {
/* handle error */
console.error("error getting URL", err);
console.warn("retries left: ", retryCount);
if (retryCount-- > 0) {
watch({url, retryCount, lastResponse});
}
})
}
watch({url: "google.com", retryCount: 3});
Or the same using async/await
async function watch({ url, retryCount, lastResponse = null}) {
try {
const res = await fetch(url);
/* do something */
watch({url, retryCount, lastResponse: res}); //launch again
} catch (err) {
/* handle error */
console.error("error getting URL", err);
console.warn("retries left: ", retryCount);
if (retryCount-- > 0) {
watch({url, retryCount, lastResponse});
}
}
}
watch({url: "google.com", retryCount: 3});

Related

How can I am make sure these chain of functions in Node.js are performed in order (using promises)?

I have a set of functions in Node.js that I would like to load in a certain order. I will provide some mockup code abstracted and simplified:
function updateMyApp() {
loadDataToServer()
.then(() => useData())
.then(() => saveData())
.then(() => { console.log("updateMyApp done") })
}
function loadDataToServer() {
return new Promise( (resolve, reject) {
...preparing data and save file to cloud...
resolve()})
}
function handleDataItem(item) {
// Function that fetches data item from database and updates each data item
console.log("Name", item.name)
}
function saveData() {
// Saves the altered data to some place
}
useData is a bit more complex. In it I would like to, in order:
console.log('Starting alterData()')
Load data, as json, from the cloud data source
Iterate through every item in the json file and do handleDataItem(item) on it.
When #2 is done -> console.log('alterData() done')
Return a resolved promise back to updateMyApp
Go on with saveData() with all data altered.
I want the logs to show:
Starting useData()
Name: Adam
Name: Ben
Name: Casey
useData() done
my take on this is the following:
function useData() {
console.log('Starting useData()')
return new Promise( function(resolve, reject) {
readFromCloudFileserver()
.then(jsonListFromCloud) => {
jsonListFromCloud.forEach((item) => {
handleDataItem(item)
}
})
.then(() => {
resolve() // I put resolve here because it is not until everything is finished above that this function is finished
console.log('useData() done')
}).catch((error) => { console.error(error.message) })
})
}
which seems to work but, as far as I understand this is not how one is supposed to do it. Also, this seems to do the handleDataItem outside of this chain so the logs look like this:
Starting useData()
useData() done
Name: Adam
Name: Ben
Name: Casey
In other words. It doesn't seem like the handleDataItem() calls are finished when the chain has moved on to the next step (.then()). In other words, I can not be sure all items have been updated when it goes on to the saveData() function?
If this is not a good way to handle it, then how should these functions be written? How do I chain the functions properly to make sure everything is done in the right order (as well as making the log events appear in order)?
Edit: As per request, this is handleDataItem less abstracted.
function handleDataItem(data) {
return new Promise( async function (resolve) {
data['member'] = true
if (data['twitter']) {
const cleanedUsername = twitterApi.cleanUsername(data['twitter']).toLowerCase()
if (!data['twitter_numeric']) {
var twitterId = await twitterApi.getTwitterIdFromUsername(cleanedUsername)
if (twitterId) {
data['twitter_numeric'] = twitterId
}
}
if (data['twitter_numeric']) {
if (data['twitter_protected'] != undefined) {
var twitterInfo = await twitterApi.getTwitterGeneralInfoToDb(data['twitter_numeric'])
data['twitter_description'] = twitterInfo.description
data['twitter_protected'] = twitterInfo.protected
data['twitter_profile_pic'] = twitterInfo.profile_image_url.replace("_normal", '_bigger')
data['twitter_status'] = 2
console.log("Tweeter: ", data)
}
} else {
data['twitter_status'] = 1
}
}
resolve(data)
}).then( (data) => {
db.collection('people').doc(data.marker).set(data)
db.collection('people').doc(data.marker).collection('positions').doc(data['report_at']).set(
{
"lat":data['lat'],
"lon":data['lon'],
}
)
}).catch( (error) => { console.log(error) })
}
The twitterAPI functions called:
cleanUsername: function (givenUsername) {
return givenUsername.split('/').pop().replace('#', '').replace('#', '').split(" ").join("").split("?")[0].trim().toLowerCase()
},
getTwitterGeneralInfoToDb: async function (twitter_id) {
var endpointURL = "https://api.twitter.com/2/users/" + twitter_id
var params = {
"user.fields": "name,description,profile_image_url,protected"
}
// this is the HTTP header that adds bearer token authentication
return new Promise( (resolve,reject) => {
needle('get', endpointURL, params, {
headers: {
"User-Agent": "v2UserLookupJS",
"authorization": `Bearer ${TWITTER_TOKEN}`
}
}).then( (res) => {
console.log("result.body", res.body);
if (res.body['errors']) {
if (res.body['errors'][0]['title'] == undefined) {
reject("Twitter API returns undefined error for :'", cleanUsername, "'")
} else {
reject("Twitter API returns error:", res.body['errors'][0]['title'], res.body['errors'][0]['detail'])
}
} else {
resolve(res.body.data)
}
}).catch( (error) => { console.error(error.message) })
})
},
// Get unique id from Twitter user
// Twitter API
getTwitterIdFromUsername: async function (cleanUsername) {
const endpointURL = "https://api.twitter.com/2/users/by?usernames="
const params = {
usernames: cleanUsername, // Edit usernames to look up
}
// this is the HTTP header that adds bearer token authentication
const res = await needle('get', endpointURL, params, {
headers: {
"User-Agent": "v2UserLookupJS",
"authorization": `Bearer ${TWITTER_TOKEN}`
}
})
if (res.body['errors']) {
if (res.body['errors'][0]) {
if (res.body['errors'][0]['title'] == undefined) {
console.error("Twitter API returns undefined error for :'", cleanUsername, "'")
} else {
console.error("Twitter API returns error:", res.body['errors'][0]['title'], res.body['errors'][0]['detail'])
}
} else {
console.error("Twitter API special error:", res.body)
}
} else {
if (res.body['data']) {
return res.body['data'][0].id
} else {
//console.log("??? Could not return ID, despite no error. See: ", res.body)
}
}
},
You have 3 options to deal with your main issue of async methods in a loop.
Instead of forEach, use map and return promises. Then use Promise.all on the returned promises to wait for them to all complete.
Use a for/of loop in combination with async/await.
Use a for await loop.
It sounds like there's a problem in the implementation of handleDataItem() and the promise that it returns. To help you with that, we need to see the code for that function.
You also need to clean up useData() so that it properly returns a promise that propagates both completion and errors.
And, if handleDataItem() returns a promise that is accurate, then you need to change how you do that in a loop here also.
Change from this:
function useData() {
console.log('Starting useData()')
return new Promise( function(resolve, reject) {
readFromCloudFileserver()
.then(jsonListFromCloud) => {
jsonListFromCloud.forEach((item) => {
handleDataItem(item)
}
})
.then(() => {
resolve() // I put resolve here because it is not until everything is finished above that this function is finished
console.log('useData() done')
}).catch((error) => { console.error(error.message) })
})
}
to this:
async function useData() {
try {
console.log('Starting useData()')
const jsonListFromCloud = await readFromCloudFileserver();
for (let item of jsonListFromCloud) {
await handleDataItem(item);
}
console.log('useData() done');
} catch (error) {
// log error and rethrow so caller gets the error
console.error(error.message)
throw error;
}
}
The structural changes here are:
Switch to use async/await to more easily handle the asynchronous items in a loop
Remove the promise anti-pattern that wraps new Promise() around an existing promise - no need for that AND you weren't capturing or propagating rejections from readFromCloudFileServer() which is a common mistake when using that anti-pattern.
rethrow the error inside your catch after logging the error so the error gets propagated back to the caller

Chunking axios.get requests with a 1 second delay per chunk - presently getting 429 error

I have a script using axios that hits an API with a limit of 5 requests per second. At present my request array length is 72 and will grow over time. I receive an abundance of 429 errors. The responses per endpoint change with each run of the script; ex: url1 on iteration1 returns 429, then url1 on iteration2 returns 200, url1 on iteration3 returns 200, url1 on iteration4 returns 429.
Admittedly my understanding of async/await and promises are spotty at best.
What I understand:
I can have multiple axios.get running because of async. The variable I set in my main that uses the async function can include the await to ensure all requests have processed before continuing the script.
Promise.all can run multiple axios.gets but, if a single request fails the chain breaks and no more requests will run.
Because the API will only accept 5 requests per second I have to chunk my axios.get requests to 5 endpoints, wait for those to finish processing before sending the next chunk of 5.
setTimeout will assign a time limit to a single request, once the time is up the request is done and will not be sent again no matter the return being other than 200.
setInterval will assign a time limit but it will send the request again after time's up and keep requesting until it receives a 200.
async function main() {
var endpoints = makeEndpoints(boards, whiteList); //returns an array of string API endpoints ['www.url1.com', 'www.url2.com', ...]
var events = await getData(endpoints);
...
}
The getData() has seen many iterations in attempt to correct the 429's. Here are a few:
// will return the 200's sometimes and not others, I believe it's the timeout but that won't attempt the hit a failed url (as I understand it)
async function getData(endpoints) {
let events = [];
for (x = 0; x < endpoints.length; x++) {
try {
let response = await axios.get(endpoints[x], {timeout: 2000});
if ( response.status == 200 &&
response.data.hasOwnProperty('_embedded') &&
response.data._embedded.hasOwnProperty('events')
) {
let eventsArr = response.data._embedded.events;
eventsArr.forEach(event => {
events.push(event)
});
}
} catch (error) {
console.log(error);
}
}
return events;
}
// returns a great many 429 errors via the setInterval, as I understand this function sets a delay of N seconds before attempting the next call
async function getData(endpoints) {
let data = [];
let promises = [];
endpoints.forEach((url) => {
promises.push(
axios.get(url)
)
})
setInterval(function() {
for (i = 0; i < promises.length; i += 5) {
let requestArr = promises.slice(i, i + 5);
axios.all(requestArr)
.then(axios.spread((...res) => {
console.log(res);
}))
.catch(err => {
console.log(err);
})
}
}, 2000)
}
// Here I hoped Promise.all would allow each request to do its thing and return the data, but after further reading I found that if a single request fails the rest will fail in the Promise.all
async getData(endpoints) {
try {
const res = await Promise.all(endpoints.map(url => axios.get(url))).catch(err => {});
} catch {
throw Error("Promise failed");
}
return res;
}
// Returns so many 429 and only 3/4 data I know to expect
async function getData(endpoints) {
const someFunction = () => {
return new Promise(resolve => {
setTimeout(() => resolve('222'), 100)
})
}
const requestArr = endpoints.map(async data => {
let waitForThisData = await someFunction(data);
return axios.get(data)
.then(response => { console.log(response.data)})
.catch(error => console.log(error.toString()))
});
Promise.all(requestArr).then(() => {
console.log('resolved promise.all')
})
}
// Seems to get close to solving but once an error is it that Promise.all stops processing endpoint
async function getData(endpoints) {
(async () => {
try {
const allResponses = await Promise.all(
endpoints.map(url => axios.get(url).then(res => console.log(res.data)))
);
console.log(allResponses[0]);
} catch(e) {
console.log(e);
// handle errors
}
})();
}
It seems like I have so many relevant pieces but I cannot connect them in an efficient and working model. Perhaps axios has something completely unknown to me? I've also tried using blurbird concurrent to limit the request to 5 per attempt but that still returned the 429 from axios.
I've been starring at this for days and with so much new information swirling in my head I'm at a loss as to how to send 5 requests per second, await the response, then send another set of 5 requests to the API.
Guidance/links/ways to improve upon the question would be much appreciated.

How to handle an HTTP request that stops sending data

I'm trying to download videos from some URLs and for some reason the downloads will sometimes stop before they're complete - as in res.on('data', ...) will stop being called (using https.get) before the response has completed. I would like to be able to detect this and retry the request after a given time, but I'm having a hard time getting working code for it. I'm hoping there would be a built in way to deal with something like this that I'm missing. I've already tried setting a timeout for the request and setting a timeout for the response and neither of those seem to do the trick.
Current attempt:
async function downloadVideo(send) {
try {
https.get(downloadUrl, res => {
let timerId;
res.on(`data`, (c) => {
clearTimeout(timerId);
timerId = setTimeout(() => {
res.destroy();
console.log(`retrying`, name);
downloadVideo(send);
}, 2000);
if (!throttle) {
send(`PercentDownloaded`, [
index,
100 * fileSize(filePath) / res.headers[`content-length`]
]);
throttle = true;
setTimeout(() => throttle = false, 500);
}
});
res
.pipe(fs.createWriteStream(filePath))
.on(`finish`, () => {
clearTimeout(timerId);
send(`Done`, index);
});
});
} catch (error) {
console.log(name, error);
send(`DownloadError`, index);
}
}

Batching and Queuing API calls in Node

I am hitting an API that takes in addresses and gives me back GPS coordinates. The API only accepts a single address, but it can handle 50 live connections at any given time. I am trying to build a function that will send 50 requests, wait until they all return and send 50 more. Or send 50 request and send the next one as a previous is returned. Below is the code I have been working with, but I am stuck.
One issue is in batchFunct. The for loop sends all the API calls, doesn’t wait for them to come back, then runs the if statement before updating returned. This makes since considering the asynchronicity of Node. I tried to put an await on the API call, but that seemingly stops all the async process (anyone have clarification on this) and effectively makes it send the requests one at a time.
Any advice on adapting this code or on finding a better way of batching and queuing API requests?
const array = ['address1', 'address2', 'address3', 'address4', '...', 'addressN']
function batchFunc(array) {
return new Promise(function (resolve, reject) {
var returned = 1
for (let ele of array) {
apiCall(ele).then(resp => { //if but an await here it will send one at a time
console.log(resp)
returned++
})
};
if (returned == array.length) {
resolve(returned);
}
})
}
async function batchCall(array) {
while (array.length > 0) {
let batchArray = []
if (array.length > 50) {
for (let i = 0; i < 50; i++) {
batchArray.push(array[0])
array.splice(0, 1)
}
} else {
batchArray = array
array = []
}
let result = await batchFunc(batchArray);
console.log(result);
}
}
batchCall(array)
I ended up using the async.queue, but I am still very interested in any other solutions.
const array = ['address1', 'address2', 'address3', 'address4', 'address5', 'address6']
function asyncTime(value) {
return new Promise(function (resolve, reject) {
apiCall(ele).then(resp => {
resolve(resp)
})
})
}
function test(array) {
var q = async.queue(async function(task, callback) {
console.log(await asyncTime(task))
if(callback) callback()
}, 3);
q.push(array, function(err) {
if (err) {
console.log(err)
return
}
console.log('finished processing item');
});
}

Axios.all, how to configure axios wait time to mitigate hung up?

My application uses an internal webservice for fetching data, i have a job which creates approx 500 requests which getsfired async to complete the fetch operation.
I make use of Axios, by creating an array of axios promises and then resolving them using using Axios.all();
It works fine until some 200 requests but post that i get socket hung up, however on the server side i see the requests are being processed.
How to configure axios to set custom time out, or is it a better idea to splice my promises array and then run them as multiple batches ?
Source code
let getAxiosPromiseArray = (urlList) => {
var axiosArrayofPromise = [];
return new Promise ( (resolve, reject) => {
try {
urlList.forEach ( (URL) => {
axiosArrayofPromise.push(axios.get(URL));
});
resolve(axiosArrayofPromise);
}
catch (err) {
reject("There is a problem getting Axios array of promises " + err);
}
})
}
async function processAxiosPromises (PromiseArray) {
try {
var results = []
results = await axios.all(PromiseArray);
return results;
}
catch(err) {
throw("There was a problem resolving promises array (Axios) " + err);
}
}
getallID().then ( (urlList) => {
return getAxiosPromiseArray(urlList);
}).then( (AxiosPromises) => {
return processAxiosPromises(AxiosPromises);
}).then ((resultData) => {
console.log(resultData);
});
Error
There was a problem resolving promises array (Axios) Error: socket hang up
First, that pair of functions getAxiosPromiseArray() and processAxiosPromises() needs fixing.
Your new Promise() construction is unnecessary. You can simply return Promise.all(arrayofPromise) (or axios.all(...) if you must) and do away with the other function.
Renaming the remaining function to something meaningful, you would end up with eg :
let getData = (urlList) => {
return Promise.all(urlList.map(URL => axios.get(URL)))
.catch(error => {
error.message = "There is a problem getting Axios array of promises " + error.message; // augment the error message ...
throw error; // ... and re-throw the errror.
});
};
And call as follows :
getallID().then(getData)
.then(resultData => {
console.log(resultData);
}).catch(error => {
console.error(error);
});
That will put you on solid ground but, on its own, is unlikely to fix a concurrency problem (if that's what it is), for which the simplest approach is to use Bluebird's Promise.map with the concurrency option.
The caller code can remain the same, just change getData(), as follows:
let getData = (urlList) => {
let concurrency = 10; // play with this value to find a reliable concurrency limit
return Promise.map(urlList, URL => axios.get(URL), {'concurrency': concurrency})
.catch(error => {
error.message = "There is a problem getting Axios array of promises " + error.message;
throw error;
});
};
// where `Promise` is Bluebird.
const axios = require('axios');
const axiosThrottle = require('axios-throttle');
//pass axios object and value of the delay between requests in ms
axiosThrottle.init(axios,200)
const options = {
method: 'GET',
};
const urlList = [
'https://jsonplaceholder.typicode.com/todos/1',
'https://jsonplaceholder.typicode.com/todos/2',
'https://jsonplaceholder.typicode.com/todos/3',
'https://jsonplaceholder.typicode.com/todos/4',
'https://jsonplaceholder.typicode.com/todos/5',
'https://jsonplaceholder.typicode.com/todos/6',
'https://jsonplaceholder.typicode.com/todos/7',
'https://jsonplaceholder.typicode.com/todos/8',
'https://jsonplaceholder.typicode.com/todos/9',
'https://jsonplaceholder.typicode.com/todos/10'
];
const promises = [];
const responseInterceptor = response => {
console.log(response.data);
return response;
};
//add interceptor to work with each response seperately when it is resolved
axios.interceptors.response.use(responseInterceptor, error => {
return Promise.reject(error);
});
for (let index = 0; index < urlList.length; index++) {
options.url = urlList[index];
promises.push(axiosThrottle.getRequestPromise(options, index));
}
//run when all promises are resolved
axios.all(promises).then(responses => {
console.log(responses.length);
});
https://github.com/arekgotfryd/axios-throttle

Resources