I created a function to find data by ID from api if there is one and create new one if there isn't. But server crashes if the first axios.get() fails with 400 code when data corresponding to the provided ID doesn't exist. Originally had try and catch block around the axios get and post and thought they might be causing the crash but removing them yields same result.
async function findOrCreateDocument(id) {
if (id == null) return;
let response = await axios.get(api + `/${id}`);
if (response.status >= 200 && response.status < 300) return response.data;
response = await axios.post(api, { id: id, data: defaultValue });
document = response.data;
return document;
}
This function is called inside of a socket.io server.
io.on("connection", (socket) => {
socket.on("join-document", async (documentId) => {
const document = await findOrCreateDocument(documentId);
socket.join(documentId);
socket.emit("load-document", document.data);
socket.on("send-text-change", (delta) => socket.broadcast.to(documentId).emit("recieve-text-change", delta));
...
}
}
Related
I have a script using axios that hits an API with a limit of 5 requests per second. At present my request array length is 72 and will grow over time. I receive an abundance of 429 errors. The responses per endpoint change with each run of the script; ex: url1 on iteration1 returns 429, then url1 on iteration2 returns 200, url1 on iteration3 returns 200, url1 on iteration4 returns 429.
Admittedly my understanding of async/await and promises are spotty at best.
What I understand:
I can have multiple axios.get running because of async. The variable I set in my main that uses the async function can include the await to ensure all requests have processed before continuing the script.
Promise.all can run multiple axios.gets but, if a single request fails the chain breaks and no more requests will run.
Because the API will only accept 5 requests per second I have to chunk my axios.get requests to 5 endpoints, wait for those to finish processing before sending the next chunk of 5.
setTimeout will assign a time limit to a single request, once the time is up the request is done and will not be sent again no matter the return being other than 200.
setInterval will assign a time limit but it will send the request again after time's up and keep requesting until it receives a 200.
async function main() {
var endpoints = makeEndpoints(boards, whiteList); //returns an array of string API endpoints ['www.url1.com', 'www.url2.com', ...]
var events = await getData(endpoints);
...
}
The getData() has seen many iterations in attempt to correct the 429's. Here are a few:
// will return the 200's sometimes and not others, I believe it's the timeout but that won't attempt the hit a failed url (as I understand it)
async function getData(endpoints) {
let events = [];
for (x = 0; x < endpoints.length; x++) {
try {
let response = await axios.get(endpoints[x], {timeout: 2000});
if ( response.status == 200 &&
response.data.hasOwnProperty('_embedded') &&
response.data._embedded.hasOwnProperty('events')
) {
let eventsArr = response.data._embedded.events;
eventsArr.forEach(event => {
events.push(event)
});
}
} catch (error) {
console.log(error);
}
}
return events;
}
// returns a great many 429 errors via the setInterval, as I understand this function sets a delay of N seconds before attempting the next call
async function getData(endpoints) {
let data = [];
let promises = [];
endpoints.forEach((url) => {
promises.push(
axios.get(url)
)
})
setInterval(function() {
for (i = 0; i < promises.length; i += 5) {
let requestArr = promises.slice(i, i + 5);
axios.all(requestArr)
.then(axios.spread((...res) => {
console.log(res);
}))
.catch(err => {
console.log(err);
})
}
}, 2000)
}
// Here I hoped Promise.all would allow each request to do its thing and return the data, but after further reading I found that if a single request fails the rest will fail in the Promise.all
async getData(endpoints) {
try {
const res = await Promise.all(endpoints.map(url => axios.get(url))).catch(err => {});
} catch {
throw Error("Promise failed");
}
return res;
}
// Returns so many 429 and only 3/4 data I know to expect
async function getData(endpoints) {
const someFunction = () => {
return new Promise(resolve => {
setTimeout(() => resolve('222'), 100)
})
}
const requestArr = endpoints.map(async data => {
let waitForThisData = await someFunction(data);
return axios.get(data)
.then(response => { console.log(response.data)})
.catch(error => console.log(error.toString()))
});
Promise.all(requestArr).then(() => {
console.log('resolved promise.all')
})
}
// Seems to get close to solving but once an error is it that Promise.all stops processing endpoint
async function getData(endpoints) {
(async () => {
try {
const allResponses = await Promise.all(
endpoints.map(url => axios.get(url).then(res => console.log(res.data)))
);
console.log(allResponses[0]);
} catch(e) {
console.log(e);
// handle errors
}
})();
}
It seems like I have so many relevant pieces but I cannot connect them in an efficient and working model. Perhaps axios has something completely unknown to me? I've also tried using blurbird concurrent to limit the request to 5 per attempt but that still returned the 429 from axios.
I've been starring at this for days and with so much new information swirling in my head I'm at a loss as to how to send 5 requests per second, await the response, then send another set of 5 requests to the API.
Guidance/links/ways to improve upon the question would be much appreciated.
I'm trying to download a bunch of files. Let's say 1.jpg, 2.jpg, 3.jpg and so on. If 1.jpg exist, then I want to try and download 2.jpg. And if that exist I will try the next, and so on.
But the current "getFile" returns a promise, so I can't loop through it. I thought I had solved it by adding await in front of the http.get method. But it looks like it doesn't wait for the callback method to finish. Is there a more elegant way to solve this than to wrap the whole thing in a new async method?
// this returns a promise
var result = getFile(url, fileToDownload);
const getFile = async (url, saveName) => {
try {
const file = fs.createWriteStream(saveName);
const request = await http.get(url, function(response) {
const { statusCode } = response;
if (statusCode === 200) {
response.pipe(file);
return true;
}
else
return false;
});
} catch (e) {
console.log(e);
return false;
}
}
I don't think your getFile method is returning promise and also there is no point of awaiting a callback. You should split functionality in to two parts
- get file - which gets the file
- saving file which saves the file if get file returns something.
try the code like this
const getFile = url => {
return new Promise((resolve, reject) => {
http.get(url, response => {
const {statusCode} = response;
if (statusCode === 200) {
resolve(response);
}
reject(null);
});
});
};
async function save(saveName) {
const result = await getFile(url);
if (result) {
const file = fs.createWriteStream(saveName);
response.pipe(file);
}
}
What you are trying to do is getting / requesting images in some sync fashion.
Possible solutions :
You know the exact number of images you want to get, then go ahead with "request" or "http" module and use promoise chain.
You do not how the exact number of images, but will stop at image no. N-1 if N not found. then go ahed with sync-request module.
your getFile does return a promise, but only because it has async keyword before it, and it's not a kind of promise you want. http.get uses old callback style handling, luckily it's easy to convert it to Promise to suit your needs
const tryToGetFile = (url, saveName) => {
return new Promise((resolve) => {
http.get(url, response => {
if (response.statusCode === 200) {
const stream = fs.createWriteStream(saveName)
response.pipe(stream)
resolve(true);
} else {
// usually it is better to reject promise and propagate errors further
// but the function is called tryToGetFile as it expects that some file will not be available
// and this is not an error. Simply resolve to false
resolve(false);
}
})
})
}
const fileUrls = [
'somesite.file1.jpg',
'somesite.file2.jpg',
'somesite.file3.jpg',
'somesite.file4.jpg',
]
const downloadInSequence = async () => {
// using for..of instead of forEach to be able to pause
// downloadInSequence function execution while getting file
// can also use classic for
for (const fileUrl of fileUrls) {
const success = await tryToGetFile('http://' + fileUrl, fileUrl)
if (!success) {
// file with this name wasn't found
return;
}
}
}
This is a basic setup to show how to wrap http.get in a Promise and run it in sequence. Add error handling wherever you want. Also it's worth noting that it will proceed to the next file as soon as it has received a 200 status code and started downloading it rather than waiting for a full download before proceeding
I am having trouble with my lambda function. This is the current setup:
Lambda makes post request to API. API fetches data from postgres database and returns this data. When I use Postman or a local version of my lambda function, this works. When I use the actual lambda function, the API returns data with null.
Below are some code snippets:
lambda:
const axios = require('axios')
exports.handler = async (event) => {
let rows = await axios.post('http:/server/getData', {
params: {
var: event.var
}
})
if(rows.data.statusCode == 204){
//no data available
}
else{
data = rows.data.info
}
};
API Router Component
var router = express.Router()
const Pool = require('pg').Pool
const mgmtdb = new Pool({ ... })
router.post('/getData', function(req, res){
database.query("SELECT info FROM table WHERE var=$1", [req.body.var], (error, results) => {
const rows = results.rows
if (error) {
throw error
}
let status = rows.length == 0 ? 204 : 200
var responseJSON ={};
responseJSON.statusCode = status;
responseJSON.info= rows[0] ? rows[0].info : null;
res.json(responseJSON);
})
})
module.exports = router
When I call the API from Postman I get statusCode: 200 (data available).
If I call the API with the exact same data from lambda I get statusCode: 204 (no data available).
I believe that this is some async timing problem. I don't know how the responses from the API can differ ..
Is it possible that the API streams the response back to the originator for some time not just an impulse? And starts by streaming "no data available" and then, after a few milliseconds "oh, i found some data, here it is"? And that Postman waits for the stream to finish and the lambda function doesn't?
Thanks in advance!
Looks like your query is returning null because the req.body.var doesn't exist. Can you try changing your lambda function to this:
const axios = require('axios')
exports.handler = async (event) => {
let rows = await axios.post('http:/server/getData', {
var: event.var
})
if(rows.data.statusCode == 204){
//no data available
}
else{
data = rows.data.info
}
};
This basically removes the extra level params and makes the req.body.var work.
Sometimes lambda have issue with async await so i suggest you to write code like this
exports.handler = async (event, callback) => {
axios.post('http:/server/getData', {
var: event.var
})
.then((response) => {
callback(null, {
statusCode: 200,
body: JSON.stringify(response)
})
})
};
I'm working on a project that requires me to:
GET IDs from API1, push the IDs into an array, then map over those IDs, using them for a second GET request, where IDs are used as params for API2 GET request, populates an array with IDs or N for "Not existing" -- this array is then called in:
A POST request. This post maps over the returned array from the GET request. IF the item is not "N", it POSTS to API1 with checked: true. IF the item is "N", it emails us telling us API2 is missing this project.
I want this system to automatically do a GET and POST every 2 hours, so I'm using setInterval (not sure this is the best idea). EDIT: Cron job would be a better solution.
I'm working with NodeJS, Express, Request-Promise, Async / Await.
Here is some of my pseudo code so far:
// Dependencies
const express = require('express');
const axios = require('axios');
const mailgun = require('mailgun-js')({ apiKey, domain });
// Static
const app = express();
app.get('/', (req, res, next) => {
// Replace setInterval with Cron job in deployment
// Get All Ids
const orders = await getGCloud();
// Check if IDs exist in other API
const validations = await getProjectManagementSystem(orders);
// If they exist, POST update to check, else, mailer
validations.map(id => {
if (id !== 'n') {
postGCloud(id);
} else {
mailer(id);
}
});
}
// Method gets all IDs
const getGCloud = async () => {
try {
let orders = [];
const response = await axios.get('gCloudURL');
for (let key in response) {
orders.push(response.key);
}
return orders;
} catch (error) {
console.log('Error: ', error);
}
}
// Method does a GET requst for each ID
const getProjectManagementSystem = async orders => {
try {
let idArr = [];
orders.map(id => {
let response = await axios.get(`projectManagementSystemURL/${id}`);
response === '404' ? idArr.push('n') : idArr.push(response)
})
return idArr;
} catch (error) {
console.log('Error: ', error);
}
}
const postGCloud = id => {
axios.post('/gcloudURL', {
id,
checked: true
})
.then(res => console.log(res))
.catch(err => console.log(err))
}
const mailer = id => {
const data = {
from: 'TESTER <test#test.com>',
to: 'customerSuppoer#test.com',
subject: `Missing Order: ${id}`,
text: `Our Project Management System is missing ${id}. Please contact client.`
}
mailgun.messages().send(data, (err, body) => {
if (err) {
console.log('Error: ', err)
} else {
console.log('Body: ', body);
}
});
}
app.listen(6000, () => console.log('LISTENING ON 6000'));
The TL;DR: Need to do a GET request to API 1, then another GET request to API 2 following it (using IDs from API 1 as params), then send data from second GET to a POST request that then either updates API 1's data or emails Customer support. This is an automatic system that runs every two hours.
Main Questions:
1. Is it okay to have a setInterval in a get req?
2. Can I have a GET request automatically call a POST request?
3. If so, how can I pass GET request data onto a POST request?
To make it work for both of your calls one post and one get you have to do an Ajax call to get post processed information in another method.
I hope this works.
My application uses an internal webservice for fetching data, i have a job which creates approx 500 requests which getsfired async to complete the fetch operation.
I make use of Axios, by creating an array of axios promises and then resolving them using using Axios.all();
It works fine until some 200 requests but post that i get socket hung up, however on the server side i see the requests are being processed.
How to configure axios to set custom time out, or is it a better idea to splice my promises array and then run them as multiple batches ?
Source code
let getAxiosPromiseArray = (urlList) => {
var axiosArrayofPromise = [];
return new Promise ( (resolve, reject) => {
try {
urlList.forEach ( (URL) => {
axiosArrayofPromise.push(axios.get(URL));
});
resolve(axiosArrayofPromise);
}
catch (err) {
reject("There is a problem getting Axios array of promises " + err);
}
})
}
async function processAxiosPromises (PromiseArray) {
try {
var results = []
results = await axios.all(PromiseArray);
return results;
}
catch(err) {
throw("There was a problem resolving promises array (Axios) " + err);
}
}
getallID().then ( (urlList) => {
return getAxiosPromiseArray(urlList);
}).then( (AxiosPromises) => {
return processAxiosPromises(AxiosPromises);
}).then ((resultData) => {
console.log(resultData);
});
Error
There was a problem resolving promises array (Axios) Error: socket hang up
First, that pair of functions getAxiosPromiseArray() and processAxiosPromises() needs fixing.
Your new Promise() construction is unnecessary. You can simply return Promise.all(arrayofPromise) (or axios.all(...) if you must) and do away with the other function.
Renaming the remaining function to something meaningful, you would end up with eg :
let getData = (urlList) => {
return Promise.all(urlList.map(URL => axios.get(URL)))
.catch(error => {
error.message = "There is a problem getting Axios array of promises " + error.message; // augment the error message ...
throw error; // ... and re-throw the errror.
});
};
And call as follows :
getallID().then(getData)
.then(resultData => {
console.log(resultData);
}).catch(error => {
console.error(error);
});
That will put you on solid ground but, on its own, is unlikely to fix a concurrency problem (if that's what it is), for which the simplest approach is to use Bluebird's Promise.map with the concurrency option.
The caller code can remain the same, just change getData(), as follows:
let getData = (urlList) => {
let concurrency = 10; // play with this value to find a reliable concurrency limit
return Promise.map(urlList, URL => axios.get(URL), {'concurrency': concurrency})
.catch(error => {
error.message = "There is a problem getting Axios array of promises " + error.message;
throw error;
});
};
// where `Promise` is Bluebird.
const axios = require('axios');
const axiosThrottle = require('axios-throttle');
//pass axios object and value of the delay between requests in ms
axiosThrottle.init(axios,200)
const options = {
method: 'GET',
};
const urlList = [
'https://jsonplaceholder.typicode.com/todos/1',
'https://jsonplaceholder.typicode.com/todos/2',
'https://jsonplaceholder.typicode.com/todos/3',
'https://jsonplaceholder.typicode.com/todos/4',
'https://jsonplaceholder.typicode.com/todos/5',
'https://jsonplaceholder.typicode.com/todos/6',
'https://jsonplaceholder.typicode.com/todos/7',
'https://jsonplaceholder.typicode.com/todos/8',
'https://jsonplaceholder.typicode.com/todos/9',
'https://jsonplaceholder.typicode.com/todos/10'
];
const promises = [];
const responseInterceptor = response => {
console.log(response.data);
return response;
};
//add interceptor to work with each response seperately when it is resolved
axios.interceptors.response.use(responseInterceptor, error => {
return Promise.reject(error);
});
for (let index = 0; index < urlList.length; index++) {
options.url = urlList[index];
promises.push(axiosThrottle.getRequestPromise(options, index));
}
//run when all promises are resolved
axios.all(promises).then(responses => {
console.log(responses.length);
});
https://github.com/arekgotfryd/axios-throttle