Getting right fetch result from previous async fetch API functions - node.js

Hi I don't understand why getFlyByTime() function is giving me an ERR_INVALID_URL.
All the way upto getFlyByTime() I am getting the right results and coordinates.
Any advice would be appreciated,
Thank you
import fetch from "node-fetch";
let myIP = ''
let myLocation = ''
let flyByInformation = ''
const findIP = 'https://api.ipify.org/?format=json'
const geolocation = 'http://ipwho.is/'
const issFly = `https://iss-pass.herokuapp.com/json?`
const getMyIP = async function() {
let response = await fetch(findIP);
let data = await response.json()
return data.ip
}
const getMyGeoLocation = async function() {
let response = await fetch(geolocation + myIP);
let data = await response.json()
let resURL = `https://iss-pass.herokuapp.com/lat=${data.latitude}&lon=${data.longitude}`
return resURL;
}
const getFlyByTime = async function() {
let response = await fetch(myLocation);
console.log(response)
let data = await response.json()
return data;
}
getMyIP()
.then(data => {
myIP = data
}).catch(err => console.log('gmi error', err))
getMyGeoLocation()
.then(data => {
myLocation = data;
console.log(myLocation);
}).catch(err => console.log('gml error', err))
getFlyByTime()
.then(data => {
flyByInformation = JSON.parse(data);
console.log('flyby', flyByInformation);
}).catch(err => console.log('gflt error', err))

You are trying to use the myLocation and myIP values BEFORE they have been filled. Your functions return a promise before their work is done. Not until that promise has been fulfilled are the resolved values available for you to use.
As such, you must sequence your operations. It is generally easiest to do this with await. Here's an example shown below:
import fetch from "node-fetch";
const findIpURL = 'https://api.ipify.org/?format=json'
const geolocationURL = 'http://ipwho.is/'
const issFly = `https://iss-pass.herokuapp.com/json?`
async function fetchJSON(url) {
const response = await fetch(url);
if (response.ok) {
return response.json();
} else {
throw new Error(`Request failed, status ${response.status}`, { cause: response });
}
}
async function getMyIP() {
const data = await fetchJSON(findIpURL);
return data.ip;
}
function getMyGeoLocation(myIP) {
return fetchJSON(geolocationURL + myIP);
}
async function getFlyInfo(lat, long) {
let resURL = `https://iss-pass.herokuapp.com/lat=${lat}&lon=${long}`;
const flyInfo = await fetchJSON(resURL);
return flyInfo;
}
async function getFlyByTime() {
const myIP = await getMyIP();
console.log(myIP);
const myLocation = await getMyGeoLocation(myIP)
console.log(myLocation.latitude, myLocation.longitude);
return getFlyInfo(myLocation.latitude, myLocation.longitude);
}
getFlyByTime().then(flyInfo => {
console.log(flyInfo);
}).catch(err => {
console.log(err);
});
When I run this, the last getFlyInfo() request ends up returning a text/plain response that just says "Not Found" and the status is a 404. So, either the URL isn't being built properly in my version of the code or something is amiss in that last part.
But, hopefully you can see how you sequence asynchronous operations with await in this example and you can make that last part do what you want it to.

Related

Async function to scrape subreddits using Cheerio returns undefined

The script by itself works great (entering the url manually, writing a json file using the fs module, node script_name.js) but within a Express get request it returns undefined.
So I've built a simple frontend to let the user enter the subreddit name to be scraped.
And here's where the problem is:
Express controller
const run = require("../run");
requestPosts: async (req, res) => {
try {
const { subreddit } = req.body;
const response = await run(subreddit);
//console.log(response);
res.json(response);
} catch (error) {
console.error(error);
}
},
Cheerio functions
const axios = require("axios");
const { load } = require("cheerio");
let posts = [];
async function getImage(postLink) {
const { data } = await axios(postLink);
const $ = load(data);
return $("a.post-link").attr("href");
}
async function run(url) {
try {
console.log(url);
const { data } = await axios(url);
const $ = load(data);
$(".thing.linkflair.link").map(async (i, e) => {
const title = $(e)
.find(".entry.unvoted .top-matter .title .title")
.text();
const user = $(e)
.find(".entry.unvoted .top-matter .tagline .author")
.text();
const profileLink = `https://old.reddit.com/user/${user}`;
const postLink = `https://old.reddit.com/${$(e).find("a").attr("href")}`;
// const thumbail = $(e).find("a img").attr("src");
const image = await getImage(postLink);
posts.push({
id: i + 1,
title,
postLink,
image,
user: { user, profileLink },
});
});
const nextPage = $(".next-button a").attr("href");
if (nextPage) {
await run(nextPage);
} else {
return posts;
}
} catch (error) {
console.error(error);
}
}
module.exports = run;
I've tried working with Promise((resolve, reject) => {}).
I think it's returning undefined because maybe the code its not synchronized.
(idk if it makes sense, i've just started programming)
.map() is not promise-aware and does not wait for your promises to finish. So, $(".thing.linkflair.link").map() finishes long before any of the asynchronous functions inside its callback do. Thus you try to return posts BEFORE it has been populated.
Passing an async callback to .map() will return an array of promises. You can use Promise.all() on those promises to know when they are done and once you're doing that, you may as well just return each post object rather that using a higher level scoped/shared object, thus making the code more self contained.
I would suggest this code:
async function run(url) {
try {
console.log(url);
const { data } = await axios(url);
const $ = load(data);
const posts = await Promise.all($(".thing.linkflair.link").map(async (i, e) => {
const title = $(e)
.find(".entry.unvoted .top-matter .title .title")
.text();
const user = $(e)
.find(".entry.unvoted .top-matter .tagline .author")
.text();
const profileLink = `https://old.reddit.com/user/${user}`;
const postLink = `https://old.reddit.com/${$(e).find("a").attr("href")}`;
// const thumbail = $(e).find("a img").attr("src");
const image = await getImage(postLink);
// return a post object
return {
id: i + 1,
title,
postLink,
image,
user: { user, profileLink },
};
}));
const nextPage = $(".next-button a").attr("href");
if (nextPage) {
const newPosts = await run(nextPage);
// add these posts to the ones we already have
posts.push(...newPosts);
}
return posts;
} catch (error) {
console.error(error);
}
}

Unable to map inside an async function

In the below code, I am fetching data from an external api. After parsing the data as json, I wanted to map through it and get a modified version.
For some reason, the console.log(jsonData) inside the map function is not getting executed. Please check the code below for clarity
const getRandomOutfit = async (req, res) => {
const { gender, countryCode } = req.params;
if (req.params.gender === "FEMALE" || req.params.gender === "MALE") {
try {
const response = await fetch(URL);
const jsonData = await response.json();
const outputData = jsonData.map((productItem) => {
console.log(productItem); // doesn't get printed
// some operation
return productItem;
});
await res.json(jsonData);
} catch (error) {
res.status(500).send("Error getting data");
}
} else {
res.status(500).send("Invalid category");
}
};
I'm confused about what I am missing here and making an error.
I rewrote the code to make it clearer to understand. In general, it is best to take the fail first approach. Notice, how the first thing I do is return upon failure.
As to why you code is not printing anything out, try printing jsonData. It might be that this is an empty array.
const getRandomOutfit = async (req, res) => {
const { gender, countryCode } = req.params;
if (gender !== "FEMALE" && gender !== "MALE")
return res.status(500).send("Invalid category");
try {
const response = await fetch(URL);
const jsonData = await response.json();
console.log(jsonData); // what does this return?
const outputData = jsonData.map((productItem) => {
console.log(productItem); // doesn't get printed
// some operation
return productItem;
});
await res.json(jsonData);
} catch (error) {
res.status(500).send("Error getting data");
};

Problem to use a Map in Firebase Functions

I am trying to get the length of a Map and I keep getting "undefined". Could please someone tell me what am I doing wrong?
This is the part of the code that gives me problems.
const GYMdetail: { [key: string]: number} = {};
GYMdetail[`${doc.data().name} (${doc.data().personalID})`] = 650;
const subtotal = 650 * GYMdetail.size;
This is the complete function code
export const addGymMonthlyExpense =
functions.https.onRequest((request, response) => {
const query1 = admin.firestore().collection("users");
const query = query1.where("subscriptions.gym.active", "==", true);
query.get()
.then(async (allUsers) => {
allUsers.docs.forEach(async (doc) => {
if (doc != undefined) {
const houseForGym = doc.data().subscriptions.gym.house;
await admin.firestore()
.doc(`houses/${houseForGym}/expenses/2022-04`)
.get().then((snapshot) => {
if (snapshot.data() == undefined) {
console.log(`${houseForGym}-${doc.data().name}: CREAR!!`);
} else if (snapshot.data()!.issued == false) {
let detail: { [key: string]: any} = {};
const GYMdetail: { [key: string]: number} = {};
detail = snapshot.data()!.detail;
GYMdetail[
`${doc.data().name} (${doc.data().personalID})`
] = 650;
const subtotal = 650 * GYMdetail.size;
detail["GYM"] = {"total": subtotal, "detail": GYMdetail};
snapshot.ref.set({"detail": detail}, {merge: true});
}
return null;
})
.catch((error) => {
console.log(
`${houseForGym} - ${doc.data().name}: ${error}`);
response.status(500).send(error);
return null;
});
}
});
response.send("i");
})
.catch((error) => {
console.log(error);
response.status(500).send(error);
});
});
Since you are executing an asynchronous call to the database in your code, you need to return a promise from the top-level code; otherwise Cloud Functions may kill the container when the final } executes and by that time the database load won't be done yet.
So:
export const addGymMonthlyExpense =
functions.https.onRequest((request, response) => {
const query1 = admin.firestore().collection("users");
const query = query1.where("subscriptions.gym.active", "==", true);
return query.get()
...
Next you'll need to ensure that all the nested get() calls also get a chance to finish before the Functions container gets terminated. For that I recommend not using await for each nested get call, but a single Promise.all for all of them:
query.get()
.then(async (allUsers) => {
const promises = [];
allUsers.docs.forEach((doc) => {
const houseForGym = doc.data().subscriptions.gym.house;
promises.push(admin.firestore()
.doc(`houses/${houseForGym}/expenses/2022-04`)
.get().then((snapshot) => {
...
});
});
response.send("i");
return Promise.all(promises);
})
.catch((error) => {
console.log(error);
response.status(500).send(error);
});

Return value after all the promises are resolved

I am working on a nodejs code that fetches data from a site, parses it, finds particular data and fetches something else for the data that was previously fetched. But the final return statement is returning without the value fetched from the second API call.
I tried to implement async await, but I am not sure where do I have to put them exactly.
const getMainData = async val => {
let result = [];
//get xml data from the API
const xmlData = await getSiteContent(`value`); //axios call
parseString(xmlData, (err, json) => { //convert xml to json
const { entry } = json.feed; // array of results.
result = entry.map(report => {
const secondInfo = getSomeMoreData(report.something); //axios call
const data = {
id: report.id,
date: report.date,
title: report.title
};
data.info = secondInfo;
return data;
});
});
return { result };
};
I was expecting the function to return the array result that has id, date, title and info. But I am getting info as null since it is going to another function that does one more API call.
Try wrapping parseString in a promise so you can await the result, then make the entry.map callback an async function so that you can use the await keyword to wait for the result of the axios fetch.
async function xml2json(xml) {
return new Promise((resolve, reject) => {
parseString(xml, function (err, json) {
if (err)
reject(err);
else
resolve(json);
});
});
}
const getMainData = async val => {
//get xml data from the API
const xmlData = await getSiteContent(`value`); //axios call
const json = await xml2json(xmlData);
const { entry } = json.feed; // array of results
const result = await Promise.all(
entry.map(async report => {
const secondInfo = await getSomeMoreData(report.something); // axios call
const data = {
id: report.id,
date: report.date,
title: report.title,
};
data.info = secondInfo;
return data;
})
)
return { result };
}
Let me know if that works. If not, I can try to help you out further.
The problem with your code is you have mixed promises concept(async/await is a syntactic sugar - so same thing) along with callback concept.
And the return statement is outside callback() of parseString() and the callback would be executed maybe after returning results only because parseString() is an asynchronous function.
So in the following solution I have wrapped parseString() in a promise so that it can be awaited.
const parseStringPromisified = async xmlData => {
return new Promise((resolve, reject) => {
parseString(xmlData, (err, json) => {
if (err) {
reject(err);
}
resolve(json);
});
});
};
const getMainData = async val => {
//get xml data from the API
const xmlData = await getSiteContent(`value`); //axios call
const json = await parseStringPromisified(xmlData);
const { entry } = json.feed; // array of results.
const result = entry.map(async report => {
const secondInfo = await getSomeMoreData(report.something); //axios call
return {
id: report.id,
date: report.date,
title: report.title,
info: secondInfo
};
});
return Promises.all(result);
};

How to wait for a url callback before send HTTP response in koa?

I have a koa router I need to call a api where will async return result. This means I cannot get my result immediately, the api will call my callback url when it's ok. But now I have to use it like a sync api which means I have to wait until the callback url is called.
My router like this:
router.post("/voice", async (ctx, next) => {
// call a API here
const params = {
data: "xxx",
callback_url: "http//myhost/ret_callback",
};
const req = new Request("http://xxx/api", {
method: "POST",
body: JSON.stringify(params),
});
const resp = await fetch(req);
const data = await resp.json();
// data here is not the result I want, this api just return a task id, this api will call my url back
const taskid = data.taskid;
// now I want to wait here until I got "ret_callback"
// .... wait .... wait
// "ret_callback" is called now
// get the answer in "ret_callback"
ctx.body = {
result: "ret_callback result here",
}
})
my callback url like this:
router.post("/ret_callback", async (ctx, next) => {
const params = ctx.request.body;
// taskid will tell me this answer to which question
const taskid = params.taskid;
// this is exactly what I want
const result = params.text;
ctx.body = {
code: 0,
message: "success",
};
})
So how can I make this aync api act like a sync api?
Just pass a resolve() to another function. For example, you can do it this way:
// use a map to save a lot of resolve()
const taskMap = new Map();
router.post("/voice", async (ctx, next) => {
// call a API here
const params = {
data: "xxx",
callback_url: "http//myhost/ret_callback",
};
const req = new Request("http://xxx/api", {
method: "POST",
body: JSON.stringify(params),
});
const resp = await fetch(req);
const data = await resp.json();
const result = await waitForCallback(data.taskid);
ctx.body = {
result,
} })
const waitForCallback = (taskId) => {
return new Promise((resolve, reject) => {
const task = {};
task.id = taskId;
task.onComplete = (data) => {
resolve(data);
};
task.onError = () => {
reject();
};
taskMap.set(task.id, task);
});
};
router.post("/ret_callback", async (ctx, next) => {
const params = ctx.request.body;
// taskid will tell me this answer to which question
const taskid = params.taskid;
// this is exactly what I want
const result = params.text;
// here you continue the waiting response
taskMap.get(taskid).onComplete(result);
// not forget to clean rubbish
taskMap.delete(taskid);
ctx.body = {
code: 0,
message: "success",
}; })
I didn't test it but I think it will work.
function getMovieTitles(substr) {
let movies = [];
let fdata = (page, search, totalPage) => {
let mpath = {
host: "jsonmock.hackerrank.com",
path: "/api/movies/search/?Title=" + search + "&page=" + page,
};
let raw = '';
https.get(mpath, (res) => {
res.on("data", (chunk) => {
raw += chunk;
});
res.on("end", () => {
tdata = JSON.parse(raw);
t = tdata;
totalPage(t);
});
});
}
fdata(1, substr, (t) => {
i = 1;
mdata = [];
for (i = 1; i <= parseInt(t.total_pages); i++) {
fdata(i, substr, (t) => {
t.data.forEach((v, index, arrs) => {
movies.push(v.Title);
if (index === arrs.length - 1) {
movies.sort();
if (parseInt(t.page) === parseInt(t.total_pages)) {
movies.forEach(v => {
console.log(v)
})
}
}
});
});
}
});
}
getMovieTitles("tom")
Okay so first of all, this should not be a "goal" for you. NodeJS works better as ASync.
However, let us assume that you still want it for some reason, so take a look at sync-request package on npm (there is a huge note on there that you should not this in production.
But, I hope you mean on how to make this API simpler (as in one call kinda thingy). You still need .next or await but it will be be one call anyway.
If that is the case, please comment on this answer I can write you a possible method I use myself.
How about this ?
router.post("/voice", async (ctx, next) => {
const params = {
data: "xxx",
callback_url: "http//myhost/ret_callback",
};
const req = new Request("http://xxx/api", {
method: "POST",
body: JSON.stringify(params),
});
const resp = await fetch(req);
const data = await resp.json();
// data here is not the result I want, this api just return a task id, this api will call my url back
const taskid = data.taskid;
let response = null;
try{
response = await new Promise((resolve,reject)=>{
//call your ret_callback and when it finish call resolve(with response) and if it fails, just reject(with error);
});
}catch(err){
//errors
}
// get the answer in "ret_callback"
ctx.body = {
result: "ret_callback result here",
}
});

Resources