Store data returned from database into variable in nodejs express - node.js

I want to fetch subject Contents based on subject code and then inside each subject content, I want to fetch its sub contents as well and then store the main contents and sub contents in one array as object and return the data to react.
Please help me with this.
Node express API code
app.post('/api/teacher/courses/maintopics', (req, res) =>
{
let SubCode = req.body.data;
let teacher = new Teacher();
teacher.getCoursesMainContent(SubCode).then(result =>
{
let Contiants = [];
result.forEach(element =>
{
SubContent = [];
element.forEach(e =>
{
let contentCode = e.ContentCode;
teacher.getCoursesSubContent(contentCode).then()
.then(res => {
SubContent.push(res)
// here I want to store the sub content inside SubContent array
});
})
});
res.json(Contiants);
});
});

the problem is that when res.json(Contiants); is executed, the promises (getCoursesSubContent) are not resolved yet.
you need to use await like jax-p said.
also note that you cannot use forEach with await/promises (well you can, but it wont work as you wish it does : Using async/await with a forEach loop)
app.post('/api/teacher/courses/maintopics', async (req, res, next) =>
{
try {
let SubCode = req.body.data;
let teacher = new Teacher();
const results = await teacher.getCoursesMainContent(SubCode);
let Contiants = [];
for (let element of results) {
SubContent = [];
for (let e of element) {
let contentCode = e.ContentCode;
let res = await teacher.getCoursesSubContent(contentCode);
SubContent.push(res)
}
}
res.json(Contiants);
} catch(err) {
next(err);
}
});

Related

How to await http requests in express

I'm trying to get a server to make a series of API calls according to a list of parameters obtained from a form. However, with the async http get method or the npm got library I cannot get my code to await for the responses of the API calls before it tries to render the response. My code looks like this:
router.post('/getPolicies', async function(req, res, next){
let issnList = req.body.issn_list
issnList = issnList.split(/\r?\n/)
await getPolicies(issnList);
res.render("policy", {data:policies})
});
async function getPolicies(issns){
for (let i = 0; i<issns.length; i++){
url = "some_url"+ issns[i]
const {data} = await got.get(url).json();
constructDataSet(issns[i], data)
}
}
function constructDataSet (issn, data){
//...
//get some information out of a json and construct a new json with data in needed format
}
The error I get is "Cannot read properties of undefined" because it's trying to read properties of the "data" json when it hasn't gotten it yet.
I think its because you are traversing the list synchronously when it should be asynchronous. Below are two diff implementation, the for await loop believe was made available in es6 and the second would be an alternative implementation.
router.post('/getPolicies', async function(req, res, next){
let issnList = req.body.issn_list
issnList = issnList.split(/\r?\n/)
await getPolicies(issnList);
res.render("policy", {data:policies})
});
// Prefer
async function getPolicies(issns){
for await (const issns of issns){
let url = "some_url" + issn;
const {data} = await got.get(url).json();
constructDataSet(issns, data)
}
}
async function getPoliciesV2(issns) {
await forEach(issns, (issn) => {
let url = "some_url" + issn;
const {data} = await got.get(url).json();
constructDataSet(issns, data);
});
}
async function forEach(data, callback) {
for(let i = 0; i < data.length; i++) {
callback(data[i]);
}
}
function constructDataSet (issn, data){
//...
//get some information out of a json and construct a new json with data in needed format
}

Comparing two arrays from async functions?

I have read a lot of posts on how to solve this problem, but I cannot understand it.
I have a database (psql) and a csv. I have a two functions. One to read a list of domains from psql. And another to read a different list of domains from the csv.
Both functions are async operations that live in separate modules.
Goal: to bring the results of both reader functions (which are arrays)into the same file and compare the files for duplicates.
Currently, I have made progress using Promise.all. However, I cannot seem to isolate the two separate arrays so I can use them.
Solution Function (not working):
This is where I am trying to read in both lists into two separate arrays.
The CSVList variable has a console.log that logs the array when the CSVList.filter is not present. Which leads me to believe that the array is actually there? Maybe?
const allData = async function () {
let [result1, result2] = await Promise.all([readCSV, DBList]);
const DBLists = result2(async (domainlist) => {
return domainlist;
});
const CSVList = result1(async (csv) => {
const csvArr = await csv.map((x) => {
return x[0];
});
console.log(csvArr);
return csvArr;
});
const main = await CSVList.filter((val) => !DBLists.includes(vals)); // this doesn't work. it says that filter is not a function. I understand why filter is not a function. What I do not understand is why the array is not being returned?
};
allData();
psql reader:
const { pool } = require("./pgConnect");
//
const DBList = async (callback) => {
await pool
.query(
`
SELECT website
FROM domains
limit 5
`
)
.then(async (data) => {
const domainList = await data.rows.map((x) => {
return x.website;
});
callback(domainList);
});
};
csv reader:
const { parseFile } = require("#fast-csv/parse");
const path = require("path");
const fs = require("fs");
const domainPath = path.join(__dirname, "domains.csv");
//reads initial domain list and pushes the domains to an array
//on end, calls a callback function with the domain data
const readCSV = async (callback) => {
let domainList = [];
let csvStream = parseFile(domainPath, { headers: false })
.on("data", (data) => {
//push csv data to domainList array
domainList.push(data);
// console.log(data);
})
.on("end", () => {
callback(domainList);
});
};
I took jFriend00 Advice and I updated my code a bit.
The biggest issue was the ReadCSV function. Fast-csv doesn't seem to be asynchronous. I wrapped it in a new promise manually. And then resolved that promise passing the domain list as an argument to resolve.
updated CSV Reader:
const readCSV2 = new Promise((resolve, reject) => {
let domainList = [];
let csvStream = parseFile(domainPath, { headers: false })
.on("data", (data) => {
//push csv data to domainList array
domainList.push(data[0]);
// console.log(data);
})
.on("end", () => {
resolve(domainList);
});
});
Updated Solution for comparing the two lists
const allData = async function () {
// get the values from the DB and CSV in one place
let [result1, result2] = await Promise.all([readCSV2, DBList]);
const CSVDomains = await result1;
const DBDomains = await result2();
//final list compares the two lists and returns the list of non duplicated domains.
const finalList = await CSVDomains.filter(
(val) => !DBDomains.includes(val)
);
console.log("The new list is: " + finalList);
};
Quick aside: I could have accomplished the same result by using psql ON CONFLICT DO NOTHING. This would have ignored duplicates when updating to the database because I have a UNIQUE constraint on the domain column.

How can i use list with nested foreach?

var app = express();
var cloud = firebase.firestore();
app.get('/getMyDatas', function (req, res) {
let tList = [];
let tJson = {};
cloud.collection('contents/').orderBy('date').get().then((contents) => {
contents.docs.forEach(cont=> {
cloud.collection('userprofile/').where('userId', '==', cont.data().userId).get().then((users) => {
users.docs.forEach(user => {
tJson = {description:cont.data().description, name:user.data().name};
tList.push(tJson);
tJson = {};
console.log("LIST IS FILLED SUCCESFULLY : " + JSON.stringify(tList));
});
});
});
console.log(" ??HERE THE LIST IS EMPTY : " + JSON.stringify(tList));
res.json(tList);
});
});
This code can create the list i want. But i can't use it on the line
that says "res.json(tList)".
I can use on the line that says "console.log('My List : ' +
JSON.stringify(tList));" (it shows my list correctly.)
res.json(tList) return "[]" empty list. How can i use this on this
line?
This code can create the list i want. But i can't use it on the line that says "res.json(tList)".
You aren't properly waiting for your asynchronous operations to be done before trying to use the result you're building in the asynchronous operations. So, you're trying to use tList before any of the items have been added to it (a timing problem).
You also don't have any proper error handling for any of your promises that might reject.
This is a much easier problem if you switch to a for loop where you can then use async/await to sequence your asynchronous operations:
app.get('/getMyDatas', async function (req, res) {
try {
let contents = await cloud.collection('contents/').orderBy('date').get();
let tList = [];
for (let cont of contents.docs) {
let users = await cloud.collection('userprofile/').where('userId', '==', cont.data().userId).get();
for (let user of users) {
tList.push({description:cont.data().description, name:user.data().name});
}
}
res.json(tList);
} catch(e) {
console.log(e);
res.sendStatus(500);
}
});

How to return an object after multiple requests are done

If I didn't need to worry about asynchronous code, this is what it'll look like:
app.get('/user/:username/events', async (req, res) => {
// Grab all events by that user
let events = await axios.get(`${baseApiUrl}/users/${req.params.username}/events`, options).then(d => d.data)
// Loop through events
events.forEach(event => {
// If event type is A_Event
if (event.type === 'A_Event') {
// Loop through objArr array
event.payload.objArr.forEach(async obj => {
// Grab additional data through endpoint
// that lives in obj
let data = await axios.get(obj.url)
// Append data back onto obj
obj.objData = data
})
}
})
// return events obj with objData appended to each obj within
// objArr
res.json({ events })
})
But this doesnt work because it returns events before its done grabbing all the data.
I tried doing something with Promise.all but couldnt get what I needed to get.
events.map(async event => {
if (event.type === 'A_Event') {
getData(event.payload.objArr)
.then(objData => {
// This returns an array of objData that
// then needs to be added back to the specific obj
// it came from
event.payload.objArr.forEach((obj, i) => {
obj.objData = objData[i]
})
})
}
res.json({ events })
})
const getData = async objArr => {
const requests = objArr.map(async obj => {
const data = await axios.get(obj.url, options)
return {
stats: data.data.stats,
created_at: data.data.created_at
}
})
return Promise.all(requests)
}
Would really appreciate the help with this. Been stuck on this problem for a good bit.
EDIT
Tried switching to a for of loop according to this post: Using async/await with a forEach loop... No luck but I think its a step in the right direction.
events.forEach(async event => {
if (event.type === 'A_Event') {
for (let obj of event.payload.objArr) {
let objData = await axios.get(obj.url, options)
obj.objData = objData
console.log('setting value')
}
}
})
res.json({ events })
console.log('returned value')
I get returned value before I get setting value...
Promise.all(requests);
Will return a promise so you can handle it somewhere else like this:
getData(objaArr).then((result) => {
// your code
});
In your second block of code, just await for Promise.all()
return await Promise.all(requests);
You would use Promise.all as suggested in the other answer(s).
To retrieve the result you would do something like:
let [languages, currencies, countries] = await Promise.all([
this.downloadLanguages(),
this.downloadCurrencies(),
this.downloadCountries()
])
Your variables can be assigned values returned from the promises using this call structure.
Got it working. I needed to switch the initial loop to a for of. Not just the inner loop. Thanks everyone!
for (let event of events) {
if (event.type === 'A_Event') {
for ( let obj of event.payload.objArr) {
let objData = await axios.get(obj.url, options)
obj.objData = objData.data
}
}
}
You could use good old-fashioned for loops:
for (let i = 0; i < events.length; i++) {
if (events[i].type === 'A_Event') {
for (let j = 0; j < events[i].payload.objArr.length; j++) {
const data = await axios.get(events[i].payload.objArr[j].url);
events[i].payload.objArr[j].objData = data;
}
}
}

How do I wait for a promise in loop to finish before do some other stuff?

I still confused about how to use promises. I have a for loop call an asynchronous method which returns a value. I use this value to push into an array. But when I print the array it is empty. Here is what I did:
async function getLink(link) {
var browser = await puppeteer.launch({headless: true});
const page = await browser.newPage();
await page.goto(LINK)
const result = await page.evaluate( async() => {
let data = [];
const $ = window.$;
$('#gallery_01 .item').each(function(index, product) {
data.push($(product).find('a').attr('data-image'));
});
return data;
});
await browser.close();
return result;
}
var final = [];
for (var i = 0; i < 10; i++) {
var data = getLink(value[i].url).then(function(data) {
console.log(data); // urls show here
final.push(data);
});
}
Promise.all(final).then(() => {
console.log(final) // empty
})
The final show empty. What did I do wrong with Promise? Pls help!
I can't see what value is, but it looks like it's supposed to be an array of objects with a url property?
Assuming the getLink() function is okay, try this for your loop:
const final = [];
for (var i = 0; i < 10; i++) {
final.push(getLink(value[i].url));
}
Promise.all(final)
.then(data => {
console.log(data);
});
Or a slightly more compact way of accomplishing the same thing:
const promises = value.map(v => getLink(v.url));
Promise.all(promises)
.then(data => {
console.log(data);
});
Update: My bad, got a bit confused. The following code would only work without () => after the var fn
You are very close. Try this:
var final = [];
var results = []; // you need a separate array for results
for (var i = 0; i < 10; i++) {
// renamed the variable, changed 'data' to 'fn'
var fn = () => getLink(value[i].url).then(function(data) {
console.log(data); // urls show here
results.push(data);
});
final.push(fn);
}
Promise.all(final).then(() => {
console.log(results)
})
Promise.all accepts an array of promises. You have an array 'final' but seem to try to store the result of the fucntion execution as well as the function itself.
To do this correctly - first get an array of promises. Then pass them to Promise.all().
P.S. Assuming your function actually works, haven't looked at it, since the question was about promises.

Resources