Resolve array of promises node.js - node.js

I am new to promises, I am trying to use RSVP promises in Node.js with PostgreSQL and I am doing it wrong, most probably.
Any suggestions on how to fix that or how to improve the code are appreciated.
What I try to achieve is: after receiving data - process the data to create SQL update queries and when they are ready - execute them. Data here is array of user ids.
What does not work: I get array of array of promises that doesn't resolve, I tried to resolve the array like so:
var promise4all = RSVP.all(
updateQueries.map((innerPromiseArray) => {
return RSVP.all(innerPromiseArray);
})
);
promise4all.then((promiseGroupResult) => {
// doesn't get here
});
But it didn't work also.
The code:
1) The function 'update' that receives data and calls function 'promiseQuery' to process the data:
const RSVP = require('rsvp');
let db;
const update = (data) => {
let users = {
items: data.user, // data to be updated in db - array of user ids
item_type: 1,
id: data.department
}
let updateQueries = [];
// adding query promises to updateQueries
updateQueries.push(promiseQuery(users.id, users.item_type, users.items));
RSVP.all(updateQueries).then((results) => {
/* here 'results' looks like that:
[ [ { query: 'INSERT INTO link_to_department (item_type, department, item) VALUES ($item_type, $department, $item)',
values: [Object] },
{ query: 'DELETE FROM link_to_department WHERE department = $(department) AND item_type = $(item_type) AND item=$(item)',
values: [Object] } ] ]
db update below fails with '[Empty or undefined query.]'*/
db.tx((trx) => {
let sqlUpdates = [];
results.forEach((query) => {
sqlUpdates.push(trx.none(query.query, query.values))
})
return trx.batch(sqlUpdates);
}).then(() => {
res.sendStatus(204);
}).catch((err) => {
console.log('error', err.message);
// handle errors
});
});
};
2) The function 'promiseQuery' processes data (it compares received data and data in db to update db with the new data):
const promiseQuery = (department_id, item_type, items) => {
return new RSVP.Promise((resolve, reject) => {
db.query('SELECT item FROM link_to_department WHERE department=' + department_id + ' AND item_type=' + item_type)
.then((db_items) => {
let promises = [];
let itemsToBeRemoved = [];
let itemsToBeAdded = [];
/* here we have array of user ids we received: 'items'
and array of user ids from db: 'db_items' */
// populating 'itemsToBeAdded' and 'itemsToBeRemoved' with user ids that need to added or removed:
populateUpdateArray(items, db_items, itemsToBeAdded);
populateUpdateArray(db_items, items, itemsToBeRemoved);
let insert_query = 'INSERT INTO link_to_department (item_type, department, item) VALUES ($item_type, $department, $item)'
let delete_query = 'DELETE FROM link_to_department WHERE department = $(department) AND item_type = $(item_type) AND item=$(item)';
// creating update sql queries
populateUpdateQuery(insert_query, itemsToBeAdded, department_id, item_type, promises);
populateUpdateQuery(delete_query, itemsToBeRemoved, department_id, item_type, promises);
RSVP.all(promises).then((results) => {
/* here 'results' looks like this:
[ { query: 'INSERT INTO link_to_department (item_type, department, item) VALUES ($item_type, $department, $item)',
values: { item_type: 19, department: 1, item: '1' } },
{ query: 'DELETE FROM link_to_department WHERE department = $(department) AND item_type = $(item_type) AND item=$(item)',
values: { item_type: 19, department: 1, item: 1 } }] */
return resolve(results);
});
}).catch(() => {
reject();
})
});
};
3) That function 'populateUpdateArray' populates array of user ids that need to be updated (basically, received user ids should replace ids in the db - for that we check what ids we received are not in db and what ids in db are not in the received ids):
const populateUpdateArray = (array_0, array_1, updateArray) => {
array_0.forEach((item) => {
if (array_1.indexOf(item) === -1) {
updateArray.push(item);
}
});
};
4) That function 'populateUpdateQuery' returns sql update queries:
const populateUpdateQuery = (query, id_array, department_id, item_type, promises) => {
return new RSVP.Promise((resolve, reject) => {
id_array.forEach((item) => {
let values = {
item_type: item_type,
department: department_id,
item: item
};
promises.push({query, values});
});
resolve(promises);
});
};
Thank you!
EDIT: I changed the code to have only one db connection and I simplified the code a little. I do not get any errors, but queries are not executed, still. I think I am missing something basic here:
const update = (data) => {
let users = {
items: data.user,
item_type: 1,
id: data.department
}
db.tx((tx) => {
let updateQueries = [];
updateQueries.push(promiseQuery(department.id, users.item_type, users.items, tx));
RSVP.all(updateQueries).then((results) => {
// results is array of array, so i flatten it
let sqlUpdates = results.reduce((a, b) => { return a.concat(b); }, []);
/* sqlUpdates here:
[ Promise {
_bitField: 0,
_fulfillmentHandler0: undefined,
_rejectionHandler0: undefined,
_promise0: undefined,
_receiver0: undefined } ]
*/
return tx.batch(sqlUpdates);
});
}).then(() => {
res.sendStatus(204);
}).catch((err) => {
console.log('error', err.message);
});
};
const promiseQuery = (department_id, item_type, items, tx) => {
return new RSVP.Promise((resolve, reject) => {
tx.query('SELECT item FROM belongs_to_departments WHERE department=' + department_id + ' AND item_type=' + item_type)
.then((db_items)=> {
let queries = [];
let itemsToBeAdded = [];
let insert_query = 'INSERT INTO belongs_to_departments (item_type, department, item) VALUES ($(item_type), $(department), $(item))';
populateUpdateArray(items, db_items, itemsToBeAdded);
populateUpdateQuery(insert_query, itemsToBeAdded, department_id, item_type, queries, tx);
resolve(queries);
}).catch(() => {
reject();
});
});
};
const populateUpdateArray = (array_0, array_1, updateArray) => {
array_0.forEach((item) => {
if (array_1.indexOf(item) === -1) {
updateArray.push(item);
}
});
};
const populateUpdateQuery = (query, id_array, department_id, item_type, queries, tx) => {
id_array.forEach((item) => {
let values = {
item_type: item_type,
department: department_id,
item: item
};
queries.push(tx.none(query, values));
});
};

Thanks to Vitaly for the help.
That worked for me:
const update = data => {
const users = {
items: data.user,
item_type: 1,
id: data.department
}
db.tx(tx => {
const updateQueries = [];
updateQueries.push(promiseQuery(department.id, users.item_type, users.items, tx));
RSVP.all(updateQueries).then(results => {
// results is array of array, so i flatten it
const sqlUpdates = results.reduce((a, b) => { return a.concat(b); }, []);
return tx.batch(sqlUpdates);
});
}).then(() => {
res.sendStatus(204);
}).catch(err => {
console.log('error', err.message);
});
};
const promiseQuery = (department_id, item_type, items, tx) => {
return new RSVP.Promise((resolve, reject) => {
tx.query('SELECT item FROM belongs_to_departments WHERE department=' + department_id + ' AND item_type=' + item_type)
.then(db_items => {
const queries = [];
const itemsToBeAdded = [];
const insert_query = 'INSERT INTO belongs_to_departments (item_type, department, item) VALUES ($(item_type), $(department), $(item))';
populateUpdateArray(items, db_items, itemsToBeAdded);
populateUpdateQuery(insert_query, itemsToBeAdded, department_id, item_type, queries, tx);
resolve(queries);
}).catch(() => {
reject();
});
});
};
const populateUpdateArray = (array_0, array_1, updateArray) => {
array_0.forEach((item) => {
if (array_1.indexOf(item) === -1) {
updateArray.push(item);
}
});
};
const populateUpdateQuery = (query, id_array, department_id, item_type, queries, tx) => {
id_array.forEach(item => {
const values = {
item_type: item_type,
department: department_id,
item: item
};
queries.push(tx.none(query, values));
});
};

Related

Firebase nodejs doesn't execute return function properly

We are trying to get timeslots from our database by pushing them into an array and then returning it. The array does get filled properly according to the firebase logs, however the function never returns the data properly at all, even though we see the data to be returned.
Basically, the execution does not reach the return statement.
Our goal is to get all of the timeslots in this photo. Is there any neat way to do this?
exports.getTimeslots = functions.region('europe-west2').https.onCall((data, context) => {
const uid = context.auth.uid;
let array = [];
if (!uid)
throw new functions.https.HttpsError('no-userid', 'The requested user was not found');
else
return admin.firestore().collection('users').doc(uid).collection('modules').where('name', '!=', '').get().then(snapshot => {
snapshot.forEach(async doc => {
await admin.firestore().collection('users').doc(uid).collection('modules').doc(doc.id).collection('timeslots').where('length', '!=', -1).get().then(snapshot2 => {
snapshot2.forEach(doc2 => {
array.push(Object.assign(doc2.data(), {id: doc2.id, modID: doc.id}))
console.log("identifier #1", array)
})
console.log("Got outside");
})
console.log("Got more outside");
})
console.log("Got the most outside")
return ({ data: array });
});
//console.log("I have escaped!")
})
As #Ragesh Ramesh also said: "The solution is to make everything async await.", I tried replicating your code using the data structure, and code below:
Data Structure:
Code:
// firebase db
const db = firebase.firestore();
exports.getTimeslots = functions.region('europe-west2').https.onCall((data, context) => {
const getData = async () => {
const uid = context.auth.uid;
let array = [];
if (!uid) {
throw new functions.https.HttpsError('no-userid', 'The requested user was not found');
} else {
const modulesRef = db.collection('users').doc(uid).collection('modules');
const modulesQuery = modulesRef.where('name', '!=', '');
const modulesQuerySnap = await modulesQuery.get();
const moduleDocuments = modulesQuerySnap.docs.map((doc) => ({ id: doc.id }));
for (const moduleDocument of moduleDocuments) {
const timeslotsRef = modulesRef.doc(moduleDocument.id).collection('timeslots');
const timeslotsQuery = timeslotsRef.where('length', '!=', -1);
const timeslotsQuerySnap = await timeslotsQuery.get();
const timeslotDocuments = timeslotsQuerySnap.docs.map((doc) => ({ id: doc.id, data: doc.data() }));
for (const timeslotDocument of timeslotDocuments) {
array.push(Object.assign(timeslotDocument.data, {id: timeslotDocument.id, modID: moduleDocument.id}))
}
}
return ({ data: array });
}
}
return getData()
.then((response) => {
// console.log(response);
return response;
});
}
The Reference page for Firestore reveals the docs property on the snapshot.
Upon running the code, here's the output:
{
data: [
{
length: 1,
id: '8UIlspnvelEkCUauZtWv',
modID: 'RmL5BWhKswEuMWytTIvZ'
},
{
title: 'Modules',
length: 120,
day: 1,
startTime: 720,
id: 'E5fjoGPyMswOeq8mDjz2',
modID: 'qa15lWTJMjkEvOU74N1j'
},
{
startTime: 360,
title: 'English',
day: 2,
length: 240,
id: '7JHtPSO83flO3nFOc0aE',
modID: 'qa15lWTJMjkEvOU74N1j'
}
]
}
This is a issue with how your function is written. Instead of
return ({ data: array });
Your function sometimes returns.
admin.firestore().collection('users').doc(uid).collection('modules').where('name', '!=', '').get()
Which is a promise by itself. You are chaining async inside then function. The solution is to make everything async await.
const data = await admin.firestore().collection('users').doc(uid).collection('modules').where('name', '!=', '').get()

MSSQL Nodejs Streaming is not waiting to finish to Return

This is my first time dealing with streams and I am stuck. I am using the mssql NodeJS package.
I have these functions below to upsert very large data. For each region, I get the data from SQL Server and for each 10000 lines upsert to Mongo.
The problem is, _refreshArsenalData function will not wait for the sqlService.refreshArsenal function to finish, so it will return undefined. I tried making sqlService.refreshArsenal async and awaiting it but it seems like it didn't change anything.
What am I doing wrong?
async function _refreshArsenalData({ orgId }, ctx) {
const lastUpdate = await dataService.getLastUpdate(orgId)
const regions = await dataService.getRegions(orgId)
await asyncForEach(regions, async (region ) => {
const query = sqlService.getQueryForRefreshArsenal(region, lastUpdate)
let { upsertedCount, matchedCount } = await sqlService.refreshArsenal(query, orgId)
return {upsertedCount, matchedCount}
})
}
async function asyncForEach(array: any, callback: any) {
for (let index = 0; index < array.length; index++) {
await callback(array[index], index, array);
}
}
SQL SERVICE
refreshArsenal = async (query: string, orgId: string) => {
let upsertedCount = 0
let matchedCount = 0
await sql.connect(config.sqlConnection, async (err: any) => {
const request = new sql.Request()
request.stream = true
request.query(query)
let rowsToProcess: any[] = []
request.on(`row`, async (row: any) => {
rowsToProcess.push(row)
if (rowsToProcess.length >= 10000) {
request.pause();
let { matched, upserted } = await bulkUpsertMongo();
upsertedCount += upserted
matchedCount += matched
}
})
request.on(`done`, async () => {
if (rowsToProcess.length) {
let { matched, upserted } = await bulkUpsertMongo();
upsertedCount += upserted
matchedCount += matched
}
})
sql.on(`error`, () => {
throw new Error(err.toString())
})
async function bulkUpsertMongo() {
let ops: any[] = []
for (const row of rowsToProcess) {
ops.push({
updateOne: {
filter: { rnc_regid: row.rnc_regid },
update: { $set: row },
upsert: true
},
})
}
const result = await bulkUpsert(ops, orgId)
rowsToProcess = []
request.resume()
return { matched: result.matchedCount, upserted: result.upsertedCount }
}
})
return { upsertedCount, matchedCount }
}

Return inside map function is not working

I'm trying to get product details like price, discount by the id i'm getting from my cart. This return function is returning null. but working perfectly in console.log.
async function store (req,res) {
const item = req.session.cart;
const cart = new Cart(item);
const results = cart.generateArray();
let result = [];
result = results.map(item => {
Products.findOne({'_id': item.id}, function (err, r) {
if(err){
console.log(err);
} else {
return ({
product_id: r._id,
price: r.price,
qty: item.qty,
total: r.price*item.qty
});
// this return is not working
}
});
});
let data = await result;
return res.send(data);
}
You should use Promise.all to solve an array of promises:
result = results.map(async item => {
try {
const r = await Products.findOne({'_id': item.id})
return ({
product_id: r._id,
price: r.price,
qty: item.qty,
total: r.price*item.qty
});
} catch (err) {
console.log(err);
}
});
let data = await Promise.all(result);

Await all Mongo queries with Node.js

Code below has a flaw as I am getting array of undefined:
let filters = [];
async function getFilters(tiers) {
return await Promise.all(
tiers.map(async t => {
let id = new ObjectId(t.filter);
filters.push(
await conn.collection('TierScheduleFilter').find({
_id: id
}).toArray(function(err, filter) {
if (err || !filter) {
reject('no filter || error');
}
return filter;
});
);
});
);
}
await getFilters(tiers);
console.log(filters); // 4 filters => [ undefined, undefined, undefined, undefined ]
The code shall retrieve all filters but its all undefined values.
This one seems to be a proper approach:
let filters = [];
async function getFilters(tiers) {
return await Promise.all(
tiers.map(async t => {
let id = new ObjectId(t.filter);
try {
return await conn.collection('TierScheduleFilter').findOne({ _id: id })
} catch (e) {
return e;
}
}))
}

Node JS Promise.all and forEach in nested Array

In a Node app with typescript, i need to iterate through a array and i call an asynchronous function inside the loop to get informations about the related items for each item in the array. The function is called for each related item to get its title in relatedItems array.
I'm able to retrieve promises for the 2nd level array (relatedItems) but not sure how to implement a then once top level finishes as well.
How to reach my goal with promise.all.
var Inputarray = [
{ Category: "cat1"
relatedItems: [
{id: "1"},
{id: "2"},
{id: "3"}
]
},
{
Category: "cat2"
relatedItems: [
{id: "1"},
{id: "2"},
{id: "3"}
]
}
];
var wantedresult= [
{ Category: "cat1"
relatedItems: [
{Title: "xxx"},
{Title: "yyy"},
{Title: "zzz"}
]
},
{
Category: "cat2"
relatedItems: [
{Title: "ttt"},
{Title: "kkk"},
{Title: "mmm"}
]
}
];
private GetAllRelattedItems(data: IListItem[]): any{
let rendredItems: RelatedItem[] = new Array();
data.forEach(item => {
let relatedItemsinfos : relatedItemInfos[]=item.Children;
let localTab:relatedItem[]=new Array();
let newItem = {
Category:item.Category,
Children: []
};
var promises = [];
relatedItemsinfos.forEach(relatedItemsInfositem =>{
promises.push(this.GetRelatedItem(relatedItemsInfositem.WebId,relatedItemsInfositem.ListId,relatedItemsInfositem.ItemId));
});
Promise.all(promises).then(function(response) {
response.forEach(obj=>{
let newNode: relatedItem ={
Title :Obj.Title,
};
newItem.Children.push(newNode);
});
rendredItems.push(newItem);
});
});
}
private GetRelatedItem(id:string) : Promise<relatedItem> {
return new Promise((resolve) => {
pnp.sp.site.openWeb()
.then(web => {
//this.getWeb()
//.then((web) => {
return web.web.lists.getList().get(); //w.web.lists.getById("").get().then(r => {
})
.then((list) => {
return this.getItem(list,id);
})
.then((item) => {
resolve(item);
});
});
}
You should use Promise.all at the top level and return the promise for each item in the data array:
private GetAllRelattedItems(data: IListItem[]): any {
//..
let allData = data.map(item => {
//..
return Promise.all(promises).then(function (response) {
//..
});
});
Promise.all(allData).then (_=> { /* After all data is retrieved */})
}
Since you are using Typescript a better approach would be to take advantage of async/await to make the code more readable:
private async GetAllRelattedItems(data: IListItem[]): Promise<RendredItem[]> {
let allData = data.map(async item => {
let relatedItemsinfos: relatedItemInfos[] = item.Children;
let localTab: relatedItem[] = new Array();
let newItem = {
Category: item.Category,
Children: []
};
var response = await Promise.all(relatedItemsinfos.map(relatedItemsInfositem => {
return this.GetRelatedItem(relatedItemsInfositem.WebId);
}));
newItem.Children = response.map(entry => ({
Title: entry.value[0].Title,
FileLeafRef: entry.value[0].FileLeafRef,
Modified: entry.value[0].Modified,
FileRef: entry.value[0].FileRef,
FieldValuesAsText: {
FileRef: entry.value[0].FileRef,
}
}));
return newItem;
});
var result = await Promise.all(allData);
// After all results are done
return result
}
private async GetRelatedItem(id: string): Promise<{ value: relatedItem[] }> {
const web = await pnp.sp.site.openWeb()
const list = await web.web.lists.getList().get(); //w.web.lists.getById("").get().then(r => {
return await this.getItem(list,id);
}
// Placeholder
public getItem(list: any[], id: string ): Promise<{ value: relatedItem[] }>{
return Promise.resolve({
value : []
});
}
Note I guessed some of the types based on usage, you should review to make sure they are correct.
private async GetAllRelattedItems(data: IListItem[]): Promise<RendredItem[]> {
let allData = data.map(async item => {
let relatedItemsinfos: relatedItemInfos[] = item.Children;
let localTab: relatedItem[] = new Array();
let newItem = {
Category: item.Category,
Children: []
};
var response = await Promise.all(relatedItemsinfos.map(relatedItemsInfositem => {
return this.GetRelatedItem(relatedItemsInfositem);
}));
newItem.Children = response.map(entry => ({
Title: entry.value[0].Title,
FileLeafRef: entry.value[0].FileLeafRef,
Modified: entry.value[0].Modified,
FileRef: entry.value[0].FileRef,
FieldValuesAsText: {
FileRef: entry.value[0].FileRef,
}
}));
return newItem;
});
var result = await Promise.all(allData);
// After all results are done
return result
}
private async GetRelatedItem(relatedItemsInfositem) : Promise<any> {
const web = await pnp.sp.site.openWebById(relatedItemsInfositem.WebId);
const list = await web.web.lists.getById(relatedItemsInfositem.ListId).get();
return await this.getItem(list,relatedItemsInfositem.ItemId);
// here i would like to call GetItemSize that is async and that take return the size as a number but i would like to return the whole item model
}
public getItem(list: any, itemId: string): Promise<any>{
let url: string;
if(list.ParentWebUrl !='/'){
url= list.ParentWebUrl + "/_api/web/lists/getbytitle('"+list.Title+"')/items?$select=FileLeafRef,FileRef,Title,File/ServerRelativeUrl,Modified,FieldValuesAsText/FileRef&$expand=File&$expand=FieldValuesAsText&$filter=Id eq "+itemId;
}
else url="/_api/web/lists/getbytitle('"+list.Title+"')/items?$select=FileLeafRef,FileRef,Title,File/ServerRelativeUrl,Modified,FieldValuesAsText/FileRef&$expand=FieldValuesAsText&$expand=File&$filter=Id eq "+itemId;
return this.context.spHttpClient.get(url,SPHttpClient.configurations.v1).then((response: Response)=>{
return response.json();
});
}
private async GetItemSize(title: string): Promise<relatedItem>{
let url:string ;
let response‌​ = await this.context.spHttpClient.get(url ,SPHttpClient.configuration‌​s.v1);
// Asuming the json has the shape of relatedItem : Here the response is just a number and not shaped RelatedItem the goal is to return the whole relatedItem with size information.
return <relatedItem>response.json(); }

Resources