recursive nodejs mysql query - node.js

I want to execute a recursive function that retrieve data from DB. In php the code below run like a charm with 15ms to execute
function GetSubCategories($catno,&$subcats, $useactive=true){
global $dbconn;
$qid = new SSQL($dbconn, "SELECT categoryno FROM article_category WHERE parent = '$catno'".($useactive?" AND active = 'Y'":"")." ORDER BY sortorder");
if ($qid->query()){
while($catrow=$qid->fetch_array()){
$subcats[]=$catrow["categoryno"];
GetSubCategories($catrow["categoryno"],$subcats, $useactive);
}
}
}
I'm a newbie in nodejs environment and Async cause trouble in this case.
If i write the same coe in js the program exit after first iteration. I can sync the process with await but execution time explode...
I try many thing with promise like
var getSubcategoriestest = function(categoryno,subcats, useactive=true){
return new Promise(async function (resolve) {
const query = `SELECT categoryno FROM article_category WHERE ?? = ? ${useactive?" AND active = 'Y'":""} ORDER BY sortorder`
let rows = await mysqlConn.query(query,['parent',categoryno])
resolve(rows)
}).then((rows)=>{
for (row of rows){
console.log(row.categoryno)
return new Promise(async function (resolve) {
await getSubcategoriestest(row.categoryno,subcats, useactive)
resolve()
}).then(()=>{console.log('end')})
}
})
}
but nothing work fine
Any guru can help me ?
Thanks
Jeremy
I test this code
var getSubcategoriestest = async function(categoryno,subcats, useactive=true,arrPromise=[]){
let promise = new Promise(function (resolve,reject) {
const query = `SELECT categoryno FROM article_category WHERE ?? = ? ${useactive?" AND active = 'Y'":""} ORDER BY sortorder`
mysqlConn.query(query,['parent',categoryno]).then((rows)=>resolve(rows)).catch(err=>console.log(err))
}).then((rows)=>{
for (row of rows){
getSubcategoriestest(row.categoryno,subcats, useactive,arrPromise).then((rows)=>{subcats.push(row.categoryno)})
}
return row.categoryno
})
arrPromise.push(promise)
Promise.all(arrPromise).then(function() {
console.log("promise all,")
return
}).catch(err=>console.log(err))
}
but function end always after first iteration. Promise.all it's call many times (cause bind at each iteration i suppose)... headache,headache,headache

Here we go
var getSubcategoriestest = function (categoryno,subcats) {
const query = `SELECT c FROM ac WHERE ?? = ? ORDER BY sortorder`
return mysqlConn.query(query,['parent',categoryno]).then(rows => {
return Promise.all(rows.map(row => {
subcats.push(row.categoryno);
return getSubcategoriestest(row.categoryno, subcats,useactive);
}));
})}
rows.map make an array of promise cause getSubcategoriestest return a promise. You can add a then after promise.all.

Related

Using batch to recursively update documents only works on small collection

I have a collection of teams containing around 80 000 documents. Every Monday I would like to reset the scores of every team using firebase cloud functions. This is my function:
exports.resetOrgScore = functions.runWith(runtimeOpts).pubsub.schedule("every monday 00:00").timeZone("Europe/Oslo").onRun(async (context) => {
let batch = admin.firestore().batch();
let count = 0;
let overallCount = 0;
const orgDocs = await admin.firestore().collection("teams").get();
orgDocs.forEach(async(doc) => {
batch.update(doc.ref, {score:0.0});
if (++count >= 500 || ++overallCount >= orgDocs.docs.length) {
await batch.commit();
batch = admin.firestore().batch();
count = 0;
}
});
});
I tried running the function in a smaller collection of 10 documents and it's working fine, but when running the function in the "teams" collection it returns "Cannot modify a WriteBatch that has been committed". I tried returning the promise like this(code below) but that doesn't fix the problem. Thanks in advance :)
return await batch.commit().then(function () {
batch = admin.firestore().batch();
count = 0;
return null;
});
There are three problems in your code:
You use async/await with forEach() which is not recommended: The problem is that the callback passed to forEach() is not being awaited, see more explanations here or here.
As detailed in the error you "Cannot modify a WriteBatch that has been committed". With await batch.commit(); batch = admin.firestore().batch(); it's exactly what you are doing.
As important, you don't return the promise returned by the asynchronous methods. See here for more details.
You'll find in the doc (see Node.js tab) a code which allows to delete, by recursively using a batch, all the docs of a collection. It's easy to adapt it to update the docs, as follows. Note that we use a dateUpdated flag to select the docs for each new batch: with the original code, the docs were deleted so no need for a flag...
const runtimeOpts = {
timeoutSeconds: 540,
memory: '1GB',
};
exports.resetOrgScore = functions
.runWith(runtimeOpts)
.pubsub
.schedule("every monday 00:00")
.timeZone("Europe/Oslo")
.onRun((context) => {
return new Promise((resolve, reject) => {
deleteQueryBatch(resolve).catch(reject);
});
});
async function deleteQueryBatch(resolve) {
const db = admin.firestore();
const snapshot = await db
.collection('teams')
.where('dateUpdated', '==', "20210302")
.orderBy('__name__')
.limit(499)
.get();
const batchSize = snapshot.size;
if (batchSize === 0) {
// When there are no documents left, we are done
resolve();
return;
}
// Delete documents in a batch
const batch = db.batch();
snapshot.docs.forEach((doc) => {
batch.update(doc.ref, { score:0.0, dateUpdated: "20210303" });
});
await batch.commit();
// Recurse on the next process tick, to avoid
// exploding the stack.
process.nextTick(() => {
deleteQueryBatch(resolve);
});
}
Note that the above Cloud Function is configured with the maximum value for the time out, i.e. 9 minutes.
If it appears that all your docs cannot be updated within 9 minutes, you will need to find another approach, for example using the Admin SDK from one of your server, or cutting the work into pieces and run the CF several times.

combining results from several async/await calls (again)

Note to would-be closers or markers-as-duplicate : this question is not answered in How do I convert an existing callback API to promises?, as all answers there treat of calls in isolation and do not explain how to deal with successive, dependent calls.
This question is basically the same as in combining results from several async/await calls, but because of my slightly different context I fail to see how the solution therein can be adapted/mimicked.
I have two successive calls to a database, using an old API which only knows about callbacks. The second call needs objects/values returned by the first.
I have a working callback-version of the code, as follows :
connection.query(sql1,function(error1,results1,fields1) {
if(error1) {
console.log("Error during first query",error1);
}
let sql2=computeUsingFirstResult(result1);
connection.query(sql2,function(error2,results2,fields2) {
if(error2) {
console.log("Error during second query",error2);
}
doSomething(connection,results1,results2);
})
});
Here is my unsuccessful attempt to do it in async/await-style :
const util = require('util');
async function firstQuery(connection) {
return util.promisify(connection.query).call(sql1).catch(
error1 => console.log("Error during first query : ", error1)
);;
}
async function secondQuery(connection, result1) {
let sql2 = computeUsingFirstResult(result1);
return util.promisify(connection.query).call(sql2).catch(
error2 => console.log("Error during second query : ", error2)
);
}
let finalResult = {};
async function main() {
const results1 = await firstQuery(connection);
const results2 = await secondQuery(connection, results1);
doSomething(connection, results1, results2);
finalResult = [results1,results2];
console.log("Here is the finalResult : ", finalResult);
}
main().catch(
err => console.log("Something went wrong towards the end", err)
);
My async/await version fails as all the intermediate results are undefined. Yet, as far as I can see it should be equivalent to the non-async version above.
What is the correct way to do this ?
There are 2 approaches I am hoping you can try and see if either of them help:
(1) Bind the util.promisify to connection for both firstQuery and secondQuery something like so:
async function firstQuery(connection) {
return util.promisify(connection.query).bind(connection,sql1)().catch(
error1 => console.log("Error during first query : ", error1)
);;
}
// bind makes sure that internal bindings of connection object are in tact
If the above approach doesn't yield any result, try the next approach:
(2) Try using Promises-Wrapper instead of util.promisify for both firstQuery and secondQuery like so:
function firstQuery(connection,sql1){
return new Promise((resolve,reject)=>{
connection.query(sql1,function(error1,results1,fields1){
return error1 ? reject(error1):resolve(results1);
})
});
}
And now call them as you were in your code using Async/Await in the main function.
Few Potential bugs/typos I noticed in the code:
(1) firstQuery doesn't seem to be passed sql1 as one of its arguments, maybe this is intentional if sql1 exists in global scope.
(2) If you attach a catch block to both your queries, they will have an impact on your 2 calls (see comments in the code below):
async function main() {
const results1 = await firstQuery(connection); //<-- if this call fails, results1 will be undefined and the call will go to next step
const results2 = await secondQuery(connection, results1); // same thing here
doSomething(connection, results1, results2);
finalResult = [results1,results2];
console.log("Here is the finalResult : ", finalResult);
}
A possible solution is to remove the catch blocks from individual functions and just return the respective promises.
Then you can deal with them via a single try/catch block like this:
async function main() {
try{
const results1 = await firstQuery(connection);
const results2 = await secondQuery(connection, results1);
doSomething(connection, results1, results2);
finalResult = [results1,results2];
console.log("Here is the finalResult : ", finalResult);
}catch(err){
console.log("Error",err);
}
}
This will ensure that as soon as the first call fails, function goes straight to catch block without executing any other lines.

How to call recursively a function that returns a promise?

I want to extract all child-Folders and child-Docs querying a node-js Service, which every time it is called, returns an array of such items. I do not know the depth fo the folders-tree so I want to recursively call a function that in the end will return an array that will contain all child-folders and child-docs, starting from a list of root-Folders. Each folder is identified by a folder id.
So I have made a "recPromise(fId)" which returns a promise. Inside, this function calls recursively the recFun(folderId).I start invoking the "recPromise(fId)" from a rootFolder so once all root-promises are resolved I can go on.
rootFolders.map( folderOfRootlevel =>{
var folderContentPromise = recPromise(folderOfRootlevel.id);
folderContentPromises.push(folderContentPromise);
})
$q.all(folderContentPromises)
.then(function(folderContent) {
// Do stuff with results.
}
function recPromise(fId){
return new Promise((resolve, reject) => {
var items = [];
function recFun( folderId) { // asynchronous recursive function
function handleFolderContent( asyncResult) { // process async result and decide what to do
items.push(asyncResult);
//Now I am in a leaf-node, no child-Folders exist so I return
if (asyncResult.data.childFolders.length === 0){
return items;
}
else {
//child_folders exist. So call again recFun() for every child-Folder
for(var item of asyncResult.data.childFolders) {
return recFun(item._id);
}
}
}
// This is the service that returns the array of child-Folders and child-Docs
return NodeJSService.ListFolders(folderId).then(handleFolderContent);
}
resolve(recFun(fId));
})
}
It works almost as expected except the loop inside else, where I call again recFun().
The NodeJSService will return an array of sub-Folders so I wish to call recfun() for every sub-Folder.
Now, I only get the result of the 1st sub-Folder of the loop,
which makes sense since I have a return statement there.
If I remove the return statement and call like this "recFun(item._id);"
then it breaks the $q.all().
Finally, I decided to remove the Promise wrapper function and make use of async-await.
var items = [];
(async() => {
for(var item of rootFolders) {
await recFun(item.id)
}
// Do stuff with items
// go on form here..
})()
function listFolders(folderId) {
return new Promise( function( resolve, reject) {
resolve(FolderService.ListFolders(folderId));
})
}
async function recFun(folderId) {
var foldersResponse= await listFolders(folderId);
items.push(foldersResponse);
if (foldersResponse.data.childFolders.length === 0){
return items ;
}
else {
for(var item of foldersResponse.data.childFolders) {
await recFun(item._id);
}
}
}

code sequence in nodejs

I am new to node.js and I thought I was beginning to understand asynchronous functions, but this code made me think that I did not understand it correctly anyway.
I am preparing for an insert to mongoDB with mongoose, and the object to insert is post. What made me wonder is that not always post.kunde or post.leverandor is set before the insert.
I thought that as long there is no asynchronous function, the code should execute line by line.
function create_post(account, dato, fakturanummer, bilag, bilagstype, supplier, customer,
descr, moms, amount, saldo, comId ) {
return new Promise(function(resolve, reject) {
var post = new Posteringer;
post.konto = account;
post.dato = dato;
post.fakturanummer = fakturanummer;
post.bilag = bilag;
post.bilagstype = bilagstype;
if (Object.keys(supplier).length) {
post.leverandor = supplier;
}
if (Object.keys(customer).length) {
post.kunde = customer;
}
post.tekst = descr;
post.moms = moms;
post.belob = amount;
post.saldo = saldo;
post.companyId = comId;
var promise = Accounts.findSingle(comId, account).exec();
promise.then(function(acc) {
post.navn = acc.Navn;
console.log(post);
post.save(function(err) {
if (err) {console.log(err.message);}
resolve(true);
});
});
});
}
So there are a few things wrong here. A promise is basically saying "hey, I'm not done doing my stuff yet, but I'll promise to get this to you when I figure out all the things I need to do."
I am not entirely sure how you are calling this, but think of this like a big callback function. So after this you'd have something like,
create_post(......).then(
function(post){
post.save(function(err){
if (err) {console.log(err.message);}
});
});
The real issue I see is you are only resolving one promise. I am not 100% sure if you have to resolve the Mongo promise, unless you are using bluebird promises with it. Your first promise never gets returned though.
I will redo your code, I think this should work:
function create_post(account, dato, fakturanummer, bilag, bilagstype, supplier, customer,
descr, moms, amount, saldo, comId ) {
return new Promise(function(resolve, reject) {
var post = new Posteringer;
post.konto = account;
post.dato = dato;
post.fakturanummer = fakturanummer;
post.bilag = bilag;
post.bilagstype = bilagstype;
if (Object.keys(supplier).length) {
post.leverandor = supplier;
}
if (Object.keys(customer).length) {
post.kunde = customer;
}
post.tekst = descr;
post.moms = moms;
post.belob = amount;
post.saldo = saldo;
post.companyId = comId;
var promise = Accounts.findSingle(comId, account).exec();
promise.then(function(acc) {
post.navn = acc.Navn;
console.log(post);
});
return (err ? reject(err) : resolve(post));
});
}
Then when you call this function call it like in my first example!
create_post(......).then(
function(post){
post.save(function(err){
if (err) {console.log(err.message);}
});
});
Literally what you are saying is if this resolves and I get my data without any errors, send back post. Then pass post into my callback function and do whatever you want to do with it.
Edit:
Make sure you always return something from your promises, as far as I know the async call will never resolve as it never receives anything. Although someone might have an example where this isn't true.

Nodejs: Resolving promises with generator function

I know there are a lot of good examples over the web and I read a lot of them, but currently I'm stucked with resolving promises with the new functionality of generators in nodejs 0.11.x.
For e.g. I have the following function:
SolrBaseDomain.prototype.promisedQuery = function(query, callback) {
var solrClient = solr.createClient(this.configuration);
var defer = Q.defer();
solrClient.search(query, function(err,obj){
if (!err) {
if (obj.response.numFound > 0) {
defer.resolve(obj.response.docs);
} else {
defer.resolve(null);
}
} else {
defer.reject(err);
}
});
var promise = defer.promise;
return Q.async(function* (){
var result = yield promise;
return result;
});
};
I expected that every call to this method will wait until the promise is fullfilled and the return-statement gives back the result of the promise.
But currently it seems that instead the code inside "Q.async..." will not be executed or the async call arrives after the return statement of the method was executed.
It's strange, in every example I know, this is one of the recommended ways in order to wait for async calls in nodejs but currently it does not work for me.
I've tried a lot of different variations of the above example, but the result is everytime the same, I get not back a valid result.
I have nodejs installed in version 0.11.10 and the --harmony-flag is set, when the code is executede.
Can anyone point me to right direction? I'm wondering if I oversee something ... :)
Thanks for your feedback.
Best regards
Udo
I expected that every call to this method will wait until the promise is fullfilled and the return-statement gives back the result of the promise.
No. Generators will not make functions synchronous, you cannot (and don't want to) block while waiting for a result. When calling a generator function and running sequentially through the async steps that it yields, the result you will get back in the end is still asynchronous - and therefore a promise. Only inside of the generator, your code can use synchronous control flow and yield.
This means that the (then-) callback-based code
SolrBaseDomain.prototype.promisedQuery = function(query) {
var promise = Q.ninvoke(solr.createClient(this.configuration), "search", query);
return promise.then(function(obj) {
if (obj.response.numFound > 0) {
return obj.response.docs;
} else {
return null;
}
});
};
becomes
SolrBaseDomain.prototype.promisedQuery = Q.async(function* (query) {
var promise = Q.ninvoke(solr.createClient(this.configuration), "search", query);
var obj = yield promise;
// ^^^^^
if (obj.response.numFound > 0) {
return obj.response.docs;
} else {
return null;
}
});
Try this
SolrBaseDomain.prototype.promisedQuery = Q.async(function*(query) {
var solrClient = solr.createClient(this.configuration);
var obj = yield Q.ninvoke(solrClient, "search", query);
return obj.response.numFound > 0 ? obj.response.docs : null;
});
This does the same thing for promises as this does for callbacks:
SolrBaseDomain.prototype.query = function (query, callback) {
var solrClient = solr.createClient(this.configuration);
solrClient.search(query, function(err, obj) {
if (err) return callback(err);
callback(null, obj.response.numFound > 0 ? obj.response.docs : null);
});
};
Therefore if the first return a promise that resolves to undefined so will the callback version call the callback with undefined.
according to your suggestions, my code looks now like this:
...
SolrBaseDomain.prototype.query = Q.async(function* (query) {
var solrClient = solr.createClient(this.configuration);
var obj = yield Q.ninvoke(solrClient, "search", query);
return obj.response.numFound > 0 ? obj.response.docs : null;
});
...
I share the above query-function over all data access layers in order to have a central method which is querying the different indexes in an asynchronous way.
For e.g. in the domain data access layer, the code which deals with that function looks like this:
SolrHostDomain.prototype.getByName = Q.async(function* (domain) {
var queryObject = {
"domain": domain
};
var query = this.getQuery("byName", queryObject);
var docs = yield this.query(query);
var domain = null;
if (docs != null && docs.length > 0) {
domain = this.dataMapper.merge(docs[0]);
}
return domain;});
Currently I'm not sure if the generator in the "getByName"-function is necessary, but it seems to work. Dealing with promises is some unclear concept for me, since I'm new to nodejs.
So maybe, if you can help me on that topic and point me in the right direction, this would be helpfull.
The main question for me is, how can I ensure, that a synchronous method can call an asynchronous method and get back not a promise but the final result of this promise.
I've searched a long time, but I could not find a good documentation which describes the use of generator functions or promises in conjunction with synchronous calls. Even examples are focusing only of using the mechanism but not working together with synchronous function.
Best regards and many thanks for your help
Udo
Got it!!!
After a few trial and errors, I think I got it now and I have a working solutions:
Query function:
SolrBaseDomain.prototype.query = Q.async(function* (query) {
var solrClient = solr.createClient(this.configuration);
var obj = yield Q.ninvoke(solrClient, "search", query);
return obj.response.numFound > 0 ? obj.response.docs : null;
});
Calling method:
SolrHostDomain.prototype.getByName = function(domain) {
var queryObject = {
"domain": domain
};
var query = this.getQuery("byName", queryObject);
var docsPromise = this.query(query);
var _self = this;
return docsPromise.then(function(docs) {
var domain = null;
if (docs != null && docs.length > 0) {
domain = _self.dataMapper.merge(docs[0]);
}
return domain;
});
};
The solution was to understand, that the "query"-method still returns a promise instead of the concrete result even if yield is used.
So I have to add every code which is working on the result of the promise within the "then"-functions (or "done" if no other caller up in the calling hierarchy of methods will follow).
After the settlement of the promise, each code which is set within the "then"-functions will be processed.
BR
Udo

Resources