Making an async loop in node.js with sql - node.js

Here is what I'm trying to do:
Start a looping (10x)
Select on sql to return 1 register (select top 1 where 'running' is null)
Sql update 'running' status to 'running'
If the record is null, I access an API and get some data
The result is updated on the initial sql record (set running = 'ok')
End looping (start over)
Thing is, node.js does not wait for start over, it does everything at the same time. That way, 'running' is always null.
var express = require('express');
var app = express();
var c_MyApi = require('./controller/call_MyApi');
var mongo = require('./controller/crud_mongo');
var c_email = require('./controller/api_email_verify');
var c_sql = require('./controller/consulta_sql');
var MyLink = '',
id = 0;
for( var i = 0 ; i < 10 ; i++){
c_sql.busca().then(function(res) {
MyLink = res[0].MyLink;
id = res[0].id;
c_sql.marca(id).then(
c_MyApi.busca(MyLink, function(a) {
if (a == 0) {
c_sql.atualiza(id, 'sem_email', 's/e');
}
if (a == 1) {
c_sql.atualiza(id, 'link_errado', 'l/e');
} else {
for (var i = 0; i < a.length; i++) {
var email = a[i].address;
c_email.busca(email, function(e_valid) {
c_sql.atualiza(id, email, e_valid)
})
}
}
})
)
})
}
}
//consulta_sql.js
var sql = require("seriate");
var search = function() {
var exec = 'select top 1 MyLink, id from sys_robo_fila where done is null';
sql.setDefaultConfig(config);
return sql.execute({
query: exec
});
}
var usado = function(id) {
var exec = "update sys_robo_fila set done = 'r' where id = " + id + "";
sql.setDefaultConfig(config);
return sql.execute({
query: exec
});
}
var update = function(id, email, valid) {
var exec = "update sys_robo_fila set email = '" + email + "' , validacao = '" + valid + "' , done = 'ok' where id = " + id + "";
sql.setDefaultConfig(config);
return sql.execute({
query: exec
});
}
module.exports = {
busca: search,
atualiza: update,
marca: usado
}
Any sugestions?

The call to c_sql.busca() is returning a Promise immediately and then continuing on to the next loop before then() is called, which is why they seem to run at the same time (they are actually just running very quickly, but before the Promise is resolved.
If you want this to run synchronously, one loop at a time, I would recommend using a recursive function to not start the loop again until the Promises have resolved.
let count = 0;
function doSomething() {
// this returns immediately, before .then() executes
return c_sql.busca()
.then(() => {
// do some more stuff after c_sql.busca() resolves
return c_sql.busca();
})
.then(() => {
// increment your count
count += 1
if (count<10) {
// less than 10 runs call the function again to start over
return doSomething();
}
});
}
This article might be helpful in understanding Promises: https://medium.com/#bluepnume/learn-about-promises-before-you-start-using-async-await-eb148164a9c8

Related

Cannot use “undefined” as a Firestore value

I am trying to retrieve data from my Firestore database using node.js, I want to collect a field from one Firestore query and pass the value into another Firestore query but I keep getting this error in my logs, the first Firestore query successfully retrieves data, but my problem is passing a value to the second query
Error: Value for argument "value" is not a valid query constraint. Cannot use "undefined" as a Firestore value. If you want to ignore undefined values, enable `ignoreUndefinedProperties`. at Object.validateUserInput (/workspace/node_modules/#google-cloud/firestore/build/src/serializer.js:271:19) at validateQueryValue (/workspace/node_modules/#google-cloud/firestore/build/src/reference.js:2048:18) at CollectionReference.where (/workspace/node_modules/#google-cloud/firestore/build/src/reference.js:988:9) at step2 (/workspace/index.js:74:43) at /workspace/index.js:65:17 at QuerySnapshot.forEach (/workspace/node_modules/#google-cloud/firestore/build/src/reference.js:748:22) at updateBets (/workspace/index.js:60:22) at processTicksAndRejections (internal/process/task_queues.js:97:5)
here is my code
async function updateBets() {
var marketRef = db.collection('matches');
var snapshot = await marketRef.where('matchStatus', '==', 'FINISHED').get();
if (snapshot.empty) {
console.log('No matching documents.');
return;
}
console.log('I found documents');
snapshot.forEach(doc => {
step2();
async function step2() {
var marketRef2 = db.collection('markets');
var snapshot2 = await marketRef2.where('marketId', '==', doc.data().matchId).get();
console.log(doc2.id, '=>', doc2.data());
snapshot2.forEach(doc2 => {
console.log(doc2.id, '=>', doc2.data());
if (doc2.data().marketTitleId == 'FULL_TIME_RESULT') {
var a = doc.data().homeTeamScore;
var b = doc.data().awayTeamScore;
var winnerIndex;
if (a > b) {
winnerIndex = 0;
var resultIndex = ['WINNER', 'LOSER', 'LOSER'];
var docName = `${doc.data().matchId}` + '000' + '1';
var sfRef = db.collection('markets').doc(docName);
batch5.update(sfRef, {
results: resultIndex
});
} else if (a == b) {
winnerIndex = 1;
var docName = `${doc.data().matchId}` + '000' + '1';
var resultIndex = ['LOSER', 'WINNER', 'LOSER'];
var sfRef = db.collection('markets').doc(docName);
batch5.update(sfRef, {
results: resultIndex
});
} else if (a < b) {
winnerIndex = 2;
var docName = `${doc.data().matchId}` + '000' + '1';
var resultIndex = ['LOSER', 'LOSER', 'WINNER'];
var sfRef = db.collection('markets').doc(docName);
batch5.update(sfRef, {
results: resultIndex
});
}
}
})
}
});
batch5.commit().then(() => {
console.log("im done with results");
}).catch((err) => {
console.log('Mac! there was an error with results: ', err);
});
}
You could try:
const data = doc.data();
const matchId = data.matchId;
and then put matchId into query.
Also log the "matchId" variable to see the value.

node wait for iteration to complete before callback

I have a lambda function in node.js to send a push notification.
In that function I need to iterate through my users sending a notification for each one prior to the callback.
Ideally I would like the iteration to perform in parallel.
What would be the best way to do this?
My code is currently as follows but it does not work as expected because the last user is not always the last to be handled:
var apnProvider = new apn.Provider(options);
var iterationComplete = false;
for (var j = 0; j < users.length; j++) {
if (j === (users.length - 1)) {
iterationComplete = true;
}
var deviceToken = users[j].user_device_token;
var deviceBadge = users[j].user_badge_count;
var notification = new apn.Notification();
notification.alert = message;
notification.contentAvailable = 1;
notification.topic = "com.example.Example";
apnProvider.send(notification, [deviceToken]).then((response) => {
if (iterationComplete) {
context.succeed(event);
}
});
}
Use Promise.all instead - map each user's associated apnProvider.send call to a Promise in an array, and when all Promises in the array are resolved, call the callback:
const apnProvider = new apn.Provider(options);
const userPromises = users.map((user) => {
const deviceToken = user.user_device_token;
const deviceBadge = user.user_badge_count;
const notification = new apn.Notification();
notification.alert = message;
notification.contentAvailable = 1;
notification.topic = "com.example.Example";
return apnProvider.send(notification, [deviceToken]);
})
Promise.all(userPromises)
.then(() => {
context.succeed(event);
})
.catch(() => {
// handle errors
});

NodeJS Script Runs on Local Machine but Not in Firebase Cloud Functions?

My script written in NodeJS connects to Firebase, checks my Firebase Database and even successfully sends notifications when results from my database return true... However, it only works when I run it from my local machine. I deploy it to Firebase and it will not work. Could someone please advise? Thank you.
I hate asking on here because I'm a newbie but I have spent hours tonight trying to find an answer...
INDEX.JS
// Firebase Functions
const functions = require('firebase-functions');
var admin = require("firebase-admin");
// Default admin firebase configuration
admin.initializeApp(functions.config().firebase);
// var serviceAccount = require("xxxxxx-80xxxxd-firebase-adminsdk- xxxxxxx.json");
var moment = require('moment');
var FCM = require('fcm-push');
var dateTime = require('node-datetime');
var serverKey = 'xxxxxxxxxxxxxpSELZBjQYwpZgmxxxxxxxxxxx';
var fcm = new FCM(serverKey);
//Initial function call:
exports.CheckDates = functions.https.onRequest((req, response) => {
// Get a database reference to our posts
var db = admin.database();
var ref = db.ref("records");
var userToken = '';
var itemExpires = '';
var itemName = '';
var reminded = '';
var itemCount = 0;
var counter = 1;
var itemFoundCount = 0;
var dt = dateTime.create();
var formatted = dt.format('m-d-Y');
ref.once("value", function (recordsSnapshot) {
recordsSnapshot.forEach(function (recordsSnapshot) {
var mainKey = recordsSnapshot.key;
recordsSnapshot.forEach(function (child) {
var key = child.key;
var value = child.val();
if (key == 'Account') {
userToken = value.userToken;
}
if (key == 'Items') {
recordsSnapshot.child("Items").forEach(function (itemsSnapshot) {
counter++;
if (itemFoundCount === 0) {
itemFoundCount = itemsSnapshot.numChildren();
}
var itemsChildkey = itemsSnapshot.key;
var itemsChildvalue = itemsSnapshot.val();
itemExpires = itemsChildvalue.itemExpires;
itemName = itemsChildvalue.itemName;
reminded = itemsChildvalue.reminded;
moment().format('YYYY-MM-DD');
var currentDate = moment();
var otherTime = moment(reminded);
if (typeof reminded !== 'undefined') {
if (currentDate.diff(otherTime, 'days') >= 30) {
if (currentDate.diff(itemExpires, 'days') <= 90) {
itemCount++;
console.log("Expire date is less than " +
currentDate + " by 90 days = " + (currentDate.diff(otherTime, 'days') <=
90));
db.ref("records/" + mainKey + "/Items/" +
itemsChildkey + '/reminded').set(formatted);
}
}
} else {
itemCount++;
db.ref("records/" + mainKey + "/Items/" + itemsChildkey +
`enter code here`'/reminded').set(formatted);
}
if (counter == itemFoundCount && itemCount > 0) {
console.log(itemFoundCount);
var message = {
to: userToken, // required fill with device token or
topics
notification: {
title: 'Item Expire Notification',
body: itemCount + ' is about to expire.'
}
};
//callback style
fcm.send(message, function (err, response) {
if (err) {
console.log("Something has gone wrong!");
} else {
console.log("Successfully sent with response: ",
response);
}
});
itemCount = 0;
itemFoundCount = 0;
counter = 1;
}
});
}
});
});
});
response.send(200, "ok");
}) // END exports.CheckDates
Obviously, I remove the Exports.CheckDates lines when I run it locally but it wasn't showing up at all without the exports on Firebase's console. It returns warnings locally and on Firebase console but it works on one and not the other.
Please disregard this question. The script adds the date that the last time a notification was sent to the item's "Reminded" key in the database... this prevents notifications from going out every day for the same thing..
It ran on my local machine, and did it's job but I couldn't figure out why it wouldn't run again... well, ha! At least we know that part works.

Proper way to make callbacks async by wrapping them using `co`?

It is 2016, Node has had nearly full ES6 support since v4, and Promises have been around since 0.12. It's time to leave callbacks in the dust IMO.
I'm working on a commander.js-based CLI util which leverages a lot of async operations - http requests and user input. I want to wrap the Commander actions in async functions so that they can be treated as promises, and also to support generators (useful for the co-prompt library I'm using for user input).
I've tried wrapping the CB with co in two ways:
1)
program.command('myCmd')
.action(program => co(function* (program) {...})
.catch(err => console.log(err.stack)) );
and
2) program.command('myCmd').action(co.wrap(function* (program) { .. }));
The problem with 1) is that the program parameter isn't passed
The problem with 2) is that errors are swallowed...
I'd really like to get this working as it yields much nicer code in my use case - involving a lot of http requests and also waiting for user input using the co-prompt library..
Is it a better option altogether perhaps to wrap program.Command.prototype.action somehow?
thanks!
I've used a bespoke version of something like co to get a db.exec function which uses yield to do database request. You can pass parameters into a generator function (I pass in a connection object - see the comment where I do it).
Here is by db.exec function that is very similar to what co does
exec(generator) {
var self = this;
var it;
debug('In db.exec iterator');
return new Promise((accept,reject) => {
debug('In db.exec Promise');
var myConnection;
var onResult = lastPromiseResult => {
debug('In db.exec onResult');
var obj = it.next(lastPromiseResult);
if (!obj.done) {
debug('db.exec Iterator NOT done yet');
obj.value.then(onResult,reject);
} else {
if (myConnection) {
myConnection.release();
debug('db.exec released connection');
}
accept(obj.value);
debug('db.exec Promise Resolved with value %d',obj.value);
}
};
self._connection().then(connection => {
debug('db.exec got a connection');
myConnection = connection;
it = generator(connection); //This passes it into the generator
onResult(); //starts the generator
}).catch(error => {
logger('database', 'Exec Function Error: ' + error.message);
reject(error);
});
});
}
the connection object also wraps by database connection object and provides a generator function ability to process the rows of the results from the database, but I won't post that here (although the example below is using it to process the rows).
Here is an example of using the exec function to run a sequence of sql
db.exec(function*(connection) {
if (params.name === ADMIN_USER) {
debug('Admin Logon');
user.name = ADMIN_DISPLAY;
user.keys = 'A';
user.uid = 0;
let sql = 'SELECT passwordsalt FROM Admin WHERE AdminID = 0';
connection.request(sql);
yield connection.execSql(function*() {
let row = yield;
if (row) {
user.nopass = (row[0].value === null);
} else {
user.nopass = false;
}
debug('Admin Password bypass ' + user.nopass.toString());
});
} else {
debug('Normal User Logon');
let sql = `SELECT u.UserID,PasswordSalt,DisplayName,AccessKey,l.LogID FROM Users u
LEFT JOIN UserLog l ON u.userID = l.userID AND DATEDIFF(D,l.LogDate,GETDATE()) = 0
WHERE u.UserName = #username`;
let request = connection.request(sql);
request.addParameter('username',db.TYPES.NVarChar,params.name);
let count = yield connection.execSql(function*() {
let row = yield;
if (row) {
user.uid = row[0].value;
user.name = row[2].value;
user.keys = (row[3].value === null) ? '' : row[3].value;
user.nopass = (row[1].value === null) ;
user.lid = (row[4].value === null) ? 0 : row[4].value;
debug('Found User with uid = %d and lid = %d, keys = %s',
user.uid, user.lid, user.keys);
}
});
if (count === 0) {
debug('Not Found User');
// couldn't find name in database
reply(false,false);
return;
}
}
if (!user.nopass) {
debug('Need a Password');
//user has a password so we must check it
passGood = false; //assume false as we go into this
let request = connection.request('CheckPassword');
request.addParameter('UserID',db.TYPES.Int,user.uid);
request.addParameter('password',db.TYPES.VarChar,params.password);
yield connection.callProcedure(function*() {
let row = yield;
if (row) {
//got a valid row means we have a valid password
passGood = true;
}
});
} else {
passGood = true;
}
if (!passGood) {
debug('Not a Good Pasword');
reply(false,true);
} else {
if (user.uid !== 0 && user.lid === 0) {
let sql = `INSERT INTO UserLog(UserID,LogDate,TimeOn,UserName) OUTPUT INSERTED.logID
VALUES(#uid,GETDATE(),GETDATE(),#username)`;
let request = connection.request(sql);
request.addParameter('uid',db.TYPES.Int,user.uid);
request.addParameter('username',db.TYPES.NVarChar,user.name);
yield connection.execSql(function*() {
let row = yield;
if (row) {
user.lid = row[0].value;
debug('Users Log Entry = %d',user.lid);
}
});
}
reply(true,user);
}
})
.catch((err) => {
logger('database','Error on logon: ' + err.message);
reply(false,false);
});
});
There is a quite simple way to do async function in Commander.js
async function run() {
/* code goes here */
}
program
.command('gettime')
.action(run);
program.parse(process.argv);

Synchronous Call : Call 2nd function after 1st function is executed completely

I recently stating coding in node.js and might be a very simple question.
Trying to write a XML parser/validator to validate xml schema and values against values/ xpath stored in an excel sheet.
Now once the validation function is complete I want to call a printResult function to print final result. However if I try to call the function immediately after the first function .. its printing variables initial values and if called within the first which is iterating though the number of xpaths present in excel sheet and printing result with increments.
var mocha = require('mocha');
var assert = require('chai').assert;
var fs = require('fs');
var parseString = require('xml2js').parseString;
var xpath = require('xpath');
var dom = require('xmldom').DOMParser;
var XLSX = require('xlsx');
var Excel = require("exceljs");
var should = require('chai').should();
var HashMap = require('hashmap');
var colors = require('colors');
require('/xmlValidator/dbConnect.js');
var map = new HashMap();
var elementMap = new HashMap();
var resultValue;
//console.log('hello'.green);
map.set("PASS", 0);
map.set("FAIL", 0);
map.set("INVALID_PATH", 0);
function computeResult(elementPath, result) {
var pass = map.get("PASS");
var fail = map.get("FAIL");
var invalidPath = map.get("INVALID_PATH");
elementMap.set(elementPath, result);
if (result == "PASS") {
pass++;
map.set("PASS", pass);
} else if (result == "FAIL") {
fail++;
map.set("FAIL", fail);
} else {
invalidPath++;
map.set("INVALID_PATH", invalidPath)
}
printResult();
}
function printResult() {
var pass = map.get("PASS");
var fail = map.get("FAIL");
var invalidPath = map.get("INVALID_PATH");
console.log(("PASS Count :" + pass).green);
console.log(("FAIL Count :" + fail).red);
console.log(("Inavlid Path :" + invalidPath).yellow);
elementMap.forEach(function(value, key) {
if (value == "INVALID_PATH")
console.log((key + ":" + value).yellow);
else if (value == "FAIL")
console.log((key + ":" + value).red);
else
console.log(key + ":" + value);
});
}
var workbook = new Excel.Workbook();
workbook.xlsx.readFile('utils/' + process.argv[2])
.then(function() {
var worksheet = workbook.getWorksheet(1);
worksheet.eachRow(function(row, rowNumber) {
//console.log(rowNumber);
var row = worksheet.getRow(rowNumber);
var dataPath1 = row.getCell("A").value;
var dataPath2 = row.getCell("B").value;
var dataPath = dataPath1 + dataPath2;
//console.log(dataPath);
var dataValue = row.getCell("D").value;
var flag = row.getCell("E").value;
//console.log(flag)
//console.log(dataValue);
if (!flag)
validate(dataPath, dataValue, rowNumber);
//else console.log("NOT EXECUTED" + rowNumber)
});
})
function validate(dataPath, dataValue, rowNumber) {
var fail = 0;
fs.readFile('utils/' + process.argv[3], 'utf8', function(err, data) {
if (err) {
console.log("ERROR ERROR ERROR ERROR ");
return console.log(err);
}
var doc = new dom().parseFromString(data);
var subId = String(xpath.select1(dataPath, doc));
if (subId == "undefined") {
/*console.log('undefined caught');
console.log("row number :" + rowNumber);*/
var resultValue = "INVALID_PATH";
computeResult(dataPath, resultValue);
} else {
var subId = xpath.select1(dataPath, doc);
var value = subId.lastChild.data;
/*console.log("row number :" + rowNumber);
console.log("actual value: " + value);
console.log("expected value:" + dataValue );*/
if (dataValue == null) {
assert.notEqual(value, dataValue, "value not found");
resultValue = "PASS";
computeResult(dataPath, resultValue);
} else {
if (value == dataValue)
resultValue = "PASS";
else resultValue = "FAIL";
computeResult(dataPath, resultValue);
}
}
});
}
In the code above i want to call printResult() function after validate function is completely executed (workbook.xlsx.readFile)
Can some one please help me out how to use done () function or make sync call ?
If fs.readFileAsync('utils/' + process.argv[3], 'utf8') can be executed once, then validate() will be completely synchronous and calling printResult() after the verification loop will be trivial.
In the main routine, you can simply aggregate two promises ...
var promise1 = workbook.xlsx.readFile();
var promise2 = fs.readFileAsync(); // requires `fs` to be promisified.
... before embarking on the verification loop.
Promise.all([promise1, promise2]).spread(/* verify here */);
Also a whole bunch of tidying can be considered, in particular :
establishing "PASS", "FAIL" and "INVALID_PATH" as constants to avoid lots of repetitive string creation,
using js plain objects in lieu of hashmap,
building map from elementMap inside the print function.
having validate() return its result and building elementMap in the main routine
Here's the whole thing, about as tight as I can get it. I may have made a few assumptions but hopefully not too many bad ones ...
var mocha = require('mocha');
var assert = require('chai').assert;
var Promise = require("bluebird");
var fs = Promise.promisifyAll(require("fs")); // allow Bluebird to take the pain out of promisification.
var parseString = require('xml2js').parseString;
var xpath = require('xpath');
var dom = require('xmldom').DOMParser;
var XLSX = require('xlsx');
var Excel = require("exceljs");
var should = require('chai').should();
// var HashMap = require('hashmap');
var colors = require('colors');
require('/xmlValidator/dbConnect.js');
const PASS = "PASS";
const FAIL = "FAIL";
const INVALID_PATH = "INVALID_PATH";
function printResult(elementMap) {
var key, result,
map = { PASS: 0, FAIL: 0, INVALID_PATH: 0 },
colorNames = { PASS: 'black', FAIL: 'red', INVALID_PATH: 'yellow' };
for(key in elementMap) {
result = elementMap[key];
map[(result === PASS || result === FAIL) ? result : INVALID_PATH] += 1;
console.log((key + ": " + result)[colorNames[result] || 'black']); // presumably colors can be applied with associative syntax? If so, then the code can be very concise.
}
console.log(("PASS Count: " + map.PASS)[colorNames.PASS]);
console.log(("FAIL Count: " + map.FAIL)[colorNames.FAIL]);
console.log(("Inavlid Path: " + map.INVALID_PATH)[colorNames.INVALID_PATH]);
}
function validate(doc, dataPath, dataValue) {
var subId = xpath.select1(dataPath, doc),
value = subId.lastChild.data,
result;
if (String(subId) == "undefined") {
result = INVALID_PATH;
} else {
if (dataValue === null) {
assert.notEqual(value, dataValue, "value not found"); // not too sure what this does
result = PASS;
} else {
result = (value === dataValue) ? PASS : FAIL;
}
}
return result;
}
//Main routine
var workbook = new Excel.Workbook();
var promise1 = workbook.xlsx.readFile('utils/' + process.argv[2]); // from the question, workbook.xlsx.readFile() appears to return a promise.
var promise2 = fs.readFileAsync('utils/' + process.argv[3], 'utf8');
Promise.all([promise1, promise2]).spread(function(data2, data3) {
var worksheet = workbook.getWorksheet(1),
doc = new dom().parseFromString(data3),
elementMap = {};
worksheet.eachRow(function(row, rowNumber) {
// var row = worksheet.getRow(rowNumber); // row is already a formal variable ???
var dataPath, dataValue;
if (!row.getCell('E').value)
dataPath = row.getCell('A').value + row.getCell('B').value;
dataValue = row.getCell('D').value;
elementMap[dataPath] = validate(doc, dataPath, dataValue);
});
printResult(elementMap);
});
Untested so may not run but at least you can raid the code for ideas.

Resources