I have an issue with handling parallel requests with cloud functions.
My scenario is to select a driver from the db and update its status.
I do check for that status property before updating it, but when I send multiple requests (database on create triggers to be specific) within a second it doesn't seem to read the updated status property. And it always updates with the information of the last request. I also have noticed that sometimes the requests are processed altogether.
What can I do to fix these issues?
index.js
const db = app.database();
const TripManagementUtil = require('./utils').TripManagementUtil;
exports.triggerNotifications = functions.database.ref('/Trip/{pushId}').onCreate( (snapshot, context) =>
{
var newTrip = snapshot.val();
var tripKey = context.params.pushId;
var tripManagementUtil = new TripManagementUtil();
tripManagementUtil.searchDrivers(tripKey, newTrip, db);
});
utils.js
searchDrivers(tripKey, trip, db){
const results = [];
var lat = trip.pickupLocation.lat, long = trip.pickupLocation.lng;
var vehicleTypeID = trip.vehicleTypeID;
var alreadyAssigned = trip.alreadyAssigned;
var self = this;
if(alreadyAssigned == null || alreadyAssigned == 'undefined'){
alreadyAssigned = [];
}
const geofireQuery = new GeoFire(db.ref('vehicleLocation').child(vehicleTypeID + "")).query({
center: [lat, long],
radius: constants.searchRadius
})
.on('key_entered', (key, coords, distance) => {
if(alreadyAssigned.indexOf(key) == -1){
var result = {
driverID: key,
distance: distance
}
results.push(result);
}
});
setTimeout(() => {
geofireQuery.cancel();
if (results.length === 0) {
self.noDriversHandler(alreadyAssigned, tripKey, db);
} else {
results.sort((a, b) => a.distance - b.distance);
var driversAvailable = false;
var index = 0;
function checkDriver(){
db.ref().child("driver").child("available").child(results[index].driverID).once('value').then(function(vehicleSnap){
var vehicle = vehicleSnap.val();
if(!driversAvailable){
if(vehicle != null && vehicle.vehicleTypeID == vehicleTypeID
&& (vehicle.tripStatus != TripVehicleActionEnum.DriverConfirmed && vehicle.tripStatus != TripVehicleActionEnum.VehicleAssigned)
&& alreadyAssigned.indexOf(vehicle.driverID +"") === -1){
driversAvailable = true;
self.driverExistsHandler(trip, tripKey, alreadyAssigned, vehicle, db);
}
if(!driversAvailable && index + 1 == results.length){
self.noDriversHandler(alreadyAssigned, tripKey, db);
}
index++;
}
else{
index++;
checkDriver();
}
});
}
checkDriver();
}
}, 1500);
}
To write data to the database where the value is based on an existing value, you'll want to use Firebase Realtime Database transactions. For more on this, and examples, see save data transactionally in the Firebase documentation.
Related
This code works well.
I have a list of available campaigns and another list of active users. For each user, I need to check which campaigns have been shown before. I would like to know if it is possible to dispense with my promise to write undefined in my return matrix.
Is there a way to cancel a promise?
In this case, if there is already a record in the redis node, the answer will be equal to one and I would like to cancel the promise to avoid undefined return.
How to fixed this?
VerificarCampanhaEnvio : async function(user,campanhas){
const CampanhaUsers = [];
let sismember = util.promisify(cliente_redis.sismember).bind(cliente_redis);
return Promise.all(campanhas.map(async(campanha) =>{
return sismember("subscription:"+user.id,campanha.id_campanha).then((response)=>{
console.log('response',user.id,response,campanha.id_campanha);
if(response != 1){
data = {};
data.user = user;
data.campanha = campanha;
return data;
}
});
})).then((response) => {
return response;
});
},
FireMessage : function(){
Sucesso = function(data,campanhas){
if(campanhas.length > 0){
self.GetInscritosAtivos(function(users){
if(users !== false){
var contador = 1;
users.map(async(user) =>{
var finalArray;
console.log('user.id',user.id);
finalArray = await self.VerificarCampanhaEnvio(user,campanhas);
var filtered = finalArray.filter(function(el){ return el != null; });
if(filtered.length > 0) self.EnviarMensagem(filtered[0]);
});
}else console.log('fnpush:FireMessage','Nao existe subscricoes ativas');
});
}else console.log('fnpush:FireMessage','Nao existem campanhas de push disponiveis no
sistema.');
}
data = {};
data.zona = {};
data.zona.tipo = 'push';
data.sucesso = Sucesso;
data.falha = fnCampanha.CampanhaInexistente;
fnCampanha.GetCampanhas(data);
}
I have a code where I need to upload bulk records like (50,000) from an CSV. From angular 6, we are making an restapi call from where we are passing the formdata to node server. In node portion we are looping that set of records(50,000) and we are uploading it in to our backend by splitting it like 1500. Records will be uploaded for every consecutive 1500.
So In our local it is working completely fine & we have tested uploading same 50000 records. But we have moved the same code to our SIT environment where we are facing the error. After uploading certain amount of record (20000 records), again its starts uploading from first record. This is some weird behaviour, which we couldnt test it in local. Can anyone please please suggest What I need to do?
var bulkUploadData = async function (jsonblob, res) {
var payloadjson = [];
var jsonResponse = {};
try {
for (let i = 0; i < jsonblob.length; i++) {
if(jsonblob[i].PN != null){
var JFB = {}
var arrayy = [];
if (jsonblob[i].Otyp != "L") {
JFB.srv = jsonblob[i].SA;
JFB.ptype = jsonblob[i].PTy;
JFB.ms = jsonblob[i].PN;
var a = JSON.stringify(JFB)
payloadjson.push(a);
}
else {
JFB.ms = jsonblob[i].PN
JFB.srv = jsonblob[i].SA;
JFB.ptype = jsonblob[i].PTy;
var a = JSON.stringify(JFB)
payloadjson.push(a);
}
if ((payloadjson.length % 1500) == 0) {
console.log("Inside first loop---------counter--------- ", i );
var result = await update.invokeSDK("fnname", payloadjson, res)
payloadjson = [];
await sleepTime(20)
console.log("-----sleeped for 20ms----- ",i)
}
if (jsonblob.length == i + 1 && payloadjson.length > 0) {
var result = await update.invokeSDK("fnname", payloadjson, res)
payloadjson = [];
}
}
console.log("FRKNG Length _________i: ",i);
}
jsonResponse = { "failedRecords": "fail" }
return jsonResponse;
} catch (err) {
console.log('error:----', err);
return err;
}
}
My script written in NodeJS connects to Firebase, checks my Firebase Database and even successfully sends notifications when results from my database return true... However, it only works when I run it from my local machine. I deploy it to Firebase and it will not work. Could someone please advise? Thank you.
I hate asking on here because I'm a newbie but I have spent hours tonight trying to find an answer...
INDEX.JS
// Firebase Functions
const functions = require('firebase-functions');
var admin = require("firebase-admin");
// Default admin firebase configuration
admin.initializeApp(functions.config().firebase);
// var serviceAccount = require("xxxxxx-80xxxxd-firebase-adminsdk- xxxxxxx.json");
var moment = require('moment');
var FCM = require('fcm-push');
var dateTime = require('node-datetime');
var serverKey = 'xxxxxxxxxxxxxpSELZBjQYwpZgmxxxxxxxxxxx';
var fcm = new FCM(serverKey);
//Initial function call:
exports.CheckDates = functions.https.onRequest((req, response) => {
// Get a database reference to our posts
var db = admin.database();
var ref = db.ref("records");
var userToken = '';
var itemExpires = '';
var itemName = '';
var reminded = '';
var itemCount = 0;
var counter = 1;
var itemFoundCount = 0;
var dt = dateTime.create();
var formatted = dt.format('m-d-Y');
ref.once("value", function (recordsSnapshot) {
recordsSnapshot.forEach(function (recordsSnapshot) {
var mainKey = recordsSnapshot.key;
recordsSnapshot.forEach(function (child) {
var key = child.key;
var value = child.val();
if (key == 'Account') {
userToken = value.userToken;
}
if (key == 'Items') {
recordsSnapshot.child("Items").forEach(function (itemsSnapshot) {
counter++;
if (itemFoundCount === 0) {
itemFoundCount = itemsSnapshot.numChildren();
}
var itemsChildkey = itemsSnapshot.key;
var itemsChildvalue = itemsSnapshot.val();
itemExpires = itemsChildvalue.itemExpires;
itemName = itemsChildvalue.itemName;
reminded = itemsChildvalue.reminded;
moment().format('YYYY-MM-DD');
var currentDate = moment();
var otherTime = moment(reminded);
if (typeof reminded !== 'undefined') {
if (currentDate.diff(otherTime, 'days') >= 30) {
if (currentDate.diff(itemExpires, 'days') <= 90) {
itemCount++;
console.log("Expire date is less than " +
currentDate + " by 90 days = " + (currentDate.diff(otherTime, 'days') <=
90));
db.ref("records/" + mainKey + "/Items/" +
itemsChildkey + '/reminded').set(formatted);
}
}
} else {
itemCount++;
db.ref("records/" + mainKey + "/Items/" + itemsChildkey +
`enter code here`'/reminded').set(formatted);
}
if (counter == itemFoundCount && itemCount > 0) {
console.log(itemFoundCount);
var message = {
to: userToken, // required fill with device token or
topics
notification: {
title: 'Item Expire Notification',
body: itemCount + ' is about to expire.'
}
};
//callback style
fcm.send(message, function (err, response) {
if (err) {
console.log("Something has gone wrong!");
} else {
console.log("Successfully sent with response: ",
response);
}
});
itemCount = 0;
itemFoundCount = 0;
counter = 1;
}
});
}
});
});
});
response.send(200, "ok");
}) // END exports.CheckDates
Obviously, I remove the Exports.CheckDates lines when I run it locally but it wasn't showing up at all without the exports on Firebase's console. It returns warnings locally and on Firebase console but it works on one and not the other.
Please disregard this question. The script adds the date that the last time a notification was sent to the item's "Reminded" key in the database... this prevents notifications from going out every day for the same thing..
It ran on my local machine, and did it's job but I couldn't figure out why it wouldn't run again... well, ha! At least we know that part works.
It is 2016, Node has had nearly full ES6 support since v4, and Promises have been around since 0.12. It's time to leave callbacks in the dust IMO.
I'm working on a commander.js-based CLI util which leverages a lot of async operations - http requests and user input. I want to wrap the Commander actions in async functions so that they can be treated as promises, and also to support generators (useful for the co-prompt library I'm using for user input).
I've tried wrapping the CB with co in two ways:
1)
program.command('myCmd')
.action(program => co(function* (program) {...})
.catch(err => console.log(err.stack)) );
and
2) program.command('myCmd').action(co.wrap(function* (program) { .. }));
The problem with 1) is that the program parameter isn't passed
The problem with 2) is that errors are swallowed...
I'd really like to get this working as it yields much nicer code in my use case - involving a lot of http requests and also waiting for user input using the co-prompt library..
Is it a better option altogether perhaps to wrap program.Command.prototype.action somehow?
thanks!
I've used a bespoke version of something like co to get a db.exec function which uses yield to do database request. You can pass parameters into a generator function (I pass in a connection object - see the comment where I do it).
Here is by db.exec function that is very similar to what co does
exec(generator) {
var self = this;
var it;
debug('In db.exec iterator');
return new Promise((accept,reject) => {
debug('In db.exec Promise');
var myConnection;
var onResult = lastPromiseResult => {
debug('In db.exec onResult');
var obj = it.next(lastPromiseResult);
if (!obj.done) {
debug('db.exec Iterator NOT done yet');
obj.value.then(onResult,reject);
} else {
if (myConnection) {
myConnection.release();
debug('db.exec released connection');
}
accept(obj.value);
debug('db.exec Promise Resolved with value %d',obj.value);
}
};
self._connection().then(connection => {
debug('db.exec got a connection');
myConnection = connection;
it = generator(connection); //This passes it into the generator
onResult(); //starts the generator
}).catch(error => {
logger('database', 'Exec Function Error: ' + error.message);
reject(error);
});
});
}
the connection object also wraps by database connection object and provides a generator function ability to process the rows of the results from the database, but I won't post that here (although the example below is using it to process the rows).
Here is an example of using the exec function to run a sequence of sql
db.exec(function*(connection) {
if (params.name === ADMIN_USER) {
debug('Admin Logon');
user.name = ADMIN_DISPLAY;
user.keys = 'A';
user.uid = 0;
let sql = 'SELECT passwordsalt FROM Admin WHERE AdminID = 0';
connection.request(sql);
yield connection.execSql(function*() {
let row = yield;
if (row) {
user.nopass = (row[0].value === null);
} else {
user.nopass = false;
}
debug('Admin Password bypass ' + user.nopass.toString());
});
} else {
debug('Normal User Logon');
let sql = `SELECT u.UserID,PasswordSalt,DisplayName,AccessKey,l.LogID FROM Users u
LEFT JOIN UserLog l ON u.userID = l.userID AND DATEDIFF(D,l.LogDate,GETDATE()) = 0
WHERE u.UserName = #username`;
let request = connection.request(sql);
request.addParameter('username',db.TYPES.NVarChar,params.name);
let count = yield connection.execSql(function*() {
let row = yield;
if (row) {
user.uid = row[0].value;
user.name = row[2].value;
user.keys = (row[3].value === null) ? '' : row[3].value;
user.nopass = (row[1].value === null) ;
user.lid = (row[4].value === null) ? 0 : row[4].value;
debug('Found User with uid = %d and lid = %d, keys = %s',
user.uid, user.lid, user.keys);
}
});
if (count === 0) {
debug('Not Found User');
// couldn't find name in database
reply(false,false);
return;
}
}
if (!user.nopass) {
debug('Need a Password');
//user has a password so we must check it
passGood = false; //assume false as we go into this
let request = connection.request('CheckPassword');
request.addParameter('UserID',db.TYPES.Int,user.uid);
request.addParameter('password',db.TYPES.VarChar,params.password);
yield connection.callProcedure(function*() {
let row = yield;
if (row) {
//got a valid row means we have a valid password
passGood = true;
}
});
} else {
passGood = true;
}
if (!passGood) {
debug('Not a Good Pasword');
reply(false,true);
} else {
if (user.uid !== 0 && user.lid === 0) {
let sql = `INSERT INTO UserLog(UserID,LogDate,TimeOn,UserName) OUTPUT INSERTED.logID
VALUES(#uid,GETDATE(),GETDATE(),#username)`;
let request = connection.request(sql);
request.addParameter('uid',db.TYPES.Int,user.uid);
request.addParameter('username',db.TYPES.NVarChar,user.name);
yield connection.execSql(function*() {
let row = yield;
if (row) {
user.lid = row[0].value;
debug('Users Log Entry = %d',user.lid);
}
});
}
reply(true,user);
}
})
.catch((err) => {
logger('database','Error on logon: ' + err.message);
reply(false,false);
});
});
There is a quite simple way to do async function in Commander.js
async function run() {
/* code goes here */
}
program
.command('gettime')
.action(run);
program.parse(process.argv);
I Have to Insert about 10,00000 documents in mongodb using nodejs.
I'm generating these documents using a for loop storing them into an array before finally inserting them into mongodb.
var codeArray = new Array();
for (var i = 0; i<1000000; i++){
var token = strNpm.generate();
var now = moment().format('YYYYMMDD hhmmss');
var doc1 = {id:token,
Discount_strId:"pending",
Promotion_strCode:token,
Promotion_strStatus:"I",
Promotion_dtmGeneratedDate:now,
User_strLogin:"test",
Promotion_strMode:"S",
Promotion_dtmValidFrom:"pending",
Promotion_dtmValidTill:"pending",
LastModified_dtmStamp:now
};
codeArray.push(doc1);
db.collection('ClPromoCodeMaster').insert(codeArray, function (err, result) {
if (err){
console.log(err);
}else{
console.log('Inserted Records - ', result.ops.length);
}
});
The problem I'm facing is mongo has an inserting limit of 16mb, so I can't insert the entire array at once.
Please suggest most optimum solutions.
The main problem is in the request size and not the document size, but it amounts to the same limitation. Bulk operations and the async library with async.whilst will handle this:
var bulk = db.collection('ClPromoCodeMaster').initializeOrderedBulkOp(),
i = 0;
async.whilst(
function() { return i < 1000000; },
function(callback) {
var token = strNpm.generate();
var now = moment().format('YYYYMMDD hhmmss');
var doc = {
id:token,
Discount_strId:"pending",
Promotion_strCode:token,
Promotion_strStatus:"I",
Promotion_dtmGeneratedDate:now,
User_strLogin:"test",
Promotion_strMode:"S",
Promotion_dtmValidFrom:"pending",
Promotion_dtmValidTill:"pending",
LastModified_dtmStamp:now
};
bulk.insert(doc);
i++;
// Drain every 1000
if ( i % 1000 == 0 ) {
bulk.execute(function(err,response){
bulk = db.collection('ClPromoCodeMaster').initializeOrderedBulkOp();
callback(err);
});
} else {
callback();
}
},
function(err) {
if (err) throw err;
console.log("done");
}
);
I should note that regardless there is an internal limit on bulk operations to 1000 operations per batch. You can submit in larger sizes, but the driver is just going to break these up and still submit in batches of 1000.
The 1000 is a good number to stay at though, since it is already in line with how the request will be handled, as well as being a reasonable number of things to hold in memory before draining the request queue and sending to the server.
For inserting millions of record at a time, Create node.js child process fork with MongoDb bulk api.
Child Process Creation:(index.js)
const {fork} = require("child_process");
let counter = 1;
function createProcess(data){
const worker = fork("./dbOperation");
worker.send(data);
worker.on("message", (msg) => {
console.log("Worker Message :",counter, msg);
counter++;
})
}
function bulkSaveUser(records) {
const singleBatchCount = 10000; // Save 10,000 records per hit
const noOfProcess = Math.ceil(records/singleBatchCount);
let data = {};
console.log("No of Process :", noOfProcess);
for(let index = 1; index <= noOfProcess; index++) {
data.startCount = (index == 1) ? index : (((index - 1) * singleBatchCount) + 1);
data.endCount = index * singleBatchCount;
createProcess(data);
}
}
bulkSaveUser(1500000);
DB Operation (dbOperation.js)
const MongoClient = require('mongodb').MongoClient;
// Collection Name
const collectionName = "";
// DB Connection String
const connString = "";
process.on("message", (msg) => {
console.log("Initialize Child Process", msg)
const {startCount, endCount} = msg;
inputStudents(startCount, endCount);
});
function initConnection() {
return new Promise(function(r, e) {
MongoClient.connect(connString, function(err, db) {
if (err) e(err)
r(db);
});
});
}
function inputStudents(startCount, endCount) {
let bulkData = [];
for(let index = startCount; index <= endCount; index++ ){
var types = ['exam', 'quiz', 'homework', 'homework'];
let scores = []
// and each class has 4 grades
for (j = 0; j < 4; j++) {
scores.push({'type':types[j],'score':Math.random()*100});
}
// there are 500 different classes that they can take
class_id = Math.floor(Math.random()*501); // get a class id between 0 and 500
record = {'student_id':index, 'scores':scores, 'class_id':class_id};
bulkData.push({ insertOne : { "document" : record } })
}
initConnection()
.then((db) => {
const studentDb = db.db("student");
const collection = studentDb.collection(colName)
console.log("Bulk Data :", bulkData.length);
collection.bulkWrite(bulkData, function(err, res) {
if (err) throw err;
//console.log("Connected Successfully",res);
process.send("Saved Successfully");
db.close();
});
})
.catch((err) => { console.log("Err :", err) });
}
Sample project to insert millions of record in mongodb using child process fork