I don't quite have the promises down I make. Anyway, I am reading a list of rows in a mysql table, then upon writing into Firestore I want to update a count of a "related" record. Basically aggregate counts.
Here is the start:
request(options, function (error, response, body) {
if (!error && response.statusCode == 200) {
var data = JSON.parse(body);
//console.log(data);
var allObjects = data.resource;
var obj = {};
for (j = 0; j < allObjects.length; j++) {
var records = [];
var record = allObjects[j];
//FireStore
var db = admin.firestore();
var docRef = db.collection(toCollection).doc(record.nfCode);
var rec = docRef.set(record);
addCounts(docRef);
}
res.render('index', {title: 'Load Table'});
offset = offset + limit
} else {
return res.send(body);//
//res.render('index', { title: 'Error' });
}
});
Then here is addCount:
function addCounts(docRef) {
// In a transaction, update the aggregate totals
var db = admin.firestore();
return db.runTransaction(transaction => {
transaction.get(docRef).then(res => {
var brandRef = db.collection('brands').doc(res.data().brandNFCode);
var transaction = db.runTransaction(t => {
return t.get(brandRef)
.then(doc => {
if (res.data().glutenFreeYN == "Y") {
var glutenFreeCount = doc.data().glutenFreeCount + 1;
var setWithOptions = transaction.set(brand, {
glutenFreeProductCount: glutenFreeProductCount
}, { merge: true });
return setWithOptions;
}
});
})
.then(result => {
console.log('Transaction success', result);
// return nil
})
.catch(err => {
// console.log('Transaction failure:', err);
return nil
});
})
})
}
Is there a better way to do this? And where is my error coming from?
Auth error:Error: socket hang up
(node:90050) UnhandledPromiseRejectionWarning: Unhandled promise rejection (rejection id: 2): Error: Getting metadata from plugin failed with error: socket hang up
There are a few things wrong with your addCounts function that may be causing this:
function addCounts(docRef) {
// In a transaction, update the aggregate totals
var db = admin.firestore();
return db.runTransaction(transaction => {
return transaction.get(docRef).then(res => {
if(res.data().glutenFreeYN == "Y"){
var glutenFreeProductCount = res.data().glutenFreeProductCount + 1;
}
if(res.data().vegetarianYN == "Y"){
var vegetarianProductCount = res.data().vegetarianProductCount + 1;
}
if(res.data().dairyFreeYN == "Y"){
var dairyFreeProductCount = res.data().dairyFreeProductCount + 1;
}
if(res.data().organicYN == "Y"){
var organicProductCount = res.data().organicProductCount + 1;
}
// LOOK HERE!!! You had brand.set()
// this version uses the transaction
var setWithOptions = transaction.set(brand, {
glutenFreeProductCount: glutenFreeProductCount,
organicProductCount: organicProductCount,
vegetarianProductCount: vegetarianProductCount,
dairyFreeProductCount: dairyFreeProductCount
}, { merge: true });
// LOOK HERE!! Make sure to return the
// set operation
return setWithOptions;
});
// I removed the .catch() here as it will obscure
// errors. You can catch the result of addCounts.
})
};
Was running into this issue and spent over an hour trying to figure it out. Make sure your system time is synced properly. My time was correct, but my time zone wasn't so the system UTC time was out of sync.
To correct on Windows 10, go to Settings -> Time & Language -> Date and Time -> Sync Now
Related
I am new to using cloud functions for Firebase with an http triggered function and I am confused on how to properly terminate the function. I'm not sure if I should be using res.sendStatus, returning a promise, or both.
The goal of my function is to loop through several documents in the collection 'communities'. Each community has a collection of documents where I query the document with the highest value of 'hotScore'. I then send an iOS push notification containing that document to a topic (all users in that given community).
Unfortunately, I am getting several errors when the code is run such as Error [ERR_HTTP_HEADERS_SENT]: Cannot set headers after they are sent to the client and Unhandled rejection. I'm pretty such this is due to my negligence in handling function termination, although I have been confused by the online resources I have looked at so far. Would someone mind taking a look at my code/pointing me in the right direction? Thank you so much!
exports.sendNotificationTrendingPost = functions.https.onRequest(async (req, res) => {
//Get communities collection from Firestore
return admin.firestore().collection('communities').get().then((communities) => {
var communityPromises = [];
//Loop through each community
communities.forEach((community) => {
let communityID = community.get('communityID');
let communityName = community.get('name');
//Get the post with the highest hotScore
let communityPromise = admin.firestore().collection('communities').doc(communityID).collection('posts').orderBy('hotScore', 'desc').limit(1).get().then((posts) => {
let hottestPost = posts[0];
let postID = hottestPost.get('postID');
let postText = hottestPost.get('text');
let currentDate = Date.now() / 1000;
var message;
//Verify that the hottest post was posted in the past 24 hours
if (hottestPost.get('date') > (currentDate - 86400)) {
//Build the notification text (shortening if too long)
let shortenedPostText = postText.substring(0,60);
var textEnd = '';
if (postText.length > 60) {
textEnd = '...';
}
let notificationText = 'Trending post on ' + communityName + ': ' + shortenedPostText + textEnd;
//Build the push notification
message = {
apns: {
headers: {
'apns-push-type': 'alert'
},
payload: {
aps: {
alert: {
body: notificationText,
},
},
postID: postID,
},
},
topic: communityID
}
}
//Send the message and return the promise
if (message === null) {
return null;
} else {
return admin.messaging().send(message);
}
})
.catch(error => {
console.log(error);
res.status(500).send(error);
})
if (communityPromise !== null) {
communityPromises.push(communityPromise);
}
})
res.sendStatus(200);
return Promise.all(communityPromises);
})
.catch(error => {
console.log(error);
res.status(500).send(error);
})
})
As samthecodingman advised, it is much better to use async/await in your case, as it will simplify the code and will make it much easier to read.
The following changes should do the trick (untested). Note how we use an Array of Community names in order to pass the names from one loop to the other. This works because, with Promise.all(), the returned values are in order of the Promises passed, regardless of completion order.
exports.sendNotificationTrendingPost = functions.https.onRequest(async (req, res) => {
try {
const db = admin.firestore();
const communitiesQuerySnap = await db.collection('communities').get();
const communityPromises = [];
const communityNames = [];
communitiesQuerySnap.forEach((community) => {
let communityID = community.get('communityID');
let communityName = community.get('name');
communityNames.push(communityName);
communityPromises.push(db.collection('communities').doc(communityID).collection('posts').orderBy('hotScore', 'desc').limit(1).get())
});
const postsQuerySnapArray = await Promise.all(communityPromises);
const messagePromises = [];
postsQuerySnapArray.forEach((postsQuerySnap, index) => {
const hottestPost = postsQuerySnap.docs[0];
const postID = hottestPost.get('postID');
const postText = hottestPost.get('text');
const currentDate = Date.now() / 1000;
let message;
if (hottestPost.get('date') > (currentDate - 86400)) {
//Build the notification text (shortening if too long)
let shortenedPostText = postText.substring(0, 60);
var textEnd = '';
if (postText.length > 60) {
textEnd = '...';
}
const communityName = communityNames[index]; // The two Arrays postsQuerySnapArray and communityName have the same order, because Promise.all keeps the order.
let notificationText = 'Trending post on ' + communityName + ': ' + shortenedPostText + textEnd;
//Build the push notification
message = {
apns: {
headers: {
'apns-push-type': 'alert'
},
payload: {
aps: {
alert: {
body: notificationText,
},
},
postID: postID,
},
},
topic: communityID
}
messagePromises.push(admin.messaging().send(message));
}
})
await Promise.all(messagePromises);
res.status(200).send({ result: "completed" }); // Or res.end()
} catch (error) {
console.log(error);
res.status(500).send(error);
}
});
I am trying to finish my script, but for some reason i don't know, it refuses to execute in the order i put it in.
I've tried placing a 'wait' function between the JoinRequest update function and the following code, but when run, it acts as if the function call and wait function were the other way round, countering the point of the wait().
const Roblox = require('noblox.js')
var fs = require('fs');
var joinRequests = []
...
function wait(ms) {
var d = new Date();
var d2 = null;
do { d2 = new Date(); }
while(d2-d < ms*1000);
};
...
function updateJReqs() {
Roblox.getJoinRequests(4745601).then((array) => {
var i;
var final = [];
for(i = 0; i < array.length; i++) {
final.push(array[i].username);
};
if(final === '') {
final = '-None';
};
joinRequests = final
console.log('Updated join requests.')
});
}
function check() {
setTimeout(() => {
fs.readFile('Request.txt',encoding = 'utf-8', function(err, data) {
if (err) {
check();
} else {
updateJReqs(); //for some reason this function is executed alongside the below, not before it.
// Tried putting wait(x) in here.
console.log('Request received: ' + data)
var solution = joinRequests
console.log('Fuffiling request with ' + solution)
fufillRequest(solution)
fs.unlink('Request.txt', function(err) {
if(err) throw err;
});
check();
}
});
}, 400)
}
check();
The script is supposed to wait until a file is created (accomplished), update the list of join requests (accomplished) and then create a new file with the list of join requests in(not accomplished).
if I understand your code you work with async code, you need to return a promise in updateJReqs and add a condition of leaving from the function because you have an infinite recursion
function updateJReqs() {
return new Promise(resolve => {
Roblox.getJoinRequests(4745601).then((array) => {
var i;
var final = [];
for(i = 0; i < array.length; i++) {
final.push(array[i].username);
};
if(final === '') {
final = '-None';
};
joinRequests = final
console.log('Updated join requests.')
resolve();
});
}
}
async function check() {
setTimeout(() => {
fs.readFile('Request.txt',encoding = 'utf-8', function(err, data) {
if (err) {
await check();
} else {
await updateJReqs();
// Tried putting wait(x) in here.
console.log('Request received: ' + data)
var solution = joinRequests
console.log('Fuffiling request with ' + solution)
fufillRequest(solution)
fs.unlink('Request.txt', function(err) {
if(err) throw err;
});
// you dont have an exit from your function check();
return 'Success';
}
});
}, 400)
}
check().then(res => console.log(res));
I am using the following to insert into MongoDB.
var tagData = JSON.parse(data);
var allTags = tagData.tags;
for (var j = 0; j < allTags.length; j++) {
var p = allTags[j].tagId.toString();
for (var k = 0; k < loggerParams.length; k++) {
var q = Object.keys(loggerParams[k]).toString();
if (p === q) {
// Prepare raw data tag
var tagRawDoc = {};
// Simple key-value assignment here
// Document prepared; ready to insert into MongoDB
database.addDocument('tagraw', tagRawDoc, function (err) {
if (err) {
log.info(util.format('Error adding document to tagrawdatas. %s', err.message));
throw err;
} else {
// Prepare history tag
var historyTagDoc = {};
historyTagDoc.tagNameAlias = tagRawDoc.tagNameAlias;
// Simple key-value assignment here
// Document prepared; ready to insert into MongoDB
database.addDocument('taghistory', historyTagDoc, function (err) {
if (err) {
log.info(util.format('Error adding document to tagrawdatas. %s', err.message));
throw err;
}
});
}
});
// Match found; exit loop
break;
}
}
}
The loggerParms is a simple JSON document read from file else-where. It allows for look-up in this code to build the document to be inserted. There will be 12 values in the allTags array. These 12 values are inserted successfully into the tagraw collection. However, in taghistory collection, the values from the last (or most recent) entry made into tagraw collection is repeated 12 times. Why does this happen?
The database.addDocument is shown below. It is a part of this article I am trying to replicate.
var MongoClient = require('mongodb').MongoClient;
var assert = require('assert');
var logger = require('../../util/logger');
var util = require('util');
function DB() {
this.db = "empty";
this.log = logger().getLogger('mongoMange-DB');
}
DB.prototype.connect = function(uri, callback) {
this.log.info(util.format('About to connect to DB'));
if (this.db != "empty") {
callback();
this.log.info('Already connected to database.');
} else {
var _this = this;
MongoClient.connect(uri, function(err, database) {
if (err) {
_this.log.info(util.format('Error connecting to DB: %s', err.message));
callback(err);
} else {
_this.db = database;
_this.log.info(util.format('Connected to database.'));
callback();
}
})
}
}
DB.prototype.close = function(callback) {
log.info('Closing database');
this.db.close();
this.log.info('Closed database');
callback();
}
DB.prototype.addDocument = function(coll, doc, callback) {
var collection = this.db.collection(coll);
var _this = this;
collection.insertOne(doc, function(err, result) {
if (err) {
_this.log.info(util.format('Error inserting document: %s', err.message));
callback(err.message);
} else {
_this.log.info(util.format('Inserted document into %s collection.', coll));
callback();
}
});
};
module.exports = DB;
That's because you are mixing a/multiple synchronous for and asynchronous code with database.addDocument which cause issues with function scope in nodejs.
A simple example of this kind of thing:
for(var i = 0; i < 10; i++){
setTimeout(() => console.log(i), 0);
}
You should use a package like async to handle flow control when iterating arrays/object asynchronously.
Simple example of your code refactored to use async:
var async = require('async');
var tagData = JSON.parse(data);
var allTags = tagData.tags;
async.each(allTags, function(tag, done){
var p = tag.tagId.toString();
var loggerParam = loggerParams.find(function(loggerParam){
var q = Object.keys(loggerParam).toString();
return p === q;
});
var tagRawDoc = {};
// Simple key-value assignment here
// Document prepared; ready to insert into MongoDB
return database.addDocument('tagraw', tagRawDoc, function (err){
if (err) return done(err);
// Prepare history tag
var historyTagDoc = {};
historyTagDoc.tagNameAlias = tagRawDoc.tagNameAlias;
// Simple key-value assignment here
// Document prepared; ready to insert into MongoDB
return database.addDocument('taghistory', historyTagDoc, done);
});
}, (err) => {
if(err) throw err;
console.log('All done');
});
I am trying to complete a few loops over firebase objects using .forEach and I am also using promises. This isn't working out how I had planned it. My basic problem is that the loops inside of my promises complete well after the promise chain itself completes. Here is my function:
var queue = new Queue(productUpdateQueue, function(data, progress, resolve, reject) {
var incomingUpdateData = data;
var receiptID = incomingUpdateData.receiptID;
var userID = incomingUpdateData.userID;
var oldProductID = incomingUpdateData.oldProductID;
var newProductID = incomingUpdateData.newProductID;
var newReceipt = incomingUpdateData.newReceipt;
var postID = "";
var updateObject = {};
updateObject['usersPrivate/'+userID+'/receipts/'+receiptID+'/items/'+oldProductID] = null;
updateObject['usersPrivate/'+userID+'/receipts/'+receiptID+'/items/'+newProductID] = newReceipt;
clicks.child('VigLink').orderByChild('item').equalTo(oldProductID).once('value', function(cuidSnapshot) {
return cuidSnapshot.forEach(function(cuidSnapshot) {
var cuid = cuidSnapshot.key;
updateObject['clicks/VigLink/'+cuid+'/item'] = newProductID;
console.log('one');
progress(20);
});
}).then(function() {
return userReceiptMetrics.child(userID).child('receipts').child(receiptID).child('items').child(oldProductID).once('value', function(oldSnapshot) {
var data = oldSnapshot.val()
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/items/'+oldProductID] = null
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/items/'+newProductID] = data
if (data != null) {
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/itemIDs/'+newProductID] = now
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/itemIDs/'+oldProductID] = null
};
console.log('two');
progress(40);
});
}).then(function() {
return userReceiptMetrics.child(userID).child('shops').child(oldProductID).once('value', function(oldSnapshot) {
var data = oldSnapshot.val()
updateObject['userReceiptMetrics/'+userID+'/shops/'+oldProductID] = null;
updateObject['userReceiptMetrics/'+userID+'/shops/'+newProductID] = data;
if (data != null) {
updateObject['userReceiptMetrics/'+userID+'/shopIDs/'+newProductID] = now;
updateObject['userReceiptMetrics/'+userID+'/shopIDs/'+oldProductID] = null;
};
console.log('three');
progress(60);
});
}).then(function() {
posts.once('value', function(postSnapshot) {
// use Promise.all and Array#map to wait for all these queries to finish
var allPosts = postSnapshot.val()
var postKeys = Object.keys(allPosts)
return Promise.all(postKeys.map(function(postKey) {
var postID = postKey;
return posts.child(postID).child('items').child(oldProductID).once('value', function(itemSnapshot) {
return itemSnapshot.forEach(function(itemSnapshot) {
var itemData = itemSnapshot.val()
console.log('post snapshot'+ itemData);
updateObject['posts/'+postID+'/items/'+oldProductID] = null
updateObject['posts/'+postID+'/items/'+newProductID] = itemData
});
});
}));
});
}).then(function() {
// Move to next item
return console.log('hey look here'+updateObject['posts/'+postID+'/items/'+newProductID]);
return firebaseRoot.update(updateObject, function(error) {
if (error) {
console.log("Error updating data:", error);
reject()
} else {
progress(100);
// resolve();
console.log('four');
}
});
}).then(function() {
// Move to next item
return console.log('second one'+updateObject['posts/'+postID+'/items/'+newProductID]);
return firebaseRoot.update(updateObject, function(error) {
if (error) {
console.log("Error updating data:", error);
reject()
} else {
progress(100);
// resolve();
console.log('four');
}
});
});
// Finish the task asynchronously
setTimeout(function() {
reject();
}, 10000);
});
And here is the output:
one
two
three
hey look hereundefined
second oneundefined
post snapshot[object Object]
post snapshot[object Object]
post snapshot[object Object]
post snapshot[object Object]
post snapshot[object Object]
post snapshot[object Object]
post snapshot[object Object]
post snapshot[object Object]
I think I might be using promises incorrectly but I don't know.
I figured out that I should be using Promise.all() in order to wait for my forEach loops to complete. Here is the code I used to solve my problem, enjoy:
var queue = new Queue(productUpdateQueue, function(data, progress, resolve, reject) {
var incomingUpdateData = data;
var receiptID = incomingUpdateData.receiptID;
var userID = incomingUpdateData.userID;
var oldProductID = incomingUpdateData.oldProductID;
var newProductID = incomingUpdateData.newProductID;
var newReceipt = incomingUpdateData.newReceipt;
var postID = "-KZOO0UII67uOmYo6DJh";
var postKeys = [];
var updateObject = {};
updateObject['usersPrivate/'+userID+'/receipts/'+receiptID+'/items/'+oldProductID] = null;
updateObject['usersPrivate/'+userID+'/receipts/'+receiptID+'/items/'+newProductID] = newReceipt;
return clicks.child('VigLink').orderByChild('item').equalTo(oldProductID).once('value', function(cuidSnapshot) {
return cuidSnapshot.forEach(function(cuidSnapshot) {
var cuid = cuidSnapshot.key;
updateObject['clicks/VigLink/'+cuid+'/item'] = newProductID;
progress(10);
});
}).then(function() {
return userReceiptMetrics.child(userID).child('receipts').child(receiptID).child('items').child(oldProductID).once('value', function(oldSnapshot) {
var data = oldSnapshot.val()
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/items/'+oldProductID] = null
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/items/'+newProductID] = data
if (data != null) {
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/itemIDs/'+newProductID] = now
updateObject['userReceiptMetrics/'+userID+'/receipts/'+receiptID+'/itemIDs/'+oldProductID] = null
};
progress(25);
});
}).then(function() {
return userReceiptMetrics.child(userID).child('shops').child(oldProductID).once('value', function(oldSnapshot) {
var data = oldSnapshot.val()
updateObject['userReceiptMetrics/'+userID+'/shops/'+oldProductID] = null;
updateObject['userReceiptMetrics/'+userID+'/shops/'+newProductID] = data;
if (data != null) {
updateObject['userReceiptMetrics/'+userID+'/shopIDs/'+newProductID] = now;
updateObject['userReceiptMetrics/'+userID+'/shopIDs/'+oldProductID] = null;
};
progress(40);
});
}).then(function() {
progress(55);
return posts.orderByChild('receipt').equalTo(receiptID).once('value');
}).then(function(postSnapshot) {
return postSnapshot.forEach(function(post) {
progress(70);
postKeys.push(post.key)
});
}).then(function() {
return Promise.all(postKeys.map(function(postKey) {
return posts.child(postKey).child('items').child(oldProductID).once('value', function(itemSnapshot) {
var itemData = itemSnapshot.val()
updateObject['posts/'+postKey+'/items/'+oldProductID] = null;
updateObject['posts/'+postKey+'/items/'+newProductID] = itemData;
});
})).then(function(results) {
progress(85);
return results;
});
}).then(function() {
return firebaseRoot.update(updateObject, function(error) {
if (error) {
console.log("Error updating data:", error);
reject()
} else {
progress(100);
resolve();
}
});
});
// Finish the task asynchronously
setTimeout(function() {
reject();
}, 10000);
});
I have a issue of concurrent request, which modifies db.
What am I doing is.
One request fetch data for user-1 , then calculate data for user-1 modified a field-1 in record, and save.
next request fetch data for user-1 , then calculate data for user-1 modified a field-1 in record, and save.
Both request operates simultaneously. so last request update the wrong data.
function calculate() {
var needUpdate = false;
user = new UserLib(user_id);
var old_config = user.config;
if (old_config[req.id]) {
old_config[req.id].value = 0;
needUpdate = true;
}
if (req.delete == void(0) || req.delete == false) {
delete req.delete;
old_config[req.id].value = old_config[req.id].value + 1;
needUpdate = true;
}
if (needUpdate) {
return user.save();
}
return true;
}
We are getting both requests at the same time.
var express = require('express');
var app = express();
app.get('/update', function(req, res) {
res.writeHead(200, {
'Content-Type': 'text/html'
});
calculate(req);
function calculate(req) {
var needUpdate = false;
user = new UserLib(user_id);
var old_config = user.config;
if (old_config[req.id]) {
old_config[req.id].value = 0;
needUpdate = true;
}
if (req.delete == void(0) || req.delete == false) {
delete req.delete;
old_config[req.id].value = old_config[req.id].value + 1;
needUpdate = true;
}
if (needUpdate) {
user.save();
}
}
res.end('Done');
});
first reuest with following parameter {
user_id: 1,
id: 1,
value: 5,
delete: false
}
Anothere request with follwing parmter {
user_id: 1,
id: 1,
delete: true
}
If you want to operate on each request simultaneously, I'd suggest to use Bluebird.map where you can handle each request as you want, with concurrency, and a final result.
For example:
let users = ['foo', 'bar']; //fetching users you want
Bluebird.map(users, (user) => {
return user.calculate()
.then((res) => res.shouldUpdate ? user.save() : Promise.resolve())
}, {concurrency: 2})
.then((results) => {
//results is an array with both resolved promises from below
})
You may also be interested in Bluebird.join where you could calculate, and join the resulting data for more than one promise.
Second example where you fetch same user twice in the same promise:
//both are promises
Bluebird.all([fetchUser1, fetchUser2])
.spread(function(user1, user2) {
//check if should update
return user1.delete !== user2.delete ? user.delete() : null
})
.then(() => {})
Bluebird.spread documentation