Meteor, MongoDb: Double inserting to collection on few occurences - node.js

I noticed this strange behaviour, when for few users only on production, it inserts every item multiple times to collection on asynchronous Meteor call. I tried multiple things, but nothing worked. I can't test on localhost, bc it never happens to me on localhost or in production.
I spent the whole night solving this, but didn't find any solution. I suppose it's caused by new Date(), but I have to call it somewhere. The production server is in Amsterdam and it seems like it happens only for users located further outside of Europe.
I found this to be similar issue, but can't really wrap my head on how to implement - https://github.com/meteor/meteor/issues/4263
Slug is what same songs are supposed to have the same.
This is the workflow, user clicks on song that triggers addNewSong function:
addNewSong = function (track) {
Globals.current_track = track;
checkIfSongAlreadySaved();
}
I need to check if song is already in collection, if it's -> route to it, else create the new song and route to it.
checkIfSongAlreadySaved = function() {
loadPrimaryGlobalItems();
Meteor.call('checkIfSongAlreadySaved', Globals.current_song_item_slug, function(error, result) {
if( result.item ) {
Globals.current_song_item_id = result.item._id;
Globals.current_song_item_slug = result.item.slug;
routeToSongPage();
if (! (result.item.download && result.item.mp3) ) {
downloadSong();
}
}
else {
loadSecondaryGlobalItems();
var item = {
slug:Globals.current_song_item_slug,
duration:Globals.current_duration,
thumbnail:Globals.current_song_thumbnail,
title:Globals.current_cleaned_song,
album:Globals.current_track.album,
artist:Globals.current_track.artists[0],
track:Globals.current_track.name,
date:result.date,
}
Globals.current_song_item_id = Songs.insert(item);
routeToSongPage();
downloadSong();
recentSongItem(result.date);
}
});
}
Add recent song
recentSongItem = function (date) {
Recentsongs.insert({
slug:Globals.current_song_item_slug,
songId:Globals.current_song_item_id,
title:Globals.current_cleaned_song,
duration:Globals.current_duration,
date:date,
});
}
If creating new song,
downloadSong = function() {
Meteor.call('findSong', Globals.current_song, function(error, result) {
console.log(result);
if (result) {
Globals.current_song_mp3 = true;
updateSongItemDownload(result.itemDetails);
}
else {
alert('not found')
}
});
}
and update song, to add download and mp3 values.
updateSongItemDownload = function(link) {
Songs.update({
_id: Globals.current_song_item_id
},
{
$set: {
download: link,
mp3: Globals.current_song_mp3,
}
});
}
Server methods:
Meteor.methods({
checkIfSongAlreadySaved: function(slug) {
return {item: Songs.findOne({slug:slug}), date: new Date()};
},
findSong:function(song) {
ServerGlobals.current_song = song;
var result = searchSite();
return result;
},
});
EDIT:
This is my subscription, just in case it might be causing the problem:
Template.songPage.onCreated(function() {
Session.set('processing', true);
var self = this;
self.autorun(function() {
var id = Router.current().params.id;
self.subscribe('singleSong', id);
var item = Songs.findOne({_id: id});
if (item) {
if (item.download) {
createSong(item.download);
}
else if( item.download === false ) {
console.log('item not found');
}
Session.set('loader', false);
Session.set('processing', false);
}
});
});
Meteor.publish('singleSong', function(id) {
check(id, String);
return Songs.find({_id: id});
});

You can apply a unique index on the slug field to ensure the same slug can only exist once and the second operation to insert will fail and show up as an error in your callback which you can discard or alert user as you desire.
db.collection.createIndex( { slug: 1 }, { unique: true } )
You will need to clear or modify the slug name on the dups from the db before applying the index though

Related

get specific keys only from mongodb

I'm trying out MongoDB via mongoose for backup. Originally, I just store locally my JSON files and just sends copies of it in mongodb, so when the free server(on heroku) restarts it just gets all old data from mongodb thus this code
registrantid:number
name:string
idnumber:string
contact:string
photoid:string
async function resyncRegistrants(){
const r = await Registrant.find({ "registrantid" : { $gt: 0 } })
console.log(r.length + ' registrationpool records found');
if (r.length === registrationpool.length) {
console.log('Nothing to Sync');
} else {
console.log('updating registrationpool database...')
registrationpool = r
}
}
but now it got bigger and it hits the storage limit I'm supposed to work it to have it free so now I'm considering constantly get records from MongoDB and throw them piece by piece (like it supposed to)
Q1: Is it still advisable to store the whole registrationpool locally or just find and edit from mongodb when needed?
Q2: Can I just get the objects without the photo and get it later when needed as like this
async function resyncRegistrants(){
const r = await Registrant.find({ "registrantid" : { $gt: 0 } })
console.log(r.length + ' registrationpool records found');
if (r.length === registrationpool.length) {
console.log('Nothing to Sync');
} else {
console.log('updating registrationpool database...')
registrationpool = []
for (var i = r.length - 1; i >= 0; i--) {
let tempregistrant = {}
tempregistrant.registrantid = r[i].registrantid
tempregistrant.name = r[i].name
tempregistrant.idnumber = r[i].idnumber
tempregistrant.contact = r[i].contact
registrationpool.push(tempregistrant)
//is there a way to shorten this?
}
}
}
app.post('/getphotoid', (req, res)=>{
if (req.body.getphotid > 0) {
const r = await Registrant.find({ "registrantid" : { $eq: req.body.getphotid } })
if (r.length===1) {
if (r[0].registrantid==req.body.getphotid) {
res.json(
[
{thephoto: r[0].photoid}
]
)
}
}
} else {
res.send('invalid registrantid')
}
})

Make sure firestore collection docChanges keeps alive

The final solution is at the bottom of this post.
I have a nodeJS server application that listens to a rather big collection:
//here was old code
This works perfectly fine: these are lots of documents and the server can serve them from cache instead of database, which saves me tons of document reads (and is a lot faster).
I want to make sure, this collection is staying alive forever, this means reconnecting if a change is not coming trough.
Is there any way to create this certainty? This server might be online for years.
Final solution:
database listener that saves the timestamp on a change
export const lastRolesChange = functions.firestore
.document(`${COLLECTIONS.ROLES}/{id}`)
.onWrite(async (_change, context) => {
return firebase()
.admin.firestore()
.collection('syncstatus')
.doc(COLLECTIONS.ROLES)
.set({
lastModified: context.timestamp,
docId: context.params.id
});
});
logic that checks if the server has the same updated timesta.mp as the database. If it is still listening, it should have, otherwise refresh listener because it might have stalled.
import { firebase } from '../google/auth';
import { COLLECTIONS } from '../../../configs/collections.enum';
class DataObjectTemplate {
constructor() {
for (const key in COLLECTIONS) {
if (key) {
this[COLLECTIONS[key]] = [] as { id: string; data: any }[];
}
}
}
}
const dataObject = new DataObjectTemplate();
const timestamps: {
[key in COLLECTIONS]?: Date;
} = {};
let unsubscribe: Function;
export const getCachedData = async (type: COLLECTIONS) => {
return firebase()
.admin.firestore()
.collection(COLLECTIONS.SYNCSTATUS)
.doc(type)
.get()
.then(async snap => {
const lastUpdate = snap.data();
/* we compare the last update of the roles collection with the last update we
* got from the listener. If the listener would have failed to sync, we
* will find out here and reset the listener.
*/
// first check if we already have a timestamp, otherwise, we set it in the past.
let timestamp = timestamps[type];
if (!timestamp) {
timestamp = new Date(2020, 0, 1);
}
// if we don't have a last update for some reason, there is something wrong
if (!lastUpdate) {
throw new Error('Missing sync data for ' + type);
}
const lastModified = new Date(lastUpdate.lastModified);
if (lastModified.getTime() > timestamp.getTime()) {
console.warn('Out of sync: refresh!');
console.warn('Resetting listener');
if (unsubscribe) {
unsubscribe();
}
await startCache(type);
return dataObject[type] as { id: string; data: any }[];
}
return dataObject[type] as { id: string; data: any }[];
});
};
export const startCache = async (type: COLLECTIONS) => {
// tslint:disable-next-line:no-console
console.warn('Building ' + type + ' cache.');
const timeStamps: number[] = [];
// start with clean array
dataObject[type] = [];
return new Promise(resolve => {
unsubscribe = firebase()
.admin.firestore()
.collection(type)
.onSnapshot(querySnapshot => {
querySnapshot.docChanges().map(change => {
timeStamps.push(change.doc.updateTime.toMillis());
if (change.oldIndex !== -1) {
dataObject[type].splice(change.oldIndex, 1);
}
if (change.newIndex !== -1) {
dataObject[type].splice(change.newIndex, 0, {
id: change.doc.id,
data: change.doc.data()
});
}
});
// tslint:disable-next-line:no-console
console.log(dataObject[type].length + ' ' + type + ' in cache.');
timestamps[type] = new Date(Math.max(...timeStamps));
resolve(true);
});
});
};
If you want to be sure you have all changes, you'll have to:
keep a lastModified type field in each document,
use a query to get documents that we modified since you last looked,
store the last time you queried on your server.
Unrelated to that, you might also be interested in the recently launched ability to serve bundled Firestore content as it's another way to reduce the number of charged reads you have to do against the Firestore server.

Reload page content if database variable has changed Meteor

I am working on meteor app. I want to reload the content of the client when the database variable has changed. I am using pub-sub. I want to load content if the status variable has changed.
Meteor.publish('activities', function(loggedInUserToken) {
var authObj = AuthenticationToken.findOne({ authToken: loggedInUserToken });
if (authObj) {
var userObj = Users.findOne({ _id: authObj.user_id });
var activities = Activities.find({}, { sort: { createdAt: -1 } });
return activities;
}
return this.ready();
});
Template.masterSku.onCreated(function() {
var instance = this;
instance.autorun(function() {
var loggedInUserToken = "xxxxxxxxxxxxxxxx"
statusSuscription = instance.subscribe("activities", loggedInUserToken);
var activitiesObj = Activities.findOne({}, { sort: { createdAt: -1 } })
if (activitiesObj && activitiesObj.status == "true") {
console.log("Status Changed load Content accordingly")
}
})
The autorun will not recompute until a reactive data has changed AND this change is requested within the computation. While a subscription is a reactive source, you also need to check for it's readyness and thus cause a new computation cycle:
instance.autorun(function() {
var loggedInUserToken = "xxxxxxxxxxxxxxxx"
var statusSuscription = instance.subscribe("activities", loggedInUserToken);
if (statusSuscription.ready()) { // causes a new compuation by the Tracker
var activitiesObj = Activities.findOne({}, { sort: { createdAt: -1 } })
if (activitiesObj && activitiesObj.status == "true") {
console.log("Status Changed load Content accordingly")
// here you can insert data into a ReactiveVar or ReactiveDict and use
// result in a Template helper, which itself will cause your Template to re-render
}
}
}
You can use Tracker in onCreated function OR you can try fill data through helpers.
import { Tracker } from 'meteor/tracker';
Tracker.autorun(() => {
var loggedInUserToken = "xxxxxxxxxxxxxxxx"
statusSuscription = instance.subscribe("activities", loggedInUserToken);
var activitiesObj = Activities.findOne({}, { sort: { createdAt: -1 } })
if (activitiesObj && activitiesObj.status == "true") {
console.log("Status Changed load Content accordingly")
}
});

How to retrieve from Firebase based on current user

Here is the deal.
I am using VueFire and would like to retrieve data from a Firebase database I have set up. In there I have a node with users and each one is defined by a unique id. Also for every user I have an array with a cart full of items they would to purchase. To make it dynamic I am calling the reference in the firebase hook like so:
export default {
firebase:{
cart: app.database().ref('users')
}, //and so on and so on
}
Instead of .ref('users') I want to retrieve the current user, using his unique id like so: .ref('users/' + user.uid + '/cart')
To get the current user uid I do the observer that is firebase.auth().onAuthStataChanged(user=>{//code here})
Problem is, since this is asynchronous, the firebase hook activates before the current user is retrieved. I have tried to simply call firebase.auth().currentUser, but this is also slow and unreliable to use.
I am open to any kinds of suggestions!
I made a solution that worked for my case and might work for yours.
What I did is attach value listeners when the user gets authenticated and then I turn them off when the user loses authentication. With this method I'm only asking to retrieve user data if the user is actually authenticated. Here's how I did it:
this.$root.firebase.auth().onAuthStateChanged(user => {
if (user) {
this.$store.dispatch('setUserUID', user.uid)
// Listen for User Data
let dataRef = this.$root.db.ref('/userdata/' + user.uid)
let self = this
dataRef.on('value', function(snap) {
let value = snap.val()
self.$store.dispatch('setUserData', value)
})
}
else {
// Stop Listening for User Data
this.$root.db.ref('/userdata/' + this.$store.state.user.uid).off()
this.$store.dispatch('setUserUID', null)
}
})
Okay I got it. Thanks to #Daniel D for his tip on binding as an array or object. So as it turns out I don't have to do it in the firebase reference hook, I just have to bind it as array in the mounted() hook for example. I just declare an empty cart: [] in data and then fill it like so:
<script>
import changeColorMixin from '../mixins/changeColorMixin'
import animateFunction from '../mixins/animationMixin'
import Firebase from 'firebase'
import app from '../firebaseApp'
export default {
data(){
return{
isLoggedIn: '',
cart: []
}
},
methods:{
changeColor: changeColorMixin,
animateEntrance: animateFunction,
promptLogin: function(){
console.log('you need to login!');
},
chooseSize: function($event){
$($event.target).parents().eq(2).addClass('chosen');
},
closeOrder: function($event) {
$($event.target).parents().eq(2).removeClass('chosen');
},
makeOrder: function($event){
var $this = this;
Firebase.auth().onAuthStateChanged(function(user){
if(user){
var cartRef = app.database().ref('users/' + user.uid + '/cart');
var currentCart;
cartRef.once('value', function(result) {
currentCart = result.val();
var size = $($event.target).html();
var name = $($event.target).parents().eq(2).attr('id');
app.database().ref('users/' + user.uid + '/cart/' + name + '/count').once('value', function(snap){
var count = snap.val();
if(count > 0){
count = count + 1;
}else{
count = 1;
}
cartRef.child(name).set({
size: size,
count: count,
name: name
});
});
});
}else{
console.log('gotta login mate');
}
});
}
},
mounted(){
Firebase.auth().onAuthStateChanged(user => {
if(user){
this.isLoggedIn = true;
var cartRef = app.database().ref('users/' + user.uid + '/cart');
this.$bindAsArray('cart', cartRef);
}else{
this.isLoggedIn = false;
}
});
this.animateEntrance();
this.changeColor();
}
}
</script>
As I said big thanks to #Daniel D

Transform publish MeteorJS with count of show

I try to use limit : count with transform observer in Meteor and don't understand how to do it without "dirty" solutions.
Code I have on Client (not all, but main part)
var self = this;
self.autorun(function() {
self.subscribe('posts', Session.get('count')); // some Number like 10
}); // client
And on server where I try to use it
Meteor.publish('posts', function(count){
check(count, Number);
let i = 0;
var transform = function(doc) {
console.log(i,count);
if (i < count){ // I try something here
doc.description = new Date();
i++;
return doc;
}
else self.ready();
}
var self = this;
var observer = Posts.find().observe({
added: function (document) {
self.added('posts', document._id, transform(document));
},
changed: function (newDocument, oldDocument) {
self.changed('posts', document._id, transform(newDocument));
},
removed: function (oldDocument) {
self.removed('posts', oldDocument._id);
}
});
self.onStop(function () {
observer.stop();
});
self.ready();
});
Any idea how to limit count of shown documents with transform in publish ?
Just use Posts.find({},{limit:count}) in your query.

Resources