I'm trying out MongoDB via mongoose for backup. Originally, I just store locally my JSON files and just sends copies of it in mongodb, so when the free server(on heroku) restarts it just gets all old data from mongodb thus this code
registrantid:number
name:string
idnumber:string
contact:string
photoid:string
async function resyncRegistrants(){
const r = await Registrant.find({ "registrantid" : { $gt: 0 } })
console.log(r.length + ' registrationpool records found');
if (r.length === registrationpool.length) {
console.log('Nothing to Sync');
} else {
console.log('updating registrationpool database...')
registrationpool = r
}
}
but now it got bigger and it hits the storage limit I'm supposed to work it to have it free so now I'm considering constantly get records from MongoDB and throw them piece by piece (like it supposed to)
Q1: Is it still advisable to store the whole registrationpool locally or just find and edit from mongodb when needed?
Q2: Can I just get the objects without the photo and get it later when needed as like this
async function resyncRegistrants(){
const r = await Registrant.find({ "registrantid" : { $gt: 0 } })
console.log(r.length + ' registrationpool records found');
if (r.length === registrationpool.length) {
console.log('Nothing to Sync');
} else {
console.log('updating registrationpool database...')
registrationpool = []
for (var i = r.length - 1; i >= 0; i--) {
let tempregistrant = {}
tempregistrant.registrantid = r[i].registrantid
tempregistrant.name = r[i].name
tempregistrant.idnumber = r[i].idnumber
tempregistrant.contact = r[i].contact
registrationpool.push(tempregistrant)
//is there a way to shorten this?
}
}
}
app.post('/getphotoid', (req, res)=>{
if (req.body.getphotid > 0) {
const r = await Registrant.find({ "registrantid" : { $eq: req.body.getphotid } })
if (r.length===1) {
if (r[0].registrantid==req.body.getphotid) {
res.json(
[
{thephoto: r[0].photoid}
]
)
}
}
} else {
res.send('invalid registrantid')
}
})
Related
I'm trying to filter some data by themes and am puzzled as to how I can go about doing it. Say I have two items with the themes ['People', 'Technology', 'Culture'] and ['Economy', 'Technology', 'Culture'], if the query is Technology, I am able to see both of these items appearing. But if the query is Technology and Culture, I'm not able to see either of them because ["Technology", "Culture"] =/= ['People', 'Technology', 'Culture'] and vice versa. My code searches for the exact list, not it's components, if the query was Technology and Culture then I want both of those items to show up since it is inside that list.
I'm not sure how to do this, I'm using MERN stack and here is my backend code:
const Project = require('../models/projectModel')
const mongoose = require('mongoose')
// get all projects
const getProjects = async (req, res) => {
const projects = await Project.find().sort({ createdAt: -1 })
// const test = await Project.find({assignment_type: 1 || 2}).sort({ createdAt: -1 })
// console.log(test)
res.status(200).json(projects)
}
// get filtered project
const getFilteredProjects = async (req, res) => {
var request = {}
console.log(req.query.sdg)
console.log('t' + req.query.theme)
console.log('a' + req.query.assignment_type)
// var t = ["Economical", "Technological"]
// const test = await Project.find({theme: ("Technological" && "Economical")}).sort({ createdAt: -1 })
// const test = await Project.find({
// $and:
// }).sort({ createdAt: -1 })
// console.log(test)
// Function to separate commas from string
function separateCommas(str) {
let t = []
for (let i = 0; i < str.length; i++) {
if (str[i] === ',') {
console.log(i)
t.push(i)
}
}
let themeArray = []
if (t.length === 1) {
let theme1 = str.slice(0, t[0])
let theme2 = str.slice(t[0]+1)
themeArray.push(theme1)
themeArray.push(theme2)
}
if (t.length === 2) {
let theme1 = str.slice(0, t[0])
let theme2 = str.slice(t[0]+1, t[1])
let theme3 = str.slice(t[1]+1)
themeArray.push(theme1)
themeArray.push(theme2)
themeArray.push(theme3)
}
request["theme"] = themeArray.sort()
}
// See if sdg selected
if (req.query.sdg !== '') {
request["sdg"] = req.query.sdg
}
// See if assignment type selected
if (req.query.assignment_type !== '') {
request["assignment_type"] = req.query.assignment_type
}
// See if theme selected
if (req.query.theme !== '') {
if (req.query.theme.length > 14) {
separateCommas(req.query.theme)
}
else {
request["theme"] = req.query.theme
}
}
console.log(request)
const projects = await Project.find(request).sort({ createdAt: -1 })
res.status(200).json(projects)
}
module.exports = {
getProjects,
getProject,
createProject,
deleteProject,
updateProject,
getFilteredProjects
}
This is how my backend code receives the data from the database, it sends it in this format where there can be multiple theme's:
{
sdg: 'SDG 2: Zero Hunger',
assignment_type: 'Discussion Topics',
theme: 'Economy'
}
const names = cookie.currentMon.name;
dexData.pokemons.find((pokemon) => {
if (pokemon.name === names) {
pokemon.amount += 1;
} else {
const dexobj = {
pokemonID: poke.num,
name: names,
amount: 1,
};
dexData.pokemons.push(dexobj);
dexData.totalPokemon += 1;
};
});
dexData.save().catch((error) => {
console.log(error);
});
I have this code, and when the else statement is run it pushes the dexobj multiple times. If theres 2 previous docs, it will push 2 times. It basically doubles what is currently in the database. How can i search through the array of pokemon like this without the multiple pushes?
changed it to
const names = cookie.currentMon.name;
const pokemon = dexData.pokemons.find(x => x.name === names)
if (pokemon) {
pokemon.amount += 1;
} else {
const dexobj = {
pokemonID: poke.num,
name: names,
amount: 1,
};
dexData.pokemons.push(dexobj);
dexData.totalPokemon += 1;
};
dexData.save().catch((error) => {
console.log(error);
});
and it works now
How's it going?
I got the example order book code in python (https://support.kraken.com/hc/en-us/articles/360027677512-Example-order-book-code-Python-) and translate it to javascript to run in node. But the book is wrong, it doesn't remove all old prices level. I'm sending my code below. I'd like help to solve this issue.
const websocket = require('ws');
const ws = new websocket('wss://ws.kraken.com');
const api_book = {'bid':[], 'ask':[]};
const api_depth = 10;
const api_output_book = () => {
bid = api_book['bid'].sort((x, y) => parseFloat(y[0])-parseFloat(x[0]));
ask = api_book['ask'].sort((x, y) => parseFloat(x[0])-parseFloat(y[0]));
console.log ('Bid\t\t\t\t\tAsk');
for (let x=0;x<api_depth;x++) {
console.log(`${bid[x][0]} (${bid[x][1]})\t\t\t${ask[x][0]} (${ask[x][1]})`);
}
}
const api_update_book = (side, data) => {
data.forEach((e) => {
let index = api_book[side].findIndex(o => o[0] == e[0]);
if (parseFloat(e[1]) > 0){
if(index < 0){
api_book[side].push([e[0],e[1]]);
} else {
api_book[side][index] = [e[0],e[1]];
}
} else {
api_book[side].splice(index,1);
}
});
if(side=='bid'){
api_book['bid'].sort((x, y) => parseFloat(y[0])-parseFloat(x[0]));
} else if(side=='ask'){
api_book['ask'].sort((x, y) => parseFloat(x[0])-parseFloat(y[0]));
}
}
ws.on('open', open = () => {
ws.send('{"event":"subscribe", "subscription":{"name":"book", "depth":'+api_depth+'}, "pair":["XBT/USD"]}');
console.log('Kraken websocket connected!');
});
ws.on('message', incoming = (data) => {
try {
data = JSON.parse(data.toString('utf8'));
if (data[1]) {
if (data[1]['as']) {
api_update_book('ask', data[1]['as'])
api_update_book('bid', data[1]['bs'])
} else if (data[1]['a'] || data[1]['b']) {
if (data[1]['a']) {
api_update_book('ask', data[1]['a']);
}
if (data[1]['b']) {
api_update_book('bid', data[1]['b']);
}
}
api_output_book();
}
} catch (error) {
console.log(error);
}
});
So I have also been playing around with Kraken's order book and came up with this solution using Angular. I also added a few console logs into the mix so that you can take it and run it in the browser. Hope this helps!
// variables
private ws = new WebSocket('wss://ws.kraken.com')
public asks = [];
public bids = [];
// Web Socket open connection
this.ws.onopen = () => {
this.ws.send(JSON.stringify(this.message));
console.log('Trade WS with Kraken connected')
}
// Fires when new data is received from web socket
this.ws.onmessage = (event) => {
var data = JSON.parse(event.data);
if (!data.event) {
if (data[1]['as']) {
this.asks = data[1]['as'];
this.bids = data[1]['bs'];
console.log('Initialised Book');
console.log(this.asks, this.bids);
} else if (data[1]['a'] || data[1]['b']) {
if (data[1]['a']) {
this.update_book(this.asks, 'ask', data[1]['a']);
}
if (data[1]['b']) {
this.update_book(this.bids, 'bid', data[1]['b']);
}
}
}
}
// Updating Orderbook
update_book (arr, side, data) {
if (data.length > 1) { // If 2 sets of data are received then the first will be deleted and the second will be added
let index = arr.findIndex(o => o[0] == data[0][0]); // Get position of first data
arr.splice(index, 1); // Delete data
arr.push([ data[1][0], data[1][1] ]); // Insert new data
console.log('Delete and Insert');
} else {
let index = arr.findIndex(o => o[0] == data[0][0]);
console.error(index);
if (index > -1) { // If the index matches a price in the list then it is an update message
arr[index] = [data[0][0], data[0][1]]; // Update matching position in the book
console.log('Updated ' + index);
} else { // If the index is -1 then it is a new price that came in
arr.push([data[0][0], data[0][1]]); // Insert new price
this.sort_book(arr, side); // Sort the book with the new price
arr.splice(10, 1); // Delete the 11th entry
console.log('Insert Only');
}
}
this.sort_book(arr, side); // Sort the order book
}
// Sort Orderbook
sort_book (arr, side) {
if (side == 'bid') {
arr.sort((x, y) => parseFloat(y[0]) - parseFloat(x[0]));
} else if (side == 'ask') {
arr.sort((x, y) => parseFloat(x[0]) - parseFloat(y[0]));
}
}
I would also recommend just having a look at this resource:
How to maintain a valid orderbook
I want to store an ordered list of image paths in a Firestore document. The images are uploaded to storage with metadata containing the position and a tag linking to a document. The amount of images to expect is known. Uploading of the images is done nearly simultaneously.
Right know, I do this by storing the list of image paths as an array.
In short: an image is uploaded, using a transaction a document is read, the array is altered and saved again. The transaction never fails, but sometimes an array item disappears again after another image is uploaded.
My two questions are: is an array the best structure in a NoSql database for this problem? Why do array item disappear despite of the transaction?
export const addImageToTag = functions.storage.object().onFinalize(async (object) => {
try {
const tag = object.metadata.Tag;
const position = object.metadata.Position;
const tagRef = db.collection(COLLECTION_TAGS).doc(tag)
const tagSnapshot = await db.runTransaction(t => t.get(tagRef));
const images = tagSnapshot.data().images;
images[position] = object.name;
let uploadCount = 0;
for (let i = 0; i < tagSnapshot.data().imageCount; i++) {
if (images[i].length > 0) {
uploadCount++;
} else {
break;
}
}
let state;
if (tagSnapshot.data().imageCount === uploadCount) {
state = STATE_ACTIVE;
} else {
state = STATE_UPLOADING;
}
await tagSnapshot.ref.update({
images: images,
state: state
});
console.log('Image ', position, ' of tag ', tag, 'added')
} catch (error) {
console.error('Call failed: ', error)
}
});
The second question is answered, actually putting the code in the transaction block did the trick. I had read that referencing to the tag document using tagSnapshot.ref was sufficient, which was false.
export const addImageToTag = functions.storage.object().onFinalize(async (object) => {
try {
const tag = object.metadata.Tag;
const position = object.metadata.Position;
const tagRef = db.collection(COLLECTION_TAGS).doc(tag)
await db.runTransaction(async transaction => {
const tagSnapshot = await transaction.get(tagRef)
const images = tagSnapshot.data().images;
images[position] = object.name;
let uploadCount = 0;
for (let i = 0; i < tagSnapshot.data().imageCount; i++) {
if (images[i].length > 0) {
uploadCount++;
} else {
break;
}
}
let state;
if (tagSnapshot.data().imageCount === uploadCount) {
state = STATE_ACTIVE;
} else {
state = STATE_UPLOADING;
}
transaction.update(tagRef, {
images: images,
state: state
});
if (state === STATE_ACTIVE) {
console.log('All images of tag ', tag, ' were succesfully added')
}
});
} catch (error) {
console.error('Call failed: ', error)
}
});
I noticed this strange behaviour, when for few users only on production, it inserts every item multiple times to collection on asynchronous Meteor call. I tried multiple things, but nothing worked. I can't test on localhost, bc it never happens to me on localhost or in production.
I spent the whole night solving this, but didn't find any solution. I suppose it's caused by new Date(), but I have to call it somewhere. The production server is in Amsterdam and it seems like it happens only for users located further outside of Europe.
I found this to be similar issue, but can't really wrap my head on how to implement - https://github.com/meteor/meteor/issues/4263
Slug is what same songs are supposed to have the same.
This is the workflow, user clicks on song that triggers addNewSong function:
addNewSong = function (track) {
Globals.current_track = track;
checkIfSongAlreadySaved();
}
I need to check if song is already in collection, if it's -> route to it, else create the new song and route to it.
checkIfSongAlreadySaved = function() {
loadPrimaryGlobalItems();
Meteor.call('checkIfSongAlreadySaved', Globals.current_song_item_slug, function(error, result) {
if( result.item ) {
Globals.current_song_item_id = result.item._id;
Globals.current_song_item_slug = result.item.slug;
routeToSongPage();
if (! (result.item.download && result.item.mp3) ) {
downloadSong();
}
}
else {
loadSecondaryGlobalItems();
var item = {
slug:Globals.current_song_item_slug,
duration:Globals.current_duration,
thumbnail:Globals.current_song_thumbnail,
title:Globals.current_cleaned_song,
album:Globals.current_track.album,
artist:Globals.current_track.artists[0],
track:Globals.current_track.name,
date:result.date,
}
Globals.current_song_item_id = Songs.insert(item);
routeToSongPage();
downloadSong();
recentSongItem(result.date);
}
});
}
Add recent song
recentSongItem = function (date) {
Recentsongs.insert({
slug:Globals.current_song_item_slug,
songId:Globals.current_song_item_id,
title:Globals.current_cleaned_song,
duration:Globals.current_duration,
date:date,
});
}
If creating new song,
downloadSong = function() {
Meteor.call('findSong', Globals.current_song, function(error, result) {
console.log(result);
if (result) {
Globals.current_song_mp3 = true;
updateSongItemDownload(result.itemDetails);
}
else {
alert('not found')
}
});
}
and update song, to add download and mp3 values.
updateSongItemDownload = function(link) {
Songs.update({
_id: Globals.current_song_item_id
},
{
$set: {
download: link,
mp3: Globals.current_song_mp3,
}
});
}
Server methods:
Meteor.methods({
checkIfSongAlreadySaved: function(slug) {
return {item: Songs.findOne({slug:slug}), date: new Date()};
},
findSong:function(song) {
ServerGlobals.current_song = song;
var result = searchSite();
return result;
},
});
EDIT:
This is my subscription, just in case it might be causing the problem:
Template.songPage.onCreated(function() {
Session.set('processing', true);
var self = this;
self.autorun(function() {
var id = Router.current().params.id;
self.subscribe('singleSong', id);
var item = Songs.findOne({_id: id});
if (item) {
if (item.download) {
createSong(item.download);
}
else if( item.download === false ) {
console.log('item not found');
}
Session.set('loader', false);
Session.set('processing', false);
}
});
});
Meteor.publish('singleSong', function(id) {
check(id, String);
return Songs.find({_id: id});
});
You can apply a unique index on the slug field to ensure the same slug can only exist once and the second operation to insert will fail and show up as an error in your callback which you can discard or alert user as you desire.
db.collection.createIndex( { slug: 1 }, { unique: true } )
You will need to clear or modify the slug name on the dups from the db before applying the index though