Mongodb UseUnifiedTopology: True causes connection errors in my code - node.js

I have a small app written in Node.js that downloads some data from an API, and then saves it into a MongoDB. When I have useunifiedtopology: false my code runs fine, connects to the mongodb fine and updates everything fine.
When I set useunifiedtopolgy: true I get the following errors
MongoError: Cannot use a session that has ended
Here is what I am trying to accomplish...I can't for the life of me figure out where it's going wrong. If I remove the db.close() from dbupdater.js, it works, but the app just hangs.
Here is how I am going about things...
It should also be noted, that if I remove the multiple locations and extraction calls from below, then it does work fine. The errors noted above when using useunifiedtopology: false only seem to happen when I am doing multiple calls to the dbupdater module. So, to be clear, if I just hit the API once, and do one call to dbupdater, it works fine. If I call the API and the dbupdater function multiple times, it fails with the error:
MongoError: Cannot use a session that has ended.
app.js Contents:
const Downloader = require('./modules/Downloader')
const dbupdater = require('./modules/dbUpdater')
var Locations = ['Location1','Location2']
var ExtractionTypes = ['Extraction1','Extraction2','Extraction3' ]
var interval = 60000; //API Request Delay in MS
var promise = Promise.resolve();
function main() {
Locations.forEach(location => {
promise = promise.then(function(){
ExtractionTypes.forEach(extraction =>{
promise = promise.then(function(){
talonDownloader(location,extraction,function(data){
if (data == null) {
console.log(`No ${extraction} Data Recieved for period requested.`)
}
else {
dbupdater(extraction,data)
}
})
return new Promise(function (resolve) {
setTimeout(resolve, interval)
})
})
})
return new Promise(function (resolve) {
setTimeout(resolve, interval)
})
})
})
}
main();
Then here is the code for my dbupdater...
dbudater.js
module.exports = function (extraction,data) {
const MongoClient = require('mongodb').MongoClient;
const date = require('date-and-time');
const crypto = require('crypto')
const dburl = "mongodb://localhost:27017/";
const dbName = 'palDB'
//Initialize Date Information
const now = new Date()
const todayDate = date.format(now, 'DD-MMM-YYYY')
if (extraction == 'operation') {
var collectionName = 'operations'
var indexValue = 'ACTIVITY_ID'
updateDBGeneral();
}
else if (extraction == 'aircraft') {
var collectionName = 'aircraft'
var indexValue = 'AIRCRAFT_ID'
updateDBGeneral();
updateDBHistorical();
}
else if (extraction == 'cancelanalysis') {
var collectionName = 'cancelAnalysis'
var indexValue = 'SCH_ACT_ID'
updateDBGeneral();
}
else if (extraction == 'dailyflylog') {
var collectionName = 'dfl'
var indexValue = 'SCH_ACT_ID'
updateDBGeneral();
}
else if (extraction == 'instructor') {
var collectionName = 'instructor'
var indexValue = 'ETA_ID'
updateDBGeneral();
}
else if (extraction == 'resschedproductivity') {
var collectionName = 'resSchedProductivity'
var indexValue = 'opid'
updateDBGeneral();
}
else if (extraction == 'student') {
var collectionName = 'student'
var indexValue = 'ETA_ID'
updateDBGeneral();
}
else if (extraction == 'trainingrecord') {
var collectionName = 'trgrec'
var indexValue = 'STUDENT_REGISTRATION_ID'
updateDBGeneral();
}
else if (extraction == 'studentprogression') {
var collectionName = 'studentProgression'
var indexValue = 'studentProgIndex'
updateDBGeneral();
}
//Initialize DB Updater functions
function updateDBGeneral(){
MongoClient.connect(dburl,{useNewUrlParser: true, useUnifiedTopology: true})
.then((db) => {
var dbo = db.db(dbName);
console.log(`Connection Established to ${dbo.databaseName} database.`)
data.forEach(element => {
if (indexValue == 'opid') {
element[indexValue] = crypto.createHash('md5').update(element.CAL_DATE + element.RES_TYPE_ID).digest('hex'); //Add Value to filter on
var filter = {[indexValue]: element[indexValue]};
} else if (indexValue =='studentProgIndex'){
element[indexValue] = crypto.createHash('md5').update(element.PERS_REGISTER_ID + element.UNIT + element.ACT_START).digest('hex'); //Add value to filter on
var filter = {[indexValue]: element[indexValue]}
} else var filter = {[indexValue]: element[indexValue]}
dbo.collection(collectionName)
.updateOne(filter, {$set:element},{upsert:true})
.then( result => db.close())
.catch( err => console.log(err));
})
})
.catch( err => console.log(err));
}
function updateDBHistorical(){
}
}

Related

MongoError: Document must be a valid JavaScript object

I have a problem where MongoDB says that my object is not a valid JavaScript Object, Even though it is! This has been staying for days!
Basically, this is an account system that uses MongoDB's client, and the ObjectId for the ID.
I want to be able to fix the MongoError that says object (sent to updateOne, not filter) is not a valid JavaScript object.
Here is the code:
const { MongoClient, ObjectId } = require("mongodb");
const fs = require("node:fs");
const uri = "mongodb://127.0.0.1:27017";
if (!fs.existsSync("./db")) {fs.mkdirSync("./db")};
const client = new MongoClient(uri,{ useUnifiedTopology: true });
async function conn() {
await client.connect();
}
conn();
const database = client.db("login");
const accs = database.collection("accounts");
const myfil = {
_id: new ObjectId('63b6441832087ccc7e3edea2')
};
const users = accs.findOne(myfil);
const path = require("node:path");
const bcrypt = require('bcrypt');
const env = process.env;
var saltRounds = 10;
const AddSet = class AddSet {
constructor(user,pass) {
console.log(pass);
this.set = {[user]:pass};
this.set = Object.keys(this.set).reduce((acc, key) => {
acc[key.toString()] = this.set[key];
return acc;
}, {});
console.log(this.set);
return this.set;
}
}
const Account = class Account {
constructor(user,password) {
conn();
if (!users[user]) {
conn();
accs.updateOne(myfil,bcrypt.hash(password, saltRounds, function(err, hash)
{
try {
var a = ""+user;
return new AddSet(a.toString(),hash);
} catch(err) {
console.error("bcrypt",err);
}
}));
this.assetDir = path.join(path.join(env.SAVED_FOLDER,"/"+this.user),"/assets");
this.metaDir = this.assetDir + '/meta';
this.starterDir = path.join(path.join(env.SAVED_FOLDER,"/"+this.user),"/starters");
this.videoDir = path.join(path.join(env.SAVED_FOLDER,"/"+this.user),"/videos");
var fs = require('fs');
if (!fs.existsSync(this.assetDir)) fs.mkdirSync(this.assetDir, { recursive: true });
if (!fs.existsSync(this.starterDir)) fs.mkdirSync(this.assetDir, { recursive: true });
if (!fs.existsSync(this.videoDir)) fs.mkdirSync(this.assetDir, { recursive: true });
}
}
getAssetDir() {
return this.assetDir;
}
getStarterDir() {
return this.starterDir;
}
getVideoDir() {
return this.videoDir;
}
getMetaDir() {
return this.metaDir;
}
checkSession(pswd) {
conn();
bcrypt.compare(pswd, users[this.user], function(err, result) {
if (result) return true;
else return false;
});
}
}
module.exports = { Account, users };
I tried fixing it, making the keys strings, removing the $set, and it did not work.

google cloud function error DEADLINE EXCEEDED

I'm trying to write a simple pub/sub triggered cloud function to update my Firestore collection of stocks.I'm getting a bunch of weird error messages with one prominent of Error: 4 DEADLINE_EXCEEDED: Deadline exceeded. Whats even more strange that some stocks gets updated correctly while others don't. Im new to Javascript/Typescript so obviously I have some misunderstanding about how to return promises in this case here. The idea here is very simple loop through each ticker in the collection make a request to updated data then update existing document data and save it
export const updateChart = functions.pubsub.schedule('35 16 * * 1-5').timeZone('America/New_York').onRun(async(_context) => {
const key = functions.config().services.key as string
const db = admin.firestore()
const charts5DRef = db.collection("charts5D")
var needsUpdate : boolean = true
// I cut off some unrelated code for brevity sake
if (needsUpdate){
const snapshot = await charts5DRef.get()
var path = ``
const timestamp = Date.now() / 1000
return snapshot.forEach(async function(document){ // this could contain 100's of stock tickers
const ticker = document.id
const docData = document.data()
var labels = docData.labels as [string]
var marketNotional = docData.marketNotional as [number]
var marketTrades = docData.marketNumberOfTrades as [number]
var dates = docData.dates as [string]
var closings = docData.close as [number]
var volume = docData.marketVolume as [number]
path = `apiUrl to get data`
const options = {
method : 'GET',
uri : path,
resolveWithFullResponse : true,
}
await req(options).then(async(response)=>{
if(response.statusCode === 200){
const resultData = JSON.parse(response.body)
const updatedPrices = resultData as [IntradayPrice]
updatedPrices.forEach(function(value){
if(value.close !== undefined){
closings.splice(0,1)
marketTrades.splice(0,1)
marketNotional.splice(0,1)
labels.splice(0,1)
dates.splice(0,1)
volume.splice(0,1)
closings.push(value.close)
dates.push(value.date)
if(value.label !== undefined){ labels.push(value.label) } else { labels.push("") }
if(value.marketNotional !== undefined) { marketNotional.push(value.marketNotional) } else { marketNotional.push(0) }
if(value.marketNumberOfTrades !== undefined) { marketTrades.push(value.marketNumberOfTrades) } else { marketTrades.push(0) }
if(value.marketVolume !== undefined) { volume.push(value.marketVolume) } else { volume.push(0) }
}
})
await charts5DRef.doc(ticker).set({lastUpdate : timestamp,close : closings, labels : labels, marketVolume : volume,marketNumberOfTrades : marketTrades, marketNotional : marketNotional, dates : dates}).then(()=>{
console.log(`Updated ${ticker} 5Dchart successfully`)
}).catch((error)=>{
console.log(error)
})
}
}).catch((error)=>{
console.log(error)
})
})
}else{
console.log("Markets closed")
return
}
})
ok so this solved the errors
export const updateChart = functions.pubsub.schedule('35 16 * * 1-5').timeZone('America/New_York').onRun(async(_context) => {
const key = functions.config().services.key as string
const db = admin.firestore()
const charts5DRef = db.collection("charts5D")
var needsUpdate : boolean = false
if (needsUpdate){
const snapshot = await charts5DRef.get()
var path = ``
const timestamp = Date.now() / 1000
const promises : bird<void>[] = []
snapshot.forEach(async function(document){
const ticker = document.id
const docData = document.data()
var labels = docData.labels as [string]
var marketNotional = docData.marketNotional as [number]
var marketTrades = docData.marketNumberOfTrades as [number]
var dates = docData.dates as [string]
var closings = docData.close as [number]
var volume = docData.marketVolume as [number]
path = `https://cloud.iexapis.com/stable/stock/${ticker}/intraday-prices?token=${key}&chartInterval=10`
const options = {
method : 'GET',
uri : path,
resolveWithFullResponse : true,
}
const promise = req(options).then(async(response)=>{
if(response.statusCode === 200){
const resultData = JSON.parse(response.body)
const updatedPrices = resultData as [IntradayPrice]
updatedPrices.forEach(function(value){
if(value.close !== undefined){
closings.splice(0,1)
marketTrades.splice(0,1)
marketNotional.splice(0,1)
labels.splice(0,1)
dates.splice(0,1)
volume.splice(0,1)
closings.push(value.close)
dates.push(value.date)
if(value.label !== undefined){ labels.push(value.label) } else { labels.push("") }
if(value.marketNotional !== undefined) { marketNotional.push(value.marketNotional) } else { marketNotional.push(0) }
if(value.marketNumberOfTrades !== undefined) { marketTrades.push(value.marketNumberOfTrades) } else { marketTrades.push(0) }
if(value.marketVolume !== undefined) { volume.push(value.marketVolume) } else { volume.push(0) }
}
})
await charts5DRef.doc(ticker).set({lastUpdate : timestamp,close : closings, labels : labels, marketVolume : volume,marketNumberOfTrades : marketTrades, marketNotional : marketNotional, dates : dates}).then(()=>{
console.log(`Updated ${ticker} 5Dchart successfully`)
}).catch((error)=>{
console.log(error)
})
}
}).catch((error)=>{
console.log(error)
})
promises.push(promise)
})
return Promise.all(promises).then(()=>{
console.log("All good")
}).catch((error)=>{
console.log(error)
})
}else{
console.log("Markets closed")
return
}
})

Why doesn't my async function return any result?

I wrote this small program to fetch data. This however is done async. Since I nonetheless need to use the function holeVertreter(kzl) as a function in another module, I'd like to get a return value which I can eventually pass on.
Excuse my spaghetti code (I usually prettify the code when I am done with my task ...).
Credentials are stored in a file and are therefore not found in this file.
I'd like to end up with "vertreter" as a return value.
Thank you in advance.
const node = require("deasync");
const DSB = require('dsbapi');
const tabletojson = require('tabletojson');
const https = require('https');
const cred = require("./vertrCred");
const dsb = new DSB(cred["dsb"]["user"], cred["dsb"]["passw"]); //Sanitized - no Credentials here
//Stackoverflow 2332811
String.prototype.capitalize = function(lower) {
return (lower ? this.toLowerCase() : this).replace(/(?:^|\s)\S/g, function(a) { return a.toUpperCase(); });
};
function holePlan(kuerzel) {
dsb.fetch()
.then(data => {
const timetables = DSB.findMethodInData('timetable', data);
const tiles = DSB.findMethodInData('tiles', data);
var tilesStr = JSON.stringify(tiles["data"][0]["url"]);
var url = JSON.parse(tilesStr);
https.get(url, (resp) => {
let data = '';
resp.on('data', (chunk) => {
data += chunk;
});
resp.on('end',() => {
var tableasjson = tabletojson.convert(data);
var erstetab = tableasjson[0];
var zweitetab = tableasjson[1];
var drittetab = tableasjson[2];
var viertetab = tableasjson[3];
var fuenftetab = tableasjson[4];
var sechstetab = tableasjson[5];
var siebtetab = tableasjson[6];
var achtetab = tableasjson[7];
if (typeof kuerzel === "undefined")
{
var regenechse = '(Aaa|Aaa[A-Za-z?]|[A-Za-z?]Aaa)';
}
else {
var name = kuerzel.capitalize(true);
var regenechse = '('+name+'|'+name+'[A-Za-z?]|[A-Za-z?]'+name+')';
}
const regex = new RegExp(regenechse,'g');
var sammel = Object.assign(drittetab,fuenftetab);
var z= 0;
var vertreter = {}
var y = JSON.parse(JSON.stringify(sammel));
for (i=0;i<y.length;i++) {
if (typeof y[i].Vertreter =='undefined') {
}
else {
if(y[i].Vertreter.match(regex))
{
z += 1;
vertreter[z] = y[i];
}
}
}
if (z == 0) {
// console.log("Es gibt nichts zu vertreten");
}
else {
//console.log("Es werden "+z+" Stunden vertreten");
return (vertreter);
} ;
});
})
})
.catch(e => {
// An error occurred :(
console.log(e);
});
}
//Stackoverflow
function warte(promise) {
var done = 0;
var result = null;
promise.then(
function (value) {
done = 1;
result = value;
return (value);
},
function (reason) {
done = 1;
throw reason;
}
);
while (!done)
node.runLoopOnce();
return (result);
}
function holeVertretung(kzl) {
var aufgabe = new Promise((resolve,reject) => {
setTimeout(resolve,1000,holePlan(kzl));
});
var ergebnis = warte(aufgabe);
if (typeof ergebnis === "undefined") {
console.log("Mist");
}
else {
console.log(ergebnis);
}
return ergebnis;
}
holeVertretung("Aaa");
That's not the right way to work with promises. If you do such infinite loop, it beats the whole purpose of using promises. Instead, return value from the promise, and use async-await like this:
function warte(promise) {
var done = 0;
var result = null;
return promise.then(
...
}
async function holeVertretung(kzl) {
var aufgabe = new Promise((resolve, reject) => {
setTimeout(resolve, 1000, holePlan(kzl));
});
var ergebnis = await warte(aufgabe);
...
If async-await does not work for some reason, use then clause:
warte(aufgabe).then(value => {
var ergebnis = value;
});

Unable to connect MongoDB with nodejs

I am creating an application with nodejs and electron. I want to use mongodb to store data and stuff but i am having a bit of a difficulty while trying to connect mongodb. I am using robomongo and i can connect to database with it but when i try to connect in my js file i get an error. error is:
Uncaught TypeError: mongo.open is not a function
here's my code:
var MongoClient = require('mongodb').MongoClient
var Server = require('mongodb').Server;
var mongo = new MongoClient(new Server('localhost', 27017));
mongo.open((err, mongo) => {
var db = mongo.db('aab');
console.log('Connected!');
if(err) throw err;
mongo.close();
});
thanks for helping :)
There is no open. Here is an example:
let mongoUri;
let dbName;
const collectionName = 'collection';
let mongo = {};
const setEnv = (req) => {
mongoUri = req.env.mongoUri;
dbName = req.env.dbName;
if (dbName === '' || mongoUri === '') {
console.warn('no request.env');
}
};
const mongoDbConnection = async (req) => {
setEnv(req);
try {
if ( !mongo.client || mongo.uri !== mongoUri || !mongo.client.isConnected() || !mongo.db ) {
mongo.uri = mongoUri;
let options = { useNewUrlParser: true };
const c = await MongoClient.connect(mongo.uri, options);
mongo.client = c ;
mongo.db = await mongo.client.db(dbName);
}
return mongo.db;
} catch (e) {
throw `Mongo connection failed \n ${e}`;
}
};
const getCollection = async (req) => {
try {
const db = await mongoDbConnection(req);
console.log('collection connected');
return await db.collection(collectionName);
} catch (e) {
throw e;
}
};

Node.js, Express.js and sqllite3.js

I have a problem with object attributes in Node.js. While attributes are set as expected in the test object, the same doesn't work for my articles object. The difference I see is that the functions of articles are called asynchronously. I must confess that I am a bit lost...
Here is app.js, that instantiates the test object and the articles object.
/**
* Load express.
*/
var express = require('express');
var app = express();
/**
* Load articles.
*/
var articles = require('./article.js');
var l_articles = new articles.Articles();
var test = require('./test.js');
var l_test = new test.Test();
app.get('/', function (req, res) {
console.log(l_test.get());
l_test.set('it is', 'midnight');
console.log(l_test.get());
articles.Articles.get(1);
res.send('OK');
})
app.listen(3001, function () {
console.log(l_test.get());
l_test.set('goodbye', 'sunshine');
console.log(l_test.get());
})
Here is my fairly simple test.js :
var app = require('./app.js');
function Test() {
this.timestamp = new Date();
console.log(this.timestamp);
this.attribute1 = 'hello';
this.attribute2 = 'world';
}
Test.prototype.get = function() {
console.log(this.timestamp);
return (this.attribute1 + ' ' + this.attribute2);
}
Test.prototype.set = function(p_param1, p_param2) {
console.log(this.timestamp);
this.attribute1 = p_param1;
this.attribute2 = p_param2;
}
module.exports = {
Test: Test
};
Here is my fairly simple article.js :
var sqlite3 = require('sqlite3').verbose();
var app = require('./app.js');
function Articles(p_id) {
this.timestamp = new Date();
console.log(this.timestamp);
this.db = new sqlite3.Database('./gescom.sqlite');
if (p_id == undefined) {
this.db.all('SELECT * FROM T_ARTICLE', this.load);
} else {
this.db.all('SELECT * FROM T_ARTICLE WHERE id = ' & p_id, this.load);
}
}
Articles.prototype.load = function(p_err, p_rows) {
console.log(this.timestamp);
var ids = [];
var articles = [];
p_rows.forEach(function(p_row) {
ids.push(p_row.ID);
articles.push([p_row.ID, p_row.SHORT_NAME]);
});
this.ids = ids;
this.articles = articles;
console.log(this.ids.length + ' articles loaded from database.');
}
Articles.prototype.get = function (p_id) {
console.log(this.timestamp);
var l_return;
if ((this.ids == undefined) || (this.articles == undefined)) {
console.log('No articles loaded from database.');
} else {
console.log(this.ids.length + ' articles loaded from database.');
if (p_id == undefined) {
l_return = this.articles;
} else {
if (this.ids.indexOf(p_id) != undefined) {
l_return = (this.articles[this.ids.indexOf(p_id)]);
} else {
l_return = undefined;
}
}
}
return l_return;
}
module.exports = {
Articles: Articles
};

Resources