Switching databases in Mongo on ExpressJS application without reconnecting each time - node.js

I want to preface by saying I have looked for this issue online and found solutions using mongoose, I do not use that so I don't know how well it translate here. Plus, I also using async/await instead of .then, so my code is different.
My question is that the API I have made in ExpressJS is super slow compared to the same API I have in Flask. I tried seeing why this is an issue, since JS is supposed to be faster than Python. I noticed it was due to me connecting to Mongo each time. I need to do this as my MongoDB has a different databases for different clients, so I cannot just have 1 constant connection to one database. Below is my Mongo code.
const {MongoClient} = require('mongodb');
const client = new MongoClient("mongodb+srv://abc#xyz.mongodb.net/<Cluster");
class MongoMethods {
constructor(company, collection){
this.company = company;
this.collection = collection;
}
async get(accessParameters){
try{
await client.connect();
const db = client.db(this.company);
const collection = db.collection(this.collection);
const val = await collection.findOne({"_id":accessParameters["at"]});
return val
} finally {
await client.close();
}
}
async putIn(putParameters, type){
try{
await client.connect();
const db = client.db(this.company);
const collection = db.collection(this.collection);
var existing = await collection.findOne({"_id":putParameters["at"]});
if(type === "array"){
var toAdd = existing !== undefined && putParameters["in"] in existing? existing[putParameters["in"]] : [];
toAdd.push(putParameters["value"]);
}else if(type === "dict"){
var toAdd = existing !== undefined && putParameters["in"] in existing? existing[putParameters["in"]] : {};
toAdd[putParameters["key"]] = putParameters["value"];
}else{
var toAdd = putParameters["value"];
}
await collection.updateOne({"_id":putParameters["at"]}, {"$set": {[putParameters["in"]]: toAdd}}, {upsert: true});
} finally {
await client.close();
}
}
async remove(removeParameters){
try{
await client.connect();
const db = client.db(this.company);
const collection = db.collection(this.collection);
if(removeParameters["key"] !== undefined){
await collection.updateOne({"_id":removeParameters["at"]}, {"$unset": {[removeParameters["in"] + "." + removeParameters["key"]] : ""}})
}else if(removeParameters["in"] !== undefined){
await collection.updateOne({"_id":removeParameters["at"]}, {"$unset": {[removeParameters["in"]] : ""}})
}else{
await collection.deleteOne({"_id":removeParameters["at"]})
}
}finally{
await client.close();
}
}
}
module.exports.MongoMethods = MongoMethods;
Below is how functions in my ExpressJS file look (I cannot post this file so this is an excerpt):
var express = require('express');
var cors = require('cors')
const bodyParser = require('body-parser');
const { validate, ValidationError } = require('express-superstruct');
const { MongoMethods } = require('./mongo.js')
let app = express()
app.use(cors())
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true }));
app.route('/enp1/')
.get(validate({uid : "string", site : "string"}), function (req, res) {
var args = req.query
var mongoClient = new MongoMethods(args.site, "collection_1")
mongoClient.get({at : args.uid}).catch(console.dir).then(result => {
if(result == undefined){ // This may need to take into account an empty dict
res.status(404).json({"Database Error" : "UID does not exists or site is not a valid website"})
}
res.status(200).json(result)
})
})
app.route('/enp2/')
.put(validate({site : "string", uid : "string", time : "string"}), function (req, res) {
var args = req.body;
var mongoClient = new MongoMethods(args.site, "time")
mongoClient.putIn({at : args.uid, in : "time", value : parseInt(args.time)}, "int").catch(console.dir).then(result => {
res.status(200).end()
})
})
It seems like this code is connecting to Mongo each time as I have to initialize the MongoMethods object each time. Can I prevent it from trying to connect each time so that my API doesn't have slow speeds? When I compare speeds, JS endpoints without Mongo are 50% faster than their Python counterpart but when using Mongo endpoints, it is around 300ms slower.
Let me know if you need anymore clarification.
Thanks in advance!
Edit 1: I wanted to mention, the API runs on AWS API Gateway and AWS Lambda#Edge functions.

Related

WA business api nodejs

I have a problem with my nodejs code and the connection to the official whatsapp business api.
The bot connects the webhook correctly, the messages arrive to the server correctly but the code I have implemented to make it respond is not being effective, I checked the code from top to bottom but I can't find the fault.
I leave you the codes so you have more context:
whatsappController.js:
const fs = require("fs");
const myConsole = new console.Console(fs.createWriteStream("./logs.txt"));
const whatsappService = require("../services/whatsappService")
const VerifyToken = (req, res) => {
try {
var accessToken = "456E7GR****************************";
var token = req.query["hub.verify_token"];
var challenge = req.query["hub.challenge"];
if(challenge != null && token != null && token == accessToken){
res.send(challenge);
}
else{
res.status(400).send();
}
} catch(e) {
res.status(400).send();
}
}
const ReceivedMessage = (req, res) => {
try {
var entry = (req.body["entry"])[0];
var changes = (entry["changes"])[0];
var value = changes["value"];
var messageObject = value["messages"];
if(typeof messageObject != "undefined"){
var messages = messageObject[0];
var text = GetTextUser(messages);
var number = messages["from"];
myConsole.log("Message: " + text + " from: " + number);
whatsappService.SendMessageWhatsApp("The user say: " + text, number);
myConsole.log(messages);
myConsole.log(messageObject);
}
res.send("EVENT_RECEIVED");
}catch(e) {
myConsole.log(e);
res.send("EVENT_RECEIVED");
}
}
function GetTextUser(messages){
var text = "";
var typeMessage = messages["type"];
if(typeMessage == "text"){
text = (messages["text"])["body"];
}
else if(typeMessage == "interactive"){
var interactiveObject = messages["interactive"];
var typeInteractive = interactiveObject["type"];
if(typeInteractive == "button_reply"){
text = (interactiveObject["button_reply"])["title"];
}
else if(typeInteractive == "list_reply"){
text = (interactiveObject["list_reply"])["title"];
}else{
myConsole.log("sin mensaje");
}
}else{
myConsole.log("sin mensaje");
}
return text;
}
module.exports = {
VerifyToken,
ReceivedMessage
}
The second file is whatsappService which I make the connection with the api using the token and I also send the format of the message I want to send when I receive a hello for example...
const https = require("https");
function SendMessageWhatsApp(textResponse, number){
const data = JSON.stringify({
"messaging_product": "whatsapp",
"recipient_type": "individual",
"to": number,
"type": "text",
"text": {
"preview_url": false,
"body": textResponse
}
});
const options = {
host:"graph.facebook.com",
path:"/v15.0/1119744*************/messages",
method:"POST",
body:data,
headers: {
"Content-Type":"application/json",
Authorization:"Bearer EAAWNbICfuWEBAK5ObPbD******************************************************"
}
};
const req = https.request(options, res => {
res.on("data", d=> {
process.stdout.write(d);
});
});
req.on("error", error => {
console.error(error);
});
req.write(data);
req.end();
}
module.exports = {
SendMessageWhatsApp
};
Then I declare the routes for the get (to check token) and post (to receive and reply to messages) methods:
const expres = require("express");
const router = expres.Router();
const whatsappController = require("../controllers/whatsappControllers");
router
.get("/", whatsappController.VerifyToken)
.post("/", whatsappController.ReceivedMessage)
module.exports = router;
Last but not least the index file for the code to run correctly:
const express = require("express");
const apiRoute = require("./routes/routes");
const app = express();
const PORT = process.env.PORT || 3000
app.use(express.json());
app.use("/whatsapp", apiRoute);
app.listen(PORT, () => (console.log("El puerto es: " + PORT)));
I should clarify that I did the tests with Postman and they were all successful, it responds and receives messages correctly, finally I did the tests by uploading the bot to the Azure service and it works without problem until it has to answer/replicate the user's message.
The bot is not responding to the user when he talks to it but everything arrives correctly to the server and it processes it with a 200 response. I attach the evidence that there is no problem in the reception.
Finally I must say that in the meta platform I have everything configured as specified by the same platform, I have already configured the api to answer the messages through the webhooks and everything is correct, I just can't get the bot to answer correctly.
The bot is hosted in the Azure service.
Solved: some numbers have a problema with the api of WAB in my country (Argentina) the phone numbers start in +54 9 11. The problem is the 9 in the phone number, and this have a conflict in the servers of meta, Solution quit number 9 to numbers of this country and the message will send to user.

API Request passing ID fetched from mongodb

I´m a Java Dev so I need help from NodeJS guys!
Task: create a script that retrieves '_id', 'document', and 'corporateName' from MongoDB, then take the retrieved '_id', and pass it as a parameter to an API request. The last part should be taking 'document', 'corporateName' + 'client_id', 'client_secret' and export it into a single csv file.
It might be a very simple script! Therefore I´ve done this till now:
const {MongoClient} = require('mongodb');
const uri = "mongodb+srv://<privateInfo>/";
const client = new MongoClient(uri);
async function run() {
try {
const database = client.db("merchant-profile");
const ecs = database.collection("merchant_wallet");
const api = `https://<prodAPI>/v1/merchant/wallet/${id}/oauth2`;
const ecOpt = {_id: 1, document: 1, corporateName: 1};
const credOpt = {client_id: 1, client_secret: 1};
const ec = ecs.find({}).project(ecOpt);
let id = ec.forEach(id => cred._id);
const cred = api.find({}).project(credOpt);
await cred.forEach(console.dir);
} finally {
await client.close();
}
}
run().catch(console.dir);
I´m trying to understand how can I take '_id' fetched in 'ec' and pass it as a param to the 'cred' call.
This would already be awesome!
If you could help me out with the CSV issue as well it would be perfect.
So I don´t want just the answer, but understand how to do this.
Thank you all in advance!
This is the way I found to do it:
const { default: axios } = require("axios");
const { MongoClient } = require("mongodb");
const uri = "mongodb+srv://admin:sLKJdsdRp4LrsVtLsnkR#pp-core-prd.fy3aq.mongodb.net/";
const client = new MongoClient(uri);
async function run() {
try {
const database = client.db("merchant-profile");
const ecs = database.collection("merchant_wallet");
const data = [];
await ecs.find({}).forEach(async function teste(response) {
const id = response._id;
const api = `https://api.pedepronto.com.br/v1/merchant/wallet/${id}/oauth2`;
try{
const res = await axios.get(api);
data.push({client_secret: res.data[0].client_secret, client_id: res.data[0].client_id})
}catch(e){
console.log(e);
}
})
} finally {
await client.close();
}
}
run().catch(console.dir);
It iterates over the find method and appends the fetched id to the uri.

Parameter obj to Document() must be an object when trying to convert array to mongoose document with redis

I have using redis to cache my queries. Its working fine with object but not when i get array. It gives me an error **"Parameter "obj" to Document() must be an object, got kids", **. It also happens with count query. Here is my code :
const mongoose = require("mongoose");
const redis = require("redis");
const util = require("util");
const client = redis.createClient(process.env.REDIS_URL);
client.hget = util.promisify(client.hget);
const exec = mongoose.Query.prototype.exec;
mongoose.Query.prototype.cache = async function (options = {}) {
this.useCache = true;
this.hashKey = JSON.stringify(options.key || "");
this.time = JSON.stringify(options.time || 36000);
return this;
};
mongoose.Query.prototype.exec = async function () {
if (!this.useCache) {
return exec.apply(this, arguments);
}
const key = JSON.stringify(
Object.assign({}, this.getQuery(), {
collection: this.mongooseCollection.name,
})
);
// client.flushdb(function (err, succeeded) {
// console.log(succeeded); // will be true if successfull
// });
const cacheValue = await client.hget(this.hashKey, key);
if (cacheValue) {
const doc = JSON.parse(cacheValue);
/*
this.model refers to the Class of the corresponding Mongoose Model of the query being executed, example: User,Blog
this function must return a Promise of Mongoose model objects due to the nature of the mongoose model object having other
functions attached once is created ( validate,set,get etc)
*/
console.log("Response from Redis");
console.log(doc);
console.log(Array.isArray(doc));
return Array.isArray(doc)
? doc.map((d) => new this.model(d))
: new this.model(doc);
}
//await the results of the query once executed, with any arguments that were passed on.
const result = await exec.apply(this, arguments);
client.hset(this.hashKey, key, JSON.stringify(result));
client.expire(this.hashKey, this.time);
console.log("Response from MongoDB");
return result;
};
module.exports = {
clearHash(hashKey) {
client.del(JSON.stringify(hashKey));
},
};
Data in redis - [ 'kids', 'men', 'women' ]
Query - const collectionType = await Product.find() .distinct("collectionType") .cache({ key: "COLLECTION_TYPE" });
can i anyone please tell me what i am doing wrong?
I have solved by directly returning the doc and its working fine. Not sure if it is the right way if i directly do return doc then sending data from redis only

Best practice running queries in Node.js with MongoDB driver 3.6?

The official documentation of the Node.js Driver version 3.6 contains the following example for the .find() method:
const { MongoClient } = require("mongodb");
// Replace the uri string with your MongoDB deployment's connection string.
const uri = "mongodb+srv://<user>:<password>#<cluster-url>?w=majority";
const client = new MongoClient(uri);
async function run() {
try {
await client.connect();
const database = client.db("sample_mflix");
const collection = database.collection("movies");
// query for movies that have a runtime less than 15 minutes
const query = { runtime: { $lt: 15 } };
const options = {
// sort returned documents in ascending order by title (A->Z)
sort: { title: 1 },
// Include only the `title` and `imdb` fields in each returned document
projection: { _id: 0, title: 1, imdb: 1 },
};
const cursor = collection.find(query, options);
// print a message if no documents were found
if ((await cursor.count()) === 0) {
console.log("No documents found!");
}
await cursor.forEach(console.dir);
} finally {
await client.close();
}
}
To me this somewhat implies that I would have to create a new connection for each DB request I make.
Is this correct? If not, then what is the best practise to keep the connection alive for various routes?
You can use mongoose to set a connection with your database.
mongoose.connect('mongodb://localhost:27017/myapp', {useNewUrlParser: true});
then you need to define your models which you will use to communicate with your DB in your routes.
const MyModel = mongoose.model('Test', new Schema({ name: String }));
MyModel.findOne(function(error, result) { /* ... */ });
https://mongoosejs.com/docs/connections.html
It's 2022 and I stumbled upon your post because I've been running into the same issue. All the tutorials and guides I've found so far have setups that require reconnecting in order to do anything with the Database.
I found one solution from someone on github, that creates a class to create, save and check if a client connection exist. So, it only recreates a client connection if it doesn't already exist.
const MongoClient = require('mongodb').MongoClient
class MDB {
static async getClient() {
if (this.client) {
return this.client
}
this.client = await MongoClient.connect(this.url);
return this.client
}
}
MDB.url='<your_connection_url>'
app.get('/yourroute', async (req, res) => {
try {
const client = await MDB.getClient()
const db = client.db('your_db')
const collection = db.collection('your_collection');
const results = await collection.find({}).toArray();
res.json(results)
} catch (error) {
console.log('error:', error);
}
})

How to use MongoDB locally and directline-js for state management in Bot Framework using NodeJs and Mongoose?

I am maintaining the bot state in a local MongoDB storage. When I am trying to hand-off the conversation to an agent using directline-js, it shows an error of BotFrameworkAdapter.sendActivity(): Missing Conversation ID. The conversation ID is being saved in MongoDB
The issue is arising when I change the middle layer from Array to MongoDB. I have already successfully implemented the same bot-human hand-off using directline-js with an Array and the default Memory Storage.
MemoryStorage in BotFramework
const { BotFrameworkAdapter, MemoryStorage, ConversationState, UserState } = require('botbuilder')
const memoryStorage = new MemoryStorage();
conversationState = new ConversationState(memoryStorage);
userState = new UserState(memoryStorage);
Middle Layer for Hand-Off to Agent
case '#connect':
const user = await this.provider.connectToAgent(conversationReference);
if (user) {
await turnContext.sendActivity(`You are connected to
${ user.userReference.user.name }\n ${ JSON.stringify(user.messages) }`);
await this.adapter.continueConversation(user.userReference, async
(userContext) => {
await userContext.sendActivity('You are now connected to an agent!');
});
}
else {
await turnContext.sendActivity('There are no users in the Queue right now.');
}
The this.adapter.continueConversation throws the error when using MongoDB.
While using Array it works fine. The MongoDB and Array object are both similar in structure.
Since this works with MemoryStorage and not your MongoDB implementation, I'm guessing that there's something wrong with your MongoDB implementation. This answer will focus on that. If this isn't the case, please provide your MongoDb implementation and/or a link to your repo and I can work off that.
Mongoose is only necessary if you want to use custom models/types/interfaces. For storage that implements BotState, you just need to write a custom Storage adapter.
The basics of this are documented here. Although written for C#, you can still apply the concepts to Node.
1. Install mongodb
npm i -S mongodb
2. Create a MongoDbStorage class file
MongoDbStorage.js
var MongoClient = require('mongodb').MongoClient;
module.exports = class MongoDbStorage {
constructor(connectionUrl, db, collection) {
this.url = connectionUrl;
this.db = db;
this.collection = collection;
this.mongoOptions = {
useNewUrlParser: true,
useUnifiedTopology: true
};
}
async read(keys) {
const client = await this.getClient();
try {
var col = await this.getCollection(client);
const data = {};
await Promise.all(keys.map(async (key) => {
const doc = await col.findOne({ _id: key });
data[key] = doc ? doc.document : null;
}));
return data;
} finally {
client.close();
}
}
async write(changes) {
const client = await this.getClient();
try {
var col = await this.getCollection(client);
await Promise.all(Object.keys(changes).map((key) => {
const changesCopy = { ...changes[key] };
const documentChange = {
_id: key,
document: changesCopy
};
const eTag = changes[key].eTag;
if (!eTag || eTag === '*') {
col.updateOne({ _id: key }, { $set: { ...documentChange } }, { upsert: true });
} else if (eTag.length > 0) {
col.replaceOne({ _id: eTag }, documentChange);
} else {
throw new Error('eTag empty');
}
}));
} finally {
client.close();
}
}
async delete(keys) {
const client = await this.getClient();
try {
var col = await this.getCollection(client);
await Promise.all(Object.keys(keys).map((key) => {
col.deleteOne({ _id: key });
}));
} finally {
client.close();
}
}
async getClient() {
const client = await MongoClient.connect(this.url, this.mongoOptions)
.catch(err => { throw err; });
if (!client) throw new Error('Unable to create MongoDB client');
return client;
}
async getCollection(client) {
return client.db(this.db).collection(this.collection);
}
};
Note: I've only done a little testing on this--enough to get it to work great with the Multi-Turn-Prompt Sample. Use at your own risk and modify as necessary.
I based this off of a combination of these three storage implementations:
memoryStorage
blobStorage
cosmosDbStorage
3. Use it in your bot
index.js
const MongoDbStorage = require('./MongoDbStorage');
const mongoDbStorage = new MongoDbStorage('mongodb://localhost:27017/', 'testDatabase', 'testCollection');
const conversationState = new ConversationState(mongoDbStorage);
const userState = new UserState(mongoDbStorage);

Resources