How can I execute db.copyDatabase through NodeJS's MongoDB native driver? - node.js

I do have a shell script that invokes
mongo --eval "db.copyDatabase('somedatabase', 'somedatabase_duplicate', 'sourcehost')"
to copy a database.
Currently I am stuck with doing the same from within a Node.JS application. Calling
mongoCommand = `db.copyDatabase("somedatabase", "somedatabase_duplicate", "localhost")`;
db.command(mongoCommand, function(commandErr, data) {
if(!commandErr) {
log.info(data);
} else {
log.error(commandErr.errmsg);
}
});
Always resulsts in a "no such command" error message.
Edit for clarification: Using db.admin().command() results in the same problem and using the command suggested in enter link description here, too.
What's the correct way to call this command or, alternatively, to clone a database from Node.JS?

Well, you are trying to copy database which is administration operation so have to do with admin account. Again, to copy database command is copydb.
try running this command in shell, db.copyDatabase and you'll see source of command.
try:
var assert = require('assert');
var MongoClient = require('mongodb').MongoClient;
var url = 'mongodb://localhost:27017/test';
MongoClient.connect(url, function(err, db) {
if (err) {
console.log(err);
}
else {
var mongoCommand = { copydb: 1, fromhost: "localhost", fromdb: "test", todb: "test_dup" };
var admin = db.admin();
admin.command(mongoCommand, function(commandErr, data) {
if (!commandErr) {
console.log(data);
} else {
console.log(commandErr.errmsg);
}
db.close();
});
}
});

//core modules
const assert = require('assert')
const MongoClient = require('mongodb').MongoClient;
const moment = require('moment');
const mongo = require('mongodb')
//custom modules
let { ip, port, database } = require('./dbUpgradeConfig')
const url = `mongodb://${ip}:${port}`
let todayDate = moment().format('DD/MM/YYYY HH:mm')
console.log(todayDate)
const myDate = new Date()
console.log(myDate)
var d = Date(Date.now());
// Converting the number of millisecond in date string
a = d.toString()
// Options for mongoDB
const mongoOptions = { useNewUrlParser: true }
let db
//TODO: handle reconnect
let connect = () => {
return new Promise((resolve, reject) => {
if (db) resolve()
else {
mongo.connect(url, mongoOptions, (err, client) => {
if (err) reject(err)
else {
db = client.db(database)
resolve()
}
})
}
})
}
/**
* #description create duplicate database from current database in mongodb
*/
let CloneDb = () => {
return new Promise((resolve, reject) => {
connect()
.then(() => {
console.log(db)
let mongoCommand = { copydb: 1, fromhost: "localhost", fromdb: "db_name", todb: "db_name_duplicate" }
let adminDB = db.admin()
adminDB.command(mongoCommand, function (commandErr, data) {
if (!commandErr) {
console.log(data)
} else {
console.log(commandErr.errmsg)
}
});
})
})
}
CloneDb().then(data => {
// debugger;
console.log("The clone db", data)
})

Related

I want to apply my MongoDB query to Javascript(select specific column)

Hi I'm working with my MongoDB and NestJS.
I was testing by query below. And it was what I expected
I only want to get '_id'. So I tested my code too.
// // This is what I use(including comments)
// const { MongoClient, ObjectID } = require('mongodb');
// const url = 'mongodb+srv://alex:~ something';
// console.log(url);
// const client = new MongoClient(url);
// // Database Name
// const dbName = 'testDB';
export async function getCreaterPubGameId(authorID: string) {
await client.connect();
console.log('Connected successfully to server : update, find or insertData');
const db = client.db(dbName);
const collection = db.collection('games');
return new Promise(function (resolve, reject) {
collection.find({ authorID }, { type: '_id' }).toArray((err, doc) => {
if (err) {
reject(err);
} else {
console.log('getAlldata : ', doc);
resolve(doc);
}
});
});
}
After I use this function, I got all the data from MongoDB.
As you see, I used the same syntax. But I got all the data.
Is there anybody who has a good idea??
You need to use projection when using MongoDb Node.js client.
export async function getCreaterPubGameId(authorID: string) {
await client.connect();
console.log('Connected successfully to server : update, find or insertData');
const db = client.db(dbName);
const collection = db.collection('games');
return new Promise(function (resolve, reject) {
collection.find({ authorID }, { "type": 1}).toArray((err, doc) => {
if (err) {
reject(err);
} else {
console.log('getAlldata : ', doc);
resolve(doc);
}
});
});
}
When you pass 1 to any field, it will be added to the prjection field and will only show those schema field. By default _id will be included if you want to not include it pass _id: 0.
export async function getCreaterPubGameId(authorID: string) {
await client.connect();
console.log('Connected successfully to server : update, find or insertData');
const db = client.db(dbName);
const collection = db.collection('games');
// I have to use 'projection'
return new Promise(function (resolve, reject) {
collection.find({ authorID }, { projection: { _id: true }).toArray((err, doc) => {
if (err) {
reject(err);
} else {
console.log('getAlldata : ', doc);
resolve(doc);
}
});
});
}
Link : https://mongodb.github.io/node-mongodb-native/3.2/api/Collection.html#find

function returning nothing in mongodb query

I'm not capturing the connect function to return in the getAllowedEmails function, when I do console.log in allowedEmails, it returns the emails correctly, but when I assign to the variable emails, it is returning empty. I think it's an async await problem but can't figured out.
static async getAllowedEmails() {
var MongoClient = require("mongodb").MongoClient;
//db url
let emails = [];
await MongoClient.connect(url, async function (err, client) {
const db = client.db("data-admin");
var myPromise = () => {
return new Promise((resolve, reject) => {
db.collection("users")
.find({})
.toArray(function (err, data) {
err ? reject(err) : resolve(data);
});
});
};
var result = await myPromise();
client.close();
let allowedEmails = [];
result.map((email) => allowedEmails.push(email.email));
console.log(allowedEmails)
emails = allowedEmails;
});
console.log(emails)
return emails;
}
Your code has couple of issues, I have fixed few and enhanced it, given below is the basic code, try to test it and if everything works then enhance it as needed:
const MongoClient = require("mongodb").MongoClient;
async function getAllowedEmails() {
let client;
try {
const allowedEmails = [];
client = await MongoClient.connect(url);
const db = client.db("data-admin");
const result = await db.collection("users").find({}).toArray();
result.map((email) => allowedEmails.push(email.email));
console.log(allowedEmails)
client.close();
return allowedEmails;
} catch (error) {
(client) && client.close();
console.log(error)
throw error;
}
}

Moving nodejs MongoDB connection code to another file

Moving the database connection code another nodejs file, no connection object is returned.
I can write data to MongoDB in nodejs. All db connection code is written in a single .js file.
Now I try to seperate the db connection code to another .js file, and now it seems that no connection can be made successfully.
Here is the working code in 1 .js file:
const mongoDBIP = '192.168.1.71';
const mongoDBPort = 27017;
const mongo= require('mongodb').MongoClient;
const mongoURL = 'mongodb://<mongo admin>:<password>#'+`${mongoDBIP}`+':'+`${mongoDBPort}`;
...
mongo.connect(mongoURL, { useNewUrlParser: true, useUnifiedTopology: true }, (err, db) => {
if (err) {}
var dbo = db.db(<databaseName>);
var collection = dbo.collection('messages');
collection.insertOne(message, (err, result) => {
if (err) {}
})
...
Now I break this into 2 .js files:
var _mongo = require('./mongodb.js');
var mongoDBO = _mongo.mongoDBO;
...
var dbo = mongoDBO('feathers');
console.log('DBO:' + dbo); <-- here, dbo is NULL
var collection = dbo.collection('messages');
collection.insertOne(message, (err, result) => {
if (err) {}
})
...
Here is the content of mongodb.js:
// mongodb.js
const mongoDBIP = '192.168.1.71';
const mongoDBPort = 27017;
const mongoClient = require('mongodb').MongoClient;
const mongoURL = 'mongodb://<mongo admin>:<password>#'+`${mongoDBIP}`+':'+`${mongoDBPort}`;
function mongoDBO(database) {
var dbo;
mongoClient.connect(mongoURL, { useNewUrlParser: true, useUnifiedTopology: true }, (err, dbase) => {
if (err) {}
dbo = dbase.db(database); <-- here, dbo is NULL
});
return dbo;
}
module.exports = Object.freeze({
mongoDBO
});
I expect the object "dbo" in mongodb.js would not be NULL, but it is. This occurs under both nodejs v10.x and v12.x.
Here is an implementation what Tom explained, in case you need.
mongodb.js
const mongoClient = require('mongodb').MongoClient;
const mongoDBIP = '192.168.1.71';
const mongoDBPort = 27017;
const mongoURL = 'mongodb://<mongo admin>:<password>#'+`${mongoDBIP}`+':'+`${mongoDBPort}`;
let _db;
const initDb = callback => {
if (_db) {
console.log('Db is already initialized!');
return callback(null, _db);
}
mongoClient .connect(mongoURL)
.then(client => {
_db = client;
callback(null, _db);
})
.catch(err => {
callback(err);
});
};
const getDb = () => {
if (!_db) {
throw Error('Db not initialized');
}
return _db;
};
module.exports = {
initDb,
getDb
};
Initialize it in your main file (index, app or server.js)
const mongodb = require('./mongodb');
mongodb.initDb((err, mongodb ) => {
if (err) {
console.log(err);
} else {
app.listen(3000);
}
});
And use it in your routes:
const mongodb = require('./mongodb');
mongodb.getDb()
.db()
.collection('your_collection_name')....
You are trying to return a value that does not exist yet. dbo is not set until the callback function is invoked. The original function has already returned at that point.
You must alter your mongodb.js file to export a function that takes a callback, and passes dbo to that callback when it is ready.
I recommend reading up on callbacks and asynchronous programming.
Try like this
const mongoDBIP = "192.168.1.71";
const mongoDBPort = 27017;
const mongoClient = require("mongodb").MongoClient;
const mongoURL =
"mongodb://<mongo admin>:<password>#" +
`${mongoDBIP}` +
":" +
`${mongoDBPort}`;
var mongoDBO = database =>
new Promise((resolve, reject) => {
mongoClient.connect(
mongoURL,
{ useNewUrlParser: true, useUnifiedTopology: true },
(err, dbase) => {
if (err) {
reject(err);
}
resolve(dbase);
}
);
});
module.exports = Object.freeze({
mongoDBO
});
&
var _mongo = require('./mongodb.js');
var mongoDBO = _mongo.mongoDBO;
...
try {
let dbo = mongoDBO('feathers');
console.log('DBO:' + dbo); <-- here, dbo is NULL
var collection = dbo.collection('messages');
collection.insertOne(message, (err, result) => {
if (err) {}
})
} catch (error) {
console.log(connection failed)
}
...

Error: server instance pool gets destroyed in nested db collection functions

I have searched for the solution of the error specified in title.
MongoError: server instance pool was destroyed
I believe it is because misplacement of db.close(). But I am nesting dbo.collection and unable to get the exact solution of this error.
Firstly, I am fetching data (array of ids having status 0) from database and then I am concatenating (each app-id) them one by one with URL to get desired appUrl which will be used for crawling data one by one and then crawled data is meant to be stored into another collection of mongoDB. This process will repeat for each id in the array. But my code is having error of "server instance pool gets destroyed" before storing data into collection. I am doing misplacement of db.close() but I am unable to resolve this. Please help me resolving this error
Here is my code
///* global sitehead */
const request = require('request');
const cheerio = require('cheerio');
//const response = require('response');
const fs = require('fs');
const express = require('express');
const app = express();
var MongoClient = require('mongodb').MongoClient;
var url = "mongodb://localhost:27017/";
var dateTime = require('node-datetime');
MongoClient.connect(url, {useNewUrlParser: true}, function (err, db) {
if (err) {
throw err;
} else {
var dbo = db.db("WebCrawler");
var app_id;
var appUrl;
let arr = [];
dbo.collection("Unique_Apps").find({"Post_Status": 0}, {projection: {_id: 0, App_Id: 1}}).toArray(function (err, result)
{
// console.log(result);
if (err) {
throw err;
// console.log(err);
} else {
for (var i = 0; i < result.length; i++)
{
arr[i] = result[i];
}
arr.forEach((el) => {
app_id = el.App_Id;
//console.log(app_id);
appUrl = 'https://play.google.com/store/apps/details?id=' + app_id;
console.log(appUrl);
request(appUrl, function (error, response, html) {
if (!error && response.statusCode === 200) {
//START Crawling ###########
const $ = cheerio.load(html); //cheerio
const appTitle = $('.AHFaub');
const iconUrl = $('.T75of.sHb2Xb').attr("src");
const developedBy = $('.T32cc.UAO9ie').children().eq(0);
const category = $('.T32cc.UAO9ie').children().eq(1);
//store in database collection: "Single_App_Data_Post"
var curr_obj = {App_Id: app_id, App_Name: appTitle.text(),
Icon_Url: iconUrl, Price: "Free", Developed_By: developedBy.text(),
Category: category.text()
};
dbo.collection("Single_App_Data_Post").insertOne(curr_obj, function (err, res) {
console.log("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
if (err) {
throw err;
// console.log(err);
} else {
console.log("inserted....");
} //main else
});
dbo.collection("Unique_Apps").updateOne({App_Id: app_id}, {$set: {Post_Status: 0}}, function (err, res) {
if (err)
throw err;
console.log("1 document updated");
//dbo.close();
});
} else
{
throw error;
}
});
});
}
db.close();
});
} //else
}); //mongoClient connect db
Output
The following is a good start about how to turn callback into promises. Try to use it, execute the code block, by block, understand it and then add your updateOne/insertOne requests into it.
const request = require('request');
const cheerio = require('cheerio');
const fs = require('fs');
const express = require('express');
const app = express();
const MongoClient = require('mongodb').MongoClient;
const dateTime = require('node-datetime');
// Class used to handle the database basic interractions
class DB {
constructor() {
this.db = false;
this.url = "mongodb://localhost:27017/";
}
// Do connect to the database
connect() {
return new Promise((resolve, reject) => {
MongoClient.connect(this.url, {
useNewUrlParser: true,
}, (err, db) => {
if (err) {
console.log('error mongodb connect');
return reject(err);
}
this.db = db;
return resolve(db);
});
});
}
disconnect() {
db.close();
this.db = false;
}
getCollection(name) {
return this.db.db(name);
}
}
// Get the data from the database
function getAppsIds(dbObj) {
return new Promise((resolve, reject) => {
const dbo = dbObj.getCollection('WebCrawler');
dbo.collection('Unique_Apps').find({
'Post_Status': 0,
}, {
projection: {
_id: 0,
App_Id: 1,
}
}).toArray(function(err, result) {
if (err) {
return reject(err);
}
return resolve(result);
});
});
}
function requestPlayStore(idApp) {
return new Promise((resolve, reject) => {
const appUrl = `https://play.google.com/store/apps/details?id=${app_id}`;
request(appUrl, function(error, response, html) {
if (error || response.statusCode !== 200) {
return reject(error);
}
return resolve({
response,
html,
});
});
});
}
// Do treat one id app at a time
function treatOneIdApp(dbObj, idApp) {
return requestPlayStore(idApp)
.then(({
response,
html,
}) => {
// Perform your requests here updateOne and insertOne ...
});
}
const dbObj = new DB();
dbObj.connect()
.then(() => getAppsIds(dbObj))
.then(rets => Promise.all(rets.map(x => treatOneIdApp(dbObj, x.App_Id))))
.then(() => dbObj.disconnect())
.catch((err) => {
console.log(err);
});

how to connect to mongodb synchronously in nodejs

I want to make use of the promises feature where in I can connect to mongodb synchronously and I can reuse the connection by passing it on to different modules.
Here is something that I came up with
class MongoDB {
constructor(db,collection) {
this.collection = db.collection(collection);
}
find(query, projection) {
if(projection)
return this.collection.find(query, projection);
else
return this.collection.find(query);
}
}
class Crew extends MongoDB {
constructor(db) {
super(db,'crews');
}
validate() {
}
}
I want to setup a connection somewhere in my initial code like the one below and then reuse the connection for different classes, just like how mongoose or monk does but using only the node-mongodb-native package.
MongoClient.connect(url)
.then( (err,dbase) => {
global.DB = dbase;
});
var Crew = new CrewModel(global.DB);
Crew.find({})
.then(function(resp) {
console.log(resp);
});
Right now, the db returns undefined inside the main MongoDB class and am not able to debug this one out through google or the documentation.
Edit: I had assumed that a promise was synchronous but that is not the case.
To reuse the connection I would create a module like this.
module.exports = {
connect: function(dbName, callback ) {
MongoClient.connect(dbName, function(err, db) {
_db = db;
return callback( err );
});
},
getDb: function() {
return _db;
}
};
After that you can connect to the database before starting your application
MongoConnection.connect("mongodb://localhost:27017/myDatabase", function(err){
app.listen(3000, function () {
// you code
});
});
Considering you created the module in a js file you can simply use require to get the databaseConnection
var dbConnection = require("./myMongoConnection.js");
and to get the connection use
var db = MongoConnection.getDb();
Another option using ES6 classes creates a singleton object that you can access repeatedly. It's inspired by #user3134009's answer here.
const EventEmitter = require('events');
const MongoClient = require('mongodb').MongoClient;
const config = require('config');
let _db = null;
class MongoDBConnection extends EventEmitter {
constructor() {
super();
this.emit("dbinit", this);
if (_db == null) {
console.log("Connecting to MongoDB...");
MongoClient.connect(config.dbs.mongo.url, config.dbs.mongo.options,
(err, db) => {
if (err) {
console.error("MongoDB Connection Error", err);
_db = null;
} else {
console.log("Connected to MongoDB", config.dbs.mongo.url);
db.on('close', () => { console.log("MongoDB closed", arguments); _db = null; });
db.on('reconnect', () => { console.log("MongoDB reconnected", arguments); _db = db; });
db.on('timeout', () => { console.log("MongoDB timeout", arguments); });
_db = db;
this.emit('dbconnect', _db);
}
});
}
}
getDB() {
return _db;
}
}
module.exports = new MongoDBConnection();
I have been struggling with this problem for a while, and in particular with setting up and persisting MongoDb connection in AWS lambda functions across invocations.
Thanks to #toszter answer I've finally come up with the following solution:
const mongodb = require('mongodb');
const config = require('./config.json')[env];
const client = mongodb.MongoClient;
const mongodbUri = `mongodb://${config.mongo.user}:${config.mongo.password}#${config.mongo.url}/${config.mongo.database}`;
const options = {
poolSize: 100,
connectTimeoutMS: 120000,
socketTimeoutMS: 1440000
};
// connection object
let _db = null;
class MongoDBConnection {
constructor() {}
// return a promise to the existing connection or the connection function
getDB() {
return (_db ? Promise.resolve(_db) : mConnect());
}
}
module.exports = new MongoDBConnection();
// transforms into a promise Mongo's client.connect
function mConnect() {
return new Promise((resolve, reject) => {
console.log('Connecting to Mongo...');
client.connect(mongodbUri, options, (error, db) => {
if (error) {
_db = null;
return reject(error);
}
else {
console.log('Connected to Mongo...');
_db = db;
resolve(db);
}
});
});
}
To use it in a controller or app.js:
const mongoConfig = require('mongoConfig');
mongoConfig.getDB()
.then(db => db.collection('collection').find({}))
.catch(error => {...});

Resources