I've started using the Q Promise package in a simple node.js application. So I'm interested in how I can close the db connection after all promise sequence is complete.
Example:
var toDbConnectionString = function(dbSettings) {
return "mongodb://" +
dbSettings.user + ":" +
dbSettings.password + "#" +
dbSettings.url;
};
var connectionString = toDbConnectionString(dbSettings);
Q.nfcall(
MongoClient.connect,
toDbConnectionString(dbSettings))
.then(function(db) {
return Q.ninvoke(db, "collectionNames");
})
.then(function(collections) {
console.log(collections);
})
.catch(function() {
console.log(arguments);
});
I want to close the connection after displaying the collection names but there's no db context in this anonymous function.
Is there the way how to handle such cases with the promise pattern?
If you don't want to do nesting (I certainly don't...) then you can just do like so:
var toDbConnectionString = function(dbSettings) {
return "mongodb://" +
dbSettings.user + ":" +
dbSettings.password + "#" +
dbSettings.url;
};
var connectionString = toDbConnectionString(dbSettings);
var db;
Q.nfcall(
MongoClient.connect,
toDbConnectionString(dbSettings))
.then(function(_db) {
db = _db;
return Q.ninvoke(db, "collectionNames");
})
.then(function(collections) {
console.log(collections);
})
.catch(function() {
console.log(arguments);
})
.finally(function() {
if (db) db.close();
});
You can nest then-callbacks arbitrarily. It's getting closer to the pyramid of doom, yes, but it's the only way to keep arguments in scope. In your case:
Q.nfcall(
MongoClient.connect,
toDbConnectionString(dbSettings))
.then(function(db) {
return Q.ninvoke(db, "collectionNames")
.then(function(collections) {
console.log(collections);
}).finally(function() {
db.close();
});
})
.catch(console.log.bind(console));
However, notice that you usually wouldn't connect and close too often.
Related
I'm just starting to use Mongodb without mongoose (to get away from the schemas), and wanted to create a simple module with various exported functions to use in the rest of my app. I've pasted the code below.
The problem I'm having is that the databasesList.databases comes back as undefined, and I'm not sure why. There should be 2 databases on my cluster, and one collection in each database.
As a tangential question, I thought maybe I would check the collections instead (now commented out), but though I found this page (https://docs.mongodb.com/manual/reference/method/db.getCollectionNames/) the function getCollectionNames seems not to exist. Now I'm wondering if I'm using the wrong documentation and that is why my databases are coming back undefined.
const client = new MongoClient(uri)
const connection = client.connect( function (err, database) {
if (err) throw err;
else if (!database) console.log('Unknown error connecting to database');
else {
console.log('Connected to MongoDB database server');
}
});
module.exports = {
getDatabaseList: function() {
console.log('start ' + client);
databasesList = client.db().admin().listDatabases();
//collectionList = client.db().getCollectionNames();
//console.log("Collections: " + collectionList);
console.log("Databases: " + databasesList.databases);
//databasesList.databases.forEach(db => console.log(` - ${db.name}`));
}
}```
your code is correct Just need to change few things.
module.exports = {
getDatabaseList: async function() {
console.log('start ' + client);
databasesList = await client.db().admin().listDatabases();
//collectionList = await client.db().getCollectionNames();
//console.log("Collections: " + collectionList);
console.log("Databases: " + databasesList.databases);
databasesList.databases.forEach(db => console.log(` - ${db.name}`));
}
}
You have to declare async function and use await also.
The async and await keywords enable asynchronous, promise-based behaviour to be written in a cleaner style, avoiding the need to explicitly configure promise chains.
You can use this modular approach to build your database access code:
index.js: Run your database application code, like list database names, collection names and read from a collection.
const connect = require('./database');
const dbFunctions = require('./dbFunctions');
const start = async function() {
const connection = await connect();
console.log('Connected...');
const dbNames = await dbFunctions.getDbNames(connection);
console.log(await dbNames.databases.map(e => e.name));
const colls = await dbFunctions.getCollNames(connection, 'test');
console.log(await colls.map(e => e.name));
console.log(await dbFunctions.getDocs(connection, 'test', 'test'));
};
start();
database.js:: Create a connection object. This connection is used for all your database access code. In general, a single connection creates a connection pool and this can be used throughout a small application
const { MongoClient } = require('mongodb');
const url = 'mongodb://localhost:27017/';
const opts = { useUnifiedTopology: true };
async function connect() {
console.log('Connecting to db server...');
return await MongoClient.connect(url, opts );
}
module.exports = connect;
dbFunctions.js:: Various functions to access database details, collection details and query a specific collection.
module.exports = {
// return list of database names
getDbNames: async function(conn) {
return await conn.db().admin().listDatabases( { nameOnly: true } );
},
// return collections list as an array for a given database
getCollNames: async function(conn, db) {
return await conn.db(db).listCollections().toArray();
},
// return documents as an array for a given database and collection
getDocs: async function(conn, db, coll) {
return await conn.db(db).collection(coll).find().toArray();
}
}
Considering that my server.js looks almost like this. Just send you the relevant part. I did not receive anything from the query, I do have data in the database, and "sendNotification" is triggered by the jQuery function in the client. Everything works and since var notis = []; returns an empty value and is what is shows as response. I know I have to debug SQL and that's what I'm going to do but anyway want to be sure of this other things. So my questions are:
1) Is a right syntax for node.js, considering this async behavior? (which I still don't understand )
2) The query always should be inside of the "io.sockets.on('connection')" part?
connection = mysql.createConnection({
host: 'localhost',
user: '',
password: "",
database: 'table' //put your database name
}),
...
connection.connect(function(err) {
// connected! (unless `err` is set)
console.log(err);
});
…
var sqlquery = function(uID,vs){
var notis = [];
connection.query("SELECT * FROM notification WHERE kid = ? AND v = ? ORDER BY id DESC",[uID,vs])
.on("result", function (data){
return notis.push(data);
});
};
io.sockets.on('connection', function(socket) {
...
socket.on("sendNotification", function(data) {
var roomBName = data.room_name.replace("room-",""),
found = [];
var roomSelected = _.find(rooms, function (room) { return room.id == roomBName });
for (var person in people) {
for (var i = 0, numAttending = roomSelected.peopleAttending.length; i < numAttending; i++) {
if (people[person].name == roomSelected.peopleAttending[i]) {
found.push(person);
}
}
}
for (var i = 0, numFound = found.length; i < numFound; i++) {
**result = sqlquery(9,2);**
io.to(found[i]).emit('notification', result);
};
});
Your sqlquery() function will not accomplish anything useful. Because connection.query() is asynchronous, that means it provides the response sometime LATER after sqlquery() has already finished.
The only way in node.js to use an async result is to actually use it in the callback that provides it. You don't just stuff it into some other variable and expect the result to be there for you in other code. Instead, you use it inside that callback or you call some other function from the callback and pass it the data.
Here's one way, you could change your sqlquery() function:
var sqlquery = function(uID, vs, callback){
connection.query("SELECT * FROM notification WHERE kid = ? AND v = ? ORDER BY id DESC",[uID,vs])
.on("result", function (data){
callback(null, data);
});
// need to add error handling here if the query returns an error
// by calling callback(err)
};
Then, you could use the sqlquery function like this:
found.forEach(function(person, index) {
sqlquery(..., function(err, result) {
if (err) {
// handle an error here
} else {
io.to(person).emit('notification', result);
}
});
});
And, it looks like you probably have similar async issues in other places too like in connection.connect().
In addition to #jfriend00, this could be done with new ES6 feature Promise :
var sqlquery = function(uID, vs){
return new Promise(function(resolve, reject){
connection.query("SELECT * FROM notification WHERE kid = ? AND v = ? ORDER BY id DESC",[uID,vs])
.on("result", function (data){
resolve(data);
});
});
};
Now you can use it like :
found.forEach(function(person, index) {
sqlquery(...)
.then(function(result){
io.to(person).emit('notification', result);
});
});
I am trying to test a function that looks like:
function sendVerifySms(patientId, data) {
var smsNumber = data.smsNumber;
var verifyCode = ((Math.random() * 1000000) | 0).toString();
var sql = "UPDATE [patient]";
sql += " SET phone_sms_verify_code = '" + verifyCode + "',";
// verification must take place within a one hour period
sql += " phone_sms_verify_code_expire = '" + moment.utc().add(1, 'hour').formatSqlDatetime() + "',";
sql += " phone_sms_verified = 0,"
sql += " phone_sms = '" + escapeSql(smsNumber) + "'";
sql += " WHERE id = " + escapeSql(patientId.toString());
return sqlServer.query(sql).then(function(result) {
twilioClient.sendMessage({
to: smsNumber,
from: twilioUser.verifyNumber,
body: verifyCode
}).then(function(res) {
console.log('sent verification');
return verifyCode;
}).fail(function(err) {
console.log('error sending verification');
console.log(err);
return err;
});
}).fail(function(err) {
console.log(err);
return err;
});
}
Easy enough, right? Well, first I need to decide what EXACTLY I'm testing for.
That the sql command matches the format given some data
That the twilioClient.sendMessage has been called.
Here is what I have for my test so far:
var should = require('should');
var methods;
var mockery = require('mockery');
var sinon = require('sinon');
describe('RPC Methods', function() {
before(function() {
mockery.enable();
mockery.registerMock('msnodesql', {
open: function() {
return true;
}
});
mockery.registerMock('../../db/sqlserver', {
query: function() {
return {
then: function() {
return true;
}
}
}
});
methods = require('../../../rpc/methods');
});
it('should send a verify SMS', function() {
var data = {
}
methods.sendVerifySms(1, data);
should(1).equal(1);
});
});
So now I'm a bit lost from here. I have a ton of require and since I want to isolate my individual functions for testing, I figured something like mockery would work. Where do I go from here?
If anything is unclear, please post a comment and I'll clarify.
Thanks!
With mocha tests you have an optional done callback that makes testing async functions easier, like this:
it('should send a verify SMS', function(done) {
var data = {};
var code = 1;
methods.sendVerifySms(code, data)
.then(function(actualCode) {
should(actualCode).equal(code);
done();
});
});
I would also have some feedback to offer on the async function you're testing. First I'd say you don't have to nest promise chains the way you are, and in this case its probably better that you didn't. If you simply return a promise from within a promise callback, you can chain it at the same level, like this:
return sqlServer.query(sql).then(function(result) {
return twilioClient.sendMessage({
to: smsNumber,
from: twilioUser.verifyNumber,
body: verifyCode
});
}).then(function(res) {
console.log('sent verification');
return verifyCode;
}).fail(function(err) {
console.log(err);
throw err;
});
Second, in the error handler you had before you were simply returning the err. This is probably a bad idea because it tells the consumer of your promise that everything is hunky dory and that they should proceed as such. Throwing an error on the other hand will allow that consumer to handle the error however they want in their own .fail block.
I want to drop some mongodb collections, but that's an asynchronous task. The code will be:
var mongoose = require('mongoose');
mongoose.connect('mongo://localhost/xxx');
var conn = mongoose.connection;
['aaa','bbb','ccc'].forEach(function(name){
conn.collection(name).drop(function(err) {
console.log('dropped');
});
});
console.log('all dropped');
The console displays:
all dropped
dropped
dropped
dropped
What is the simplest way to make sure all dropped will be printed after all collections has been dropped? Any 3rd-party can be used to simplify the code.
Use Promises.
var mongoose = require('mongoose');
mongoose.connect('your MongoDB connection string');
var conn = mongoose.connection;
var promises = ['aaa', 'bbb', 'ccc'].map(function(name) {
return new Promise(function(resolve, reject) {
var collection = conn.collection(name);
collection.drop(function(err) {
if (err) { return reject(err); }
console.log('dropped ' + name);
resolve();
});
});
});
Promise.all(promises)
.then(function() { console.log('all dropped)'); })
.catch(console.error);
This drops each collection, printing “dropped” after each one, and then prints “all dropped” when complete. If an error occurs, it is displayed to stderr.
Previous answer (this pre-dates Node’s native support for Promises):
Use Q promises or Bluebird promises.
With Q:
var Q = require('q');
var mongoose = require('mongoose');
mongoose.connect('your MongoDB connection string');
var conn = mongoose.connection;
var promises = ['aaa','bbb','ccc'].map(function(name){
var collection = conn.collection(name);
return Q.ninvoke(collection, 'drop')
.then(function() { console.log('dropped ' + name); });
});
Q.all(promises)
.then(function() { console.log('all dropped'); })
.fail(console.error);
With Bluebird:
var Promise = require('bluebird');
var mongoose = Promise.promisifyAll(require('mongoose'));
mongoose.connect('your MongoDB connection string');
var conn = mongoose.connection;
var promises = ['aaa', 'bbb', 'ccc'].map(function(name) {
return conn.collection(name).dropAsync().then(function() {
console.log('dropped ' + name);
});
});
Promise.all(promises)
.then(function() { console.log('all dropped'); })
.error(console.error);
I see you are using mongoose so you are talking about server-side JavaScript. In that case I advice looking at async module and use async.parallel(...). You will find this module really helpful - it was developed to solve the problem you are struggling with. Your code may look like this
var async = require('async');
var calls = [];
['aaa','bbb','ccc'].forEach(function(name){
calls.push(function(callback) {
conn.collection(name).drop(function(err) {
if (err)
return callback(err);
console.log('dropped');
callback(null, name);
});
}
)});
async.parallel(calls, function(err, result) {
/* this code will run after all calls finished the job or
when any of the calls passes an error */
if (err)
return console.log(err);
console.log(result);
});
The way to do it is to pass the tasks a callback that updates a shared counter. When the shared counter reaches zero you know that all tasks have finished so you can continue with your normal flow.
var ntasks_left_to_go = 4;
var callback = function(){
ntasks_left_to_go -= 1;
if(ntasks_left_to_go <= 0){
console.log('All tasks have completed. Do your stuff');
}
}
task1(callback);
task2(callback);
task3(callback);
task4(callback);
Of course, there are many ways to make this kind of code more generic or reusable and any of the many async programing libraries out there should have at least one function to do this kind of thing.
Expanding upon #freakish answer, async also offers a each method, which seems especially suited for your case:
var async = require('async');
async.each(['aaa','bbb','ccc'], function(name, callback) {
conn.collection(name).drop( callback );
}, function(err) {
if( err ) { return console.log(err); }
console.log('all dropped');
});
IMHO, this makes the code both more efficient and more legible. I've taken the liberty of removing the console.log('dropped') - if you want it, use this instead:
var async = require('async');
async.each(['aaa','bbb','ccc'], function(name, callback) {
// if you really want the console.log( 'dropped' ),
// replace the 'callback' here with an anonymous function
conn.collection(name).drop( function(err) {
if( err ) { return callback(err); }
console.log('dropped');
callback()
});
}, function(err) {
if( err ) { return console.log(err); }
console.log('all dropped');
});
I do this without external libaries:
var yourArray = ['aaa','bbb','ccc'];
var counter = [];
yourArray.forEach(function(name){
conn.collection(name).drop(function(err) {
counter.push(true);
console.log('dropped');
if(counter.length === yourArray.length){
console.log('all dropped');
}
});
});
All answers are quite old. Since the beginning of 2013 Mongoose started to support promises gradually for all queries, so that would be the recommended way of structuring several async calls in the required order going forward I guess.
With deferred (another promise/deferred implementation) you can do:
// Setup 'pdrop', promise version of 'drop' method
var deferred = require('deferred');
mongoose.Collection.prototype.pdrop =
deferred.promisify(mongoose.Collection.prototype.drop);
// Drop collections:
deferred.map(['aaa','bbb','ccc'], function(name){
return conn.collection(name).pdrop()(function () {
console.log("dropped");
});
}).end(function () {
console.log("all dropped");
}, null);
If you are using Babel or such transpilers and using async/await you could do :
function onDrop() {
console.log("dropped");
}
async function dropAll( collections ) {
const drops = collections.map(col => conn.collection(col).drop(onDrop) );
await drops;
console.log("all dropped");
}
I have the following code in nodejs that uses the pg (https://github.com/brianc/node-postgres)
My code to create subscriptions for an employee is as such.
client.query(
'INSERT INTO subscriptions (subscription_guid, employer_guid, employee_guid)
values ($1,$2,$3)', [
datasetArr[0].subscription_guid,
datasetArr[0].employer_guid,
datasetArr[0].employee_guid
],
function(err, result) {
done();
if (err) {
set_response(500, err, res);
logger.error('error running query', err);
return console.error('error running query', err);
}
logger.info('subscription with created');
set_response(201);
});
As you have already noticed datasetArr is an array. I would like to create mass subscriptions for more than one employee at a time. However I would not like to loop through the array. Is there a way to do it out of the box with pg?
I did a search for the same question, but found no solution yet.
With the async library it is very simple to use the query several times, and do the necessary error handling.
May be this code variant helps.
(for inserting 10.000 small json objects to an empty database it took 6 sec).
Christoph
function insertData(item,callback) {
client.query('INSERT INTO subscriptions (subscription_guid, employer_guid, employee_guid)
values ($1,$2,$3)', [
item.subscription_guid,
item.employer_guid,
item.employee_guid
],
function(err,result) {
// return any err to async.each iterator
callback(err);
})
}
async.each(datasetArr,insertData,function(err) {
// Release the client to the pg module
done();
if (err) {
set_response(500, err, res);
logger.error('error running query', err);
return console.error('error running query', err);
}
logger.info('subscription with created');
set_response(201);
})
It looks for me that the best way is the usage PostgreSQL json functions:
client.query('INSERT INTO table (columns) ' +
'SELECT m.* FROM json_populate_recordset(null::your_custom_type, $1) AS m',
[JSON.stringify(your_json_object_array)], function(err, result) {
if(err) {
console.log(err);
} else {
console.log(result);
}
});
To do Bulk insert into Postgresql from NodeJS, the better option would be to use 'COPY' Command provided by Postgres and pg-copy-streams.
Code snippet from : https://gist.github.com/sairamkrish/477d20980611202f46a2d44648f7b14b
/*
Pseudo code - to serve as a help guide.
*/
const copyFrom = require('pg-copy-streams').from;
const Readable = require('stream').Readable;
const { Pool,Client } = require('pg');
const fs = require('fs');
const path = require('path');
const datasourcesConfigFilePath = path.join(__dirname,'..','..','server','datasources.json');
const datasources = JSON.parse(fs.readFileSync(datasourcesConfigFilePath, 'utf8'));
const pool = new Pool({
user: datasources.PG.user,
host: datasources.PG.host,
database: datasources.PG.database,
password: datasources.PG.password,
port: datasources.PG.port,
});
export const bulkInsert = (employees) => {
pool.connect().then(client=>{
let done = () => {
client.release();
}
var stream = client.query(copyFrom('COPY employee (name,age,salary) FROM STDIN'));
var rs = new Readable;
let currentIndex = 0;
rs._read = function () {
if (currentIndex === employees.length) {
rs.push(null);
} else {
let employee = employees[currentIndex];
rs.push(employee.name + '\t' + employee.age + '\t' + employee.salary + '\n');
currentIndex = currentIndex+1;
}
};
let onError = strErr => {
console.error('Something went wrong:', strErr);
done();
};
rs.on('error', onError);
stream.on('error', onError);
stream.on('end',done);
rs.pipe(stream);
});
}
Finer details explained in this link
Create your data structure as:
[ [val1,val2],[val1,val2] ...]
Then convert it into a string:
JSON.stringify([['a','b'],['c']]).replace(/\[/g,"(").replace(/\]/g,")").replace(/"/g,'\'').slice(1,-1)
append it to the query and you are done!
Agreed it has string parsing costs but its way cheaper than single inserts.
Use an ORM; eg: Objection.
Also, Increase the Connection pool size based on your db server and the number of active connection you need.
someMovie
.$relatedQuery('actors')
.insert([
{firstName: 'Jennifer', lastName: 'Lawrence'},
{firstName: 'Bradley', lastName: 'Cooper'}
])
.then(function (actors) {
console.log(actors[0].firstName);
console.log(actors[1].firstName);
});