How to run node js mongdb operation synchronously - node.js

I have following code, I need to wait till mongodb operation completed.
const nativeClient = require("mongodb").MongoClient;
async function ingestData() {
var data = [
{name: "a", address: "a_add"},
{name: "b", address: "b_add"}
];
console.log(" start ingestData");
await insertPersons(data);
console.log("end ingestData");
}
async function insertPersons(data) {
nativeClient.connect("mongodb://localhost:27017", function(err, client) {
if (err) throw err;
var db = client.db("testdb");
db.collection("person")
.insertMany(data)
.then(function(res) {
console.log(`Number of rec inserted: ${res.insertedCount}`);
return true;
})
.catch(function(err) {
console.log(`Failed to persist ${data.length} rec in db: ${err}`);
return false;
});
});
}
ingestData();
Current Output
start ingestData
end ingestData
Number of rec inserted: 2
Expected output
start ingestData
Number of rec inserted: 2
end ingestData

You are using async/await incorrectly.
If you use await, the called function must return a promise, so there is something to wait for.
If you are not using await anywhere in a function, that function does not need to be marked as async.
You're not doing either of these things.
You can return a promise that resolves when the work is done:
function insertPersons(data) {
return nativeClient.connect("mongodb://localhost:27017").then(client => {
return client.db("testdb").collection("person").insertMany(data).then(res => {
console.log(`Number of rec inserted: ${res.insertedCount}`);
return true;
});
}).catch(function(err) {
console.log(`Failed to persist ${data.length} rec in db: ${err}`);
return false;
});
}
or you can use await to wait until the work is done.
async function insertPersons(data) {
try {
let client = await nativeClient.connect("mongodb://localhost:27017"):
let db = client.db("testdb");
let res = await db.collection("person").insertMany(data);
console.log(`Number of rec inserted: ${res.insertedCount}`);
return true;
} catch (err) {
console.log(`Failed to persist ${data.length} rec in db: ${err}`);
return false;
}
}

Related

Not able to return value from promise in Nodejs

I have written the following code in Nodejs which is saving data in MongoDB:
function insertDoc(db,data){
return new Promise(resolve => {
callback=db.collection('AnalysisCollection').insertOne(data).then(function(response,obj){
console.log("Inserted record");
resolve(obj);
//console.log(obj);
// response.on('end',function(){
// resolve(obj);
// });
//return resolve(obj);
}).then(() => { return obj }
).catch(function(error){
throw new Error(error);
});
})
}
I am calling the above function from the main function like this:
async function cosmosDBConnect(nluResultJSON){
try{
//console.log("Inserting to cosmos DB");
console.log(nluResultJSON);
var url = config.cosmos_endpoint;
var result="";
var data = JSON.parse(JSON.stringify(nluResultJSON));
MongoClient.connect(url, function(err, client) {
assert.equal(null, err);
var db = client.db('NLUAnalysisDB');
// insertDoc(db, data, function() {
result=insertDoc(db, data, function() {
console.log(result);
client.close();
//return data._id;
});
});
}
catch (e) {
console.log(e);
}
}
module.exports = { cosmosDBConnect };
But in cosmosDBConnect, I am getting 'undefined' for the result, though in insertDoc I am getting the output for'obj' with _id for the inserted record.
Please help me to return this _id to cosmosDBConnect.
You are use callbacks inside of async function, which creates internal scopes. So your return aplies to them instead of whole function. You should use Promise-based methods inside of async function using await (without callbacks) or wrap whole function into own Promise otherwise.
Example:
function cosmosDBConnect(nluResultJSON) {
return new Promise((resolve, reject) => {
var url = config.cosmos_endpoint;
var result = '';
var data = JSON.parse(JSON.stringify(nluResultJSON));
MongoClient.connect(url, function(err, client) {
if (err) return reject(err);
assert.equal(null, err);
var db = client.db('NLUAnalysisDB');
insertDoc(db, data).then(obj => {
console.log(obj);
client.close();
return resolve(data._id);
});
});
});
}
Also you need to understand that your insertDoc return Promise and do not accept callback you tried to pass.
Ref: async function
result = insertDoc(db, data).then((data) => {
console.log(data);
}).catch(err => console.error(err));

Can't work out promises to return values - NodeJS

After searching through countless posts I don't understand why I'm still getting a Promise pending after my awaits. The code below should explain it but I'm trying to pull a MongoDB query of the max value of a column/schema. The console.log within the function is giving me the correct timestamp but I'm trying to pass that out of the inner scope and function to another function.
This is pure NodeJS with only MongoDB imported. Can this be done without any external packages?
export async function getMaxDate() {
var time = MongoClient.connect(url, { useUnifiedTopology: true }, function (err, db) {
if (err)
throw err;
var dbo = db.db(getDB);
dbo.collection(getColl)
.find()
.limit(1)
.sort({ 'timestamp': -1 })
.toArray(function (err, result) {
if (err)
throw err;
time = result[0].time; // THIS IS GIVING THE CORRECT VALUE
console.log(time)
db.close();
});
});
return time
}
export async function getMax() {
var block = await getMaxDate();
return block
}
var t = getMax();
console.log(t); // THIS IS GIVE ME A PROMISE PENDING
getMax() returns a promise, you have to wait for it.
var t = await getMax()
Also getMaxDate uses an async callback that you want to promisify:
export async function getMaxDate() {
return new Promise((resolve,reject) => {
MongoClient.connect(url, { useUnifiedTopology: true }, function (err, db) {
if (err)
return reject(err);
var dbo = db.db(getDB);
dbo.collection(getColl)
.find()
.limit(1)
.sort({ 'timestamp': -1 })
.toArray(function (err, result) {
if(err)
reject(err);
else {
let time = result[0].time; // THIS IS GIVING THE CORRECT VALUE
console.log(time)
db.close();
resolve(time);
}
});
})
});
}
For reference, A is the same thing as B here:
async function A(x) {
if(x)
throw new Error('foo');
else
return 'bar';
}
function B(x) {
return new Promise((resolve,reject)=>{
if(x)
reject(new Error('foo'));
else
resolve('bar');
});
}
Promises came first, then async/await notation was introduced to make common Promise coding practices easier
You can use A and B interchangeably:
async function example1() {
try {
await A(1);
await B(0);
}
catch(err) {
console.log('got error from A');
}
}
async function example2() {
return A(0).then(()=>B(1)).catch((err)=>{
console.log('got error from B');
})
}

Different reponses for mongodb query using callback and promise

Have you any idea why I don't get any error in the first example? But if I use a callback, I get MongoTimeoutError.
await server.stop();
try {
const r = await db.things.insertOne({ a: 1 }); // no error, r is undefined
assert(!r);
} catch (err) {
console.log(err);
}
await server.restart();
const r = await db.things.insertOne({ a: 1 });
assert(r.insertedCount === 1);
db.things.insertOne({ a: 1 }, (err, result) => {
if (err) {
//"MongoTimeoutError" "Server selection timed out after 10000 ms"
return reject(err);
}
resolve(result);
})
In options I have bufferMaxEntries = 0.
There is an issue with mongodb driver. See the JIRA ticket

Node js stop long process task when Promise reject it

I create a promise function to processing a long-time query task. Some time the task will block for hours. I want set a time out to stop the task. Below is the code.
It can return error message correctly, but it still running connection.execute() for long time before stop. So how can stop it immediately when it return reject message?
Thanks!
function executeQuery(connection, query) {
return new Promise((resolve, reject) => {
"use strict";
//long time query
connection.execute(query, function (err, results) {
if (err) reject('Error when fetch data');
else resolve(results);
clearTimeout(t);
});
let t = setTimeout(function () {
reject('Time Out');
}, 10);
})
(async () => {
"use strict";
oracle.outFormat = oracle.OBJECT;
try {
let query = fs.readFileSync("query.sql").toString();
let results = await executeQuery(connection, query);
console.log(results.rows);
} catch (e) {
console.log(`error:${e}`);
}
So how can stop it immediately when it return reject message?
According to the docs, you can use connection.break:
return new Promise((resolve, reject) => {
connection.execute(query, (err, results) => {
if (err) reject(err);
else resolve(results);
clearTimeout(t);
});
const t = setTimeout(() => {
connection.break(reject); // is supposed to call the execute callback with an error
}, 10);
})
Make sure to also release the connection in a finally block.
Try this (using bluebird promises):
var execute = Promise.promisify(connection.execute);
function executeQuery(connection, query) {
return execute.call(connection, query)
.timeout(10000)
.then(function (results) {
// handle results here
})
.catch(Promise.TimeoutError, function (err) {
// handle timeout error here
});
.catch(function (err) {
// handle other errors here
});
};
If this still blocks, there's a possibility that the database driver you are using is actually synchronous rather than asynchronous. In that case, that driver would be incompatible with the node event loop and you may want to look into another one.
As Bergi mentioned, you'll need to use the connection.break method.
Given the following function:
create or replace function wait_for_seconds(
p_seconds in number
)
return number
is
begin
dbms_lock.sleep(p_seconds);
return 1;
end;
Here's an example of its use:
const oracledb = require('oracledb');
const config = require('./dbConfig.js');
let conn;
let err;
let timeout;
oracledb.getConnection(config)
.then((c) => {
conn = c;
timeout = setTimeout(() => {
console.log('Timeout expired, invoking break');
conn.break((err) => {
console.log('Break finished', err);
});
}, 5000);
return conn.execute(
`select wait_for_seconds(10)
from dual`,
[],
{
outFormat: oracledb.OBJECT
}
);
})
.then(result => {
console.log(result.rows);
clearTimeout(timeout);
})
.catch(err => {
console.log('Error in processing', err);
if (/^Error: ORA-01013/.test(err)) {
console.log('The error was related to the timeout');
}
})
.then(() => {
if (conn) { // conn assignment worked, need to close
return conn.close();
}
})
.catch(err => {
console.log('Error during close', err)
});
Keep in mind that the setTimeout call is just before the execute (because of the return statement). That timeout will start counting down immediately. However, the execute call isn't guaranteed to start immediately as it uses a thread from the thread pool and it may have to wait till one is available. Just something to keep in mind...

Do something async with underscore map

function addSomething(data) {
var defer = q.defer();
data = _.map(data, function(item) {
item['something'] = callSomethingAsync();
return item;
});
return defer.promise;
}
How can I handle this problem. The only way I found is using Async.js.
But maybe there is a better way using $q?
EDIT:
function getScopes(item) {
var defer = q.defer();
var query = "SELECT somevalue FROM Something WHERE ID = '" + item.id + "'";
mysql.query(query, function(err, data) {
if (err) {
defer.reject(err);
} else {
item[newkey] = data
defer.resolve(item);
}
});
defer.resolve(data)
return defer.promise;
}
//add necessary scopes to the audit
function addScopes(data) {
var promises = _.map(data, function(item) {
return getScopes(item);
});
return Promise.all(promises);
}
How I can prevent using defer in the getScopes function?
Edit 2:
var query = "SELECT * FROM tiscope";
Q.nfcall(mysql.query, query).then(function(data) {
console.log(data);
});
there is nothing returned.
Here is how I use mysql:
var sql = require('mysql');
var connection = sql.createConnection({
host : 'xxx',
user : 'xxx',
password : 'xxx',
database : 'xxx'
});
connection.connect(function(err) {
if (err) {
console.error('error connecting: ' + err.stack);
} else {
console.log('mysql connection established');
}
});
module.exports = connection;
Maybe there is the mistake.
A lot of promise libraries provide a map function. Seems Q does not. No matter the the same can be accomplished with vanilla promises (and Q) anyway using the all function.
First things first. Avoid defer. It makes code more difficult to reason and maintain. There are only a few rare cases when defer is needed. The rest of the time a normal promise constructor/helper functions will work better.
Normal Promises Example
function addSomething() {
var promises = _.map(data, function(item) {
return callSomethingAsync(item);
});
return Promise.all(promises);
}
Q Promises Example
function addSomething() {
var promises = _.map(data, function(item) {
return callSomethingAsync(item);
});
return $q.all(promises);
}
Presumably callSomethingAsync returns a promise. If not use the promise constructor pattern:
function toPromise(asyncFn, args) {
return new Promise(function (resolve, reject) {
function callback(err, result) {
if (err) {
reject(err);
} else {
resolve(result);
}
}
asyncFn(callback, args);
});
}
function addSomething() {
var promises = _.map(data, function(item) {
return toPromise(callSomethingAsync, item);
});
return Promise.all(promises);
}

Resources