Nested insert in Mongodb with Node.js - node.js

I'm receiving a CSV file which contains a list of products. Each product has a category. When I save a product I have to find if the category already exists in the database and if it does not I have to create it (and save it) before saving the product.
module.exports.do_import = function (req, res) {
var final_json = {};
fs.createReadStream(req.files.file.path)
.pipe(csv({
separator: ';'
}))
.on('data', function (data) {
mongoose.model("Category").findOne({nombre: data.NOMBRE}, function (err, category) {
if (category == null) {
var created_category = new Category({nombre: data.NOMBRE});
created_category.save(function (err, __category) {
var product = new Product({
nombre: data.NOMVDA,
categoria: __category,
codigo: data.CODVAR
});
product.save(function (err, prod) {
if (err) {
final_json.success = false;
res.json(final_json);
}
});
});
} else {
var product = new Product({
nombre: data.NOMVDA,
categoria: category,
codigo: data.CODVAR
});
product.save(function (err, prod) {
if (err) {
final_json.success = false;
res.json(final_json);
}
});
}
});
}).on('finish', function () {
final_json.success = true;
}).on('error', function () {
final_json.success = false;
}).on('end', function () {
res.json(final_json);
});
So, the trouble I've got here is that the .save() function of the category is executed asynchronously, so when the next record of the file is requested, the category of the previous one isn't saved yet, so it creates a the same category twice. How can I make the function wait until the save operation is completed?

Finally a friend of mine help me and thanks to this https://nodejs.org/api/stream.html#stream_readable_pause we can make the flow stop until the product has already been created.
Here is the final code:
module.exports.do_import = function (req, res) {
var final_json = {};
var readable = fs.createReadStream(req.files.file.path)
.pipe(csv({separator: ';'}));
function saveErrorHandler(err, prod) {
if (err) {
final_json.success = false;
res.json(final_json);
}
readable.resume();
console.log('next category');
}
readable.on('data', function (data) {
console.log('data');
readable.pause();
console.log('pausa');
mongoose.xmodel("Category").findOne({nombre: data.NOMBRE}, function (err, category) {
// Manejá el error que no lo tenes.
if (category) {
// esto se podría sacar ya que esta duplicado, pasale la categoría a una función
var product = new Product({
nombre: data.NOMVDA,
categoria: category,
codigo: data.CODVAR
});
product.save(saveErrorHandler);
} else {
var created_category = new Category({nombre: data.NOMBRE});
created_category.save(function (err, __category) {
var product = new Product({
nombre: data.NOMVDA,
categoria: __category,
codigo: data.CODVAR
});
product.save(saveErrorHandler);
});
}
});
});
readable.on('finish', function () {
final_json.success = true;
});
readable.on('error', function () {
final_json.success = false;
});
readable.on('end', function () {
res.json(final_json);
});

Related

Error: NJS-012: encountered invalid bind data type in parameter 2

Even though I have searched for the solution of this error and i found some answers but none of them helped me fix this error, Error: NJS-012: encountered invalid bind data type in parameter 2.Maybe, one error can occur in a different scenarios.
Stored procedure definition
create or replace PROCEDURE SP_MEAL_GETMEALTYPES
(
p_DataSource OUT Sys_RefCursor
)
AS
BEGIN
OPEN p_DataSource FOR
select mealtypeid,description from mealtypes;
END;
File name: menusStoredProc.js
"use strict";
var dbParams = require('../../oracle/dbParams');
function storedProcs() {
this.SP_USER_GETMENUS = {
name: 'sp_meal_getmealtypes',
params: {
dataSource: {val: null, type: dbParams.CURSOR, dir: dbParams.BIND_OUT}
},
resultSetColumns: ['mealTypeId','description']
}
}
module.exports = storedProcs;
File name: menus.js
var express = require('express');
var MenusStoreProc = require('../storedProcedures/menusStoredProc');
var oraDbAssist = require('../../oracle/oracleDbAssist');
var router = express.Router();
router.get('/getmenus', (req, res, next) => {
var sp = new MenusStoreProc().SP_USER_GETMENUS;
oraDbAssist.getConnection(function (err, conn) {
if (err)
return console.log('Connecting to db failed - ' + err);
oraDbAssist.executeSqlWithConn(sp, false, conn, function (err, menus) {
if (err)
return console.log('Executing ' + sp.name + ' failed - ' + err);
res.status(200).json(JSON.stringify(menus));
});
});
});
module.exports = router;
Function definition added - executeSqlWithConn
function executeSqlWithConn(sp, autoCommit, connection, next) {
var sql = createProcedureSqlString(sp.name, sp.params);
var params = buildParams(sp.params);
connection.execute(sql, params, {autoCommit: autoCommit}, function(err, result) {
if (err) {
next(err, null);
return;
}
var allRows = [];
var numRows = 50; // number of rows to return from each call to getRows()
for(var attributeName in result.outBinds) {
if(result.outBinds[attributeName] && result.outBinds[attributeName].metaData) { // db response is a result set
function fetchRowsFromResultSet(pResultSet, pNumRows) {
pResultSet.getRows(pNumRows, function(readErr, rows) {
if(err) {
pResultSet.close(function (err) { // always close the result set
next(readErr);
});
return;
}
allRows.push(rows);
if (rows.length === pNumRows) {
fetchRowsFromResultSet(result.outBinds[attributeName], numRows);
return;
}
var allRowsResult = Array.prototype.concat.apply([], allRows);
generateJsonFromDbResultSet(pResultSet.metaData, allRowsResult, sp, function(resultSet) {
pResultSet.close(function (err) { // always close the result set
next(null, resultSet);
});
});
});
}
fetchRowsFromResultSet(result.outBinds[attributeName], numRows);
return;
}
}
next(null, result.outBinds);
});
}
Function definition added - buildParams
function buildParams(params) {
for(var attributeName in params) {
params[attributeName].val = typeof params[attributeName].val === 'undefined' ? null : params[attributeName].val;
if(params[attributeName].type.is(dbParams.DATE))
params[attributeName].val = params[attributeName].val ? new Date(params[attributeName].val) : null;
params[attributeName].type = params[attributeName].type.value;
params[attributeName].dir = params[attributeName].dir.value;
}
return params;
}
Any help, dear members ?

Unable to retrive data and push inside loop in node js

I am trying to retrieve attendance list along with user details.
I am using caminte.js(http://www.camintejs.com/) Cross-db ORM for database interaction.
Here is my code sample of model function "attendanceList".
exports.attendanceList = function (req, callback) {
var query = req.query;
var searchfilters = {};
if(!req.user){
callback({ code:400, status:'error', message: 'Invalid Request', data:{}});
}else{
searchfilters["vendor_id"] = parseInt(req.user._id);
}
if(query.location && parseString(query.location) != '') {
searchfilters["location"] = parseString(query.location);
}
if (query.device_details && parseString(query.device_details) != '') {
searchfilters["device_details"] = parseString(query.device_details);
}
if(query.created_on) {
searchfilters["created_on"] = query.created_on;
}
if(query.status) {
searchfilters["status"] = { regex: new RegExp(query.status.toLowerCase(), "i") };
}
var SkipRecord = 0;
var PageSize = 10;
var LimitRecord = PageSize;
var PageIndex = 1;
if(query.pagesize) {
PageSize = parseInt(query.pagesize);
}
if(query.pageindex) {
PageIndex = parseInt(query.pageindex);
}
if (PageIndex > 1) {
SkipRecord = (PageIndex - 1) * PageSize;
}
LimitRecord = PageSize;
var SortRecord = "created_on";
if(query.sortby && query.sorttype) {
var sortingBy = query.sortby;
var sortingType = 'ASC';
if(typeof query.sorttype !== 'undefined') {
sortingType = query.sorttype;
}
SortRecord = sortingBy + ' ' + sortingType;
}
Attendance.find({ where: searchfilters, order: SortRecord, limit: LimitRecord, skip: SkipRecord }, async function (err, result) {
if(err){
callback({ code:400, status:'error', message:'Unable to connect server', errors:err });
} else {
await result.map(function(row, i){
User.findById(parseInt(row.user_id), function(err, data){
if(err){
console.log(err);
} else {
result[i]['userDetails'] = data;
}
});
});
await Attendance.count({ where: searchfilters }, function (err, count) {
callback({ code:200, status:'success', message:'OK', total:count, data:result });
});
}
});
};
I am getting only attendance list without user details. How do I force to push user details into attendance list? Any Help!!
Thank You
This behavior is asynchronous. When you're making request to DB, your code keeps running, while task to get data comes to task queue.
To keep things simple, you need to use promises while handling asynchronous jobs.
Rewrite your code from this:
Attendance.find({ where: searchfilters, order: SortRecord, limit: LimitRecord, skip: SkipRecord }, async function (err, result) {
if(err){
callback({ code:400, status:'error', message:'Unable to connect server', errors:err });
} else {
await result.map(function(row, i){
User.findById(parseInt(row.user_id), function(err, data){
if(err){
console.log(err);
} else {
result[i]['userDetails'] = data;
}
});
});
await Attendance.count({ where: searchfilters }, function (err, count) {
callback({ code:200, status:'success', message:'OK', total:count, data:result });
});
}
});
To this:
const findAttendanceFirst = (searchFilters, SortRecord, LimitRecord, SkipRecord) => {
return new Promise((resolve, reject) => {
Attendance.find({ where: searchFilters, order: SortRecord, limit: LimitRecord, skip: SkipRecord }, (err, result) => {
if(err) return reject(err);
resolve(result);
});
});
}
const findUserByIdForUserDetails = (userId) => {
return new Promise((resolve, reject) => {
User.findById(parseInt(userId), function(err, data){
if(err) return reject(err);
resolve(data);
})
});
}
const getAttendanceCount = (searchFilters) => {
return new Promise((resolve, reject) => {
Attendance.count({ where: searchFilters }, (err, count) => {
if(err) return reject(err);
resolve(count);
});
})
}
So, now we can use this separate functions to make async behavior looks like sync.
try {
const data = await findAttendanceFirst(searchFilters, SortRecord, LimitRecord, SkipRecord);
for(let userData of data){
try {
userData.userDetails = await findUserByIdForUserDetails(userData.user_id);
} catch(e) {
// Some error happened, so no user details.
// you can set here null or nothing to userDetails.
}
}
let count;
try {
count = await getAttendanceCount(searchFilters);
} catch(e){
// Same as before.
}
const callBackData = { code:200, status:'success', message:'OK', total:count, data:result };
// And here you can do whatever you want with callback data. Send to client etc.
} catch(e) {
}
NB: I've not tested this code, it will be easier for yu to play with your actual data and use Promises and async/await
Just remember that each request to db is asynchronous, and you need to make your code wait for this data.

Node JS - function not executed

I have a module which i wrote myself. It contains 3 functions which i call as chained promises. The last one does not execute and i cannot figure out why.
The module geodata.js
require('use-strict');
var Promise = require('promise');
var NodeGeocoder = require('node-geocoder');
var mongoose = require('mongoose');
var User = require('../models/user');
var options = {
provider: 'google',
httpAdapter: 'https',
apiKey: 'AIzaSyA4v81WbNOMeRL7p911Mxr6PBZnidX0cIM',
formatter: null
};
module.exports = {
//*******************************************************
// Find user and return his adress (street + town)
//
//*******************************************************
findUser: function(username) {
return new Promise(function(resolve, reject) {
User.findOne({
'username': username
}, function(err, doc) {
if (!err) {
resolve(doc);
} else {
reject(err);
}
});
});
},
//*******************************************************
// Fetch geodata (latitude + longitude) to users record in
// the user collection.
//*******************************************************
fetchGeoData: function(userObj) {
return new Promise(function(resolve, reject) {
var geocoder = NodeGeocoder(options);
var adress = userObj.street + ' ' + userObj.town;
geocoder.geocode(adress, function(err, res) {
if (!err) {
var res2 = {
'username': userObj.username
}
console.log(res);
resolve([res, res2]);
} else {
reject(err);
}
});
});
},
//*******************************************************
// Fetch geodata (latitude + longitude) to users record in
// the user collection.
//*******************************************************
addGeoData: function(message) {
return new Promise(function(resolve, reject) {
User.findOne({
'username': message.username
}, function(err, doc) {
console.log(doc);
if (err) {
console.log(err);
reject(err);
}
if (!doc) {
console.log('User not found.');
reject('User not found.');
}
doc.longitude = String(message.longitude);
doc.latitude = String(message.latitude);
doc.save();
resolve(doc);
});
});
},
fixJSON: function(inpJSON) {
var strung = JSON.stringify(inpJSON);
obj = strung.substring(1, strung.length - 1);
return (JSON.parse(obj));
}
};
.. and here is the calling script:
require('use-strict');
var mongoose = require('mongoose');
var dbConfig = require('./db');
var geodata = require('./lib/geodata.js');
// Connect to DB
mongoose.connect(dbConfig.url, {
auth: {
authdb: "admin"
}
});
mongoose.set('debug', true);
geodata.findUser('jimmy').then(function(result) {
console.log('findUser() done');
return geodata.fetchGeoData(result);
}).then(function(result) {
console.log('fetchGeoData() done');
var res1Fixed = geodata.fixJSON(result[0]);
var params = {
username: result[1].username,
longitude: res1Fixed.longitude,
latitude: res1Fixed.latitude
}
console.log(params);
return geodata.addGeoData(params);
}).then(function(result) {
console.log('addGeoData() done');
});
mongoose.connection.close();
the execution stops after this line:
console.log('fetchGeoData() done');
and i can't figure out why?
Regards
Jimmy
If you are referring to mongoose.connection.close(); that is actually being run before your console.log('addGeoData() done');
Notice that mongoose.connection.close() is synchronous whereas the mongoose.findUser is asynchronous.

Waiting for asynchronous task to complete nodejs

I am creating an application where a user can have many rooms and each room can have many channels, here is my code when retrieving the rooms and corresponding channels:
getRooms: function (req, res) {
User.find({id: req.cookies.claver_id}).exec(function (err, result) {
if (err) {
return res.send(400);
}
rooms = result[0].rooms;
if (rooms.length === 1) {//No room defaults to ['']
return res.send(400);
}
var roomsObj = {};
var roomsArr = [];//we will place the roomsObj inside the roomsArr
var chansObj = {};
var chansArr = [];
async.each(rooms, function (roomId, cb){
roomsObj = {};
if (roomId !== '') {
Rooms.findOne({id: roomId}).exec(function (err, room){
roomName = room.name;
inviteLink = room.inviteLink;
roomsObj.name = roomName;
roomsObj.id = roomId;
roomsObj.inviteLink = inviteLink;
var channels = room.channels;
async.each(channels, function (channelId, cb) {
chansObj = {};
Channels.findOne({id: channelId}).exec(function (err, channel){
chansObj.name = channel.channelName;
chansObj.id = channelId;
chansObj.type = channel.channelType;
chansArr.push(chansObj);
cb();
});
},
function (err) {
});
});
}
cb();
}, function (err) {
roomsObj.channels = chansArr;
roomsArr.push(roomsObj);
sails.log(roomsArr);
});
});
}
It is suppose to return a javascript object with the following structure:
[ { name: "Room Name",
roomId: "Room Id",
inviteLink: "Room Invite Link",
channels: [
{
name: "Channel Name",
id: "channel Id"
}
]
}
]
But I always get an empty array because async.each(rooms, function (roomId, cb){ }) does not wait for async.each(channels, function (channelId, cb) {}) to complete, so I have empty room object. Please how do I solve this issue ?
You should call your rooms's callback loop after completing you channels loop.
You should do something like this:
getRooms: function (req, res) {
User.find({id: req.cookies.claver_id}).exec(function (err, result) {
if (err) {
return res.send(400);
}
rooms = result[0].rooms;
if (rooms.length === 1) {//No room defaults to ['']
return res.send(400);
}
var roomsObj = {};
var roomsArr = [];//we will place the roomsObj inside the roomsArr
var chansObj = {};
var chansArr = [];
async.each(rooms, function (roomId, callback1){
roomsObj = {};
if (roomId !== '') {
Rooms.findOne({id: roomId}).exec(function (err, room){
roomName = room.name;
inviteLink = room.inviteLink;
roomsObj.name = roomName;
roomsObj.id = roomId;
roomsObj.inviteLink = inviteLink;
var channels = room.channels;
var i=0;
async.each(channels, function (channelId, callback2) {
chansObj = {};
Channels.findOne({id: channelId}).exec(function (err, channel){
chansObj.name = channel.channelName;
chansObj.id = channelId;
chansObj.type = channel.channelType;
chansArr.push(chansObj);
i++;
if(i===(channels.length-1)){
i=0;
callback1();
}else{
callback2();
}
});
},
function (err) {
});
});
}
}, function (err) {
roomsObj.channels = chansArr;
roomsArr.push(roomsObj);
sails.log(roomsArr);
});
});
}
I solved it, it really was a case for promises, I used bluebird promise combined with async - the modified code:
getRooms: function (req, res) {
User.find({id: req.cookies.claver_id}).exec(function (err, result) {
if (err) {
return res.send(400);
}
rooms = result[0].rooms;
if (rooms.length === 1) {//No room defaults to ['']
return res.send(400);
}
var roomsObj = {};
var roomsArr = [];//we will place the roomsObj inside the roomsArr
var chansObj = {};
var chansArr = [];
Promise.each(rooms, function (roomId, callback1){
roomsObj = {};
if (roomId !== '') {
async.series ([
function () {
Rooms.findOne({id: roomId}).then(function (room){
roomName = room.name;
inviteLink = room.inviteLink;
roomsObj.name = roomName;
roomsObj.id = roomId;
roomsObj.inviteLink = inviteLink;
channels = room.channels;
sails.log(roomName);
})
}
]);
return Promise.each(channels, function (channelId) {
return Promise.all([
Channels.findOne({id: channelId}).then(function (channel){
chansObj = {};
chansObj.name = channel.channelName;
chansObj.id = channelId;
chansObj.type = channel.channelType;
chansArr.push(chansObj);
sails.log(chansObj);
})
]).then(function () {
sails.log('done one');
});
}).then(function () {
roomsObj.channels = chansArr;
roomsArr.push(roomsObj);
sails.log('done all');
chansArr = [];
});
}
}).then(function () {
sails.log(roomsArr);
sails.log("grand finish");
});
});
}
Thanks to everyone who contributed.

node js mongo db dependencies (doc not being found)

I have the following code:
var method = PushLoop.prototype;
var agent = require('./_header')
var request = require('request');
var User = require('../models/user_model.js');
var Message = require('../models/message_model.js');
var async = require('async')
function PushLoop() {};
method.startPushLoop = function() {
getUserList()
function getUserList() {
User.find({}, function(err, users) {
if (err) throw err;
if (users.length > 0) {
getUserMessages(users)
} else {
setTimeout(getUserList, 3000)
}
});
}
function getUserMessages(users) {
// console.log("getUserMessages")
async.eachSeries(users, function (user, callback) {
var params = {
email: user.email,
pwd: user.password,
token: user.device_token
}
messageRequest(params)
callback();
}, function (err) {
if (err) {
console.log(err)
setTimeout(getUserList, 3000)
}
});
}
function messageRequest(params) {
var url = "https://voip.ms/api/v1/rest.php?api_username="+ params.email +"&api_password="+ params.pwd +"&method=getSMS&type=1&limit=5"
request(url, function(err, response, body){
if (!err) {
var responseObject = JSON.parse(body);
var messages = responseObject.sms
if (responseObject["status"] == "success") {
async.eachSeries(messages, function(message, callback){
console.log(params.token)
saveMessage(message, params.token)
callback();
}, function(err) {
if (err) {
console.log(err)
}
// setTimeout(getUserList, 3000)
})
} else {
// setTimeout(getUserList, 3000)
}
} else {
console.log(err)
// setTimeout(getUserList, 3000)
}
});
setTimeout(getUserList, 3000)
}
function saveMessage(message, token) {
// { $and: [ { price: { $ne: 1.99 } }, { price: { $exists: true } }
// Message.find({ $and: [{ message_id: message.id}, {device_token: token}]}, function (err, doc){
Message.findOne({message_id: message.id}, function (err, doc){
if (!doc) {
console.log('emtpy today')
var m = new Message({
message_id: message.id,
did: message.did,
contact: message.contact,
message: message.message,
date: message.date,
created_at: new Date().toLocaleString(),
updated_at: new Date().toLocaleString(),
device_token: token
});
m.save(function(e) {
if (e) {
console.log(e)
} else {
agent.createMessage()
.device(token)
.alert(message.message)
.set('contact', message.contact)
.set('did', message.did)
.set('id', message.id)
.set('date', message.date)
.set('message', message.message)
.send();
}
});
}
}) //.limit(1);
}
};
module.exports = PushLoop;
Which actually works perfectly fine in my development environment - However in production (i'm using Openshift) the mongo documents get saved in an endless loop so it looks like the (if (!doc)) condition always return true therefore the document gets created each time. Not sure if this could be a mongoose issue - I also tried the "find" method instead of "findOne". My dev env has node 0.12.7 and Openshift has 0.10.x - this could be the issue, and i'm still investigating - but if anybody can spot an error I cannot see in my logic/code please let me know
thanks!
I solved this issue by using a "series" like pattern and using the shift method on the users array. The mongoose upsert findOneOrCreate is good however if there is a found document, the document is returned, if one isn't found and therefore created, it's also returned. Therefore I could not distinguish between the newly insert doc vs. a found doc, so used the same findOne function which returns null if no doc is found I just create it and send the push notification. Still abit ugly, and I know I could have used promises or the async lib, might refactor in the future. This works for now
function PushLoop() {};
var results = [];
method.go = function() {
var userArr = [];
startLoop()
function startLoop() {
User.find({},function(err, users) {
if (err) throw err;
users.forEach(function(u) {
userArr.push(u)
})
function async(arg, callback) {
var url = "https://voip.ms/api/v1/rest.php?api_username="+ arg.email +"&api_password="+ arg.password +"&method=getSMS&type=1&limit=5"
request.get(url, {timeout: 30000}, function(err, response, body){
if (!err) {
var responseObject = JSON.parse(body);
var messages = responseObject.sms
var status = responseObject.status
if (status === "success") {
messages.forEach(function(m) {
var message = new Message({
message_id: m.id,
did: m.did,
contact: m.contact,
message: m.message,
date: m.date,
created_at: new Date().toLocaleString(),
updated_at: new Date().toLocaleString(),
device_token: arg.device_token
});
var query = { $and : [{message_id: m.id}, {device_token: arg.device_token}] }
var query1 = { message_id: m.id }
Message.findOne(query).lean().exec(function (err, doc){
if (!doc || doc == null) {
message.save(function(e) {
console.log("message saved")
if (e) {
console.log("there is an error")
console.log(e)
} else {
console.log(message.device_token)
var messageStringCleaned = message.message.toString().replace(/\\/g,"");
var payload = {
"contact" : message.contact,
"did" : message.did,
"id" : message.message_id,
"date" : message.date,
"message" : messageStringCleaned
}
var note = new apns.Notification();
var myDevice = new apns.Device(message.device_token);
note.expiry = Math.floor(Date.now() / 1000) + 3600; // Expires 1 hour from now.
note.badge = 3;
note.alert = messageStringCleaned;
note.payload = payload;
apnsConnection.pushNotification(note, myDevice);
}
})
}
});
});
}
else {
console.log(err)
}
}
});
setTimeout(function() {
callback(arg + "testing 12");
}, 1000);
}
// Final task (same in all the examples)
function series(item) {
if(item) {
async( item, function(result) {
results.push(result);
return series(userArr.shift());
});
} else {
return final();
}
}
function final() {
console.log('Done');
startLoop();
}
series(userArr.shift())
});
}
}
module.exports = PushLoop;

Resources