How to query two collection in mongoose on nodeJs? - node.js

hi am new to nodeJs i have to query a table from the result of the query, i have to query another table. i tried the code as follows but it returns only null.
action(function getpositions(req){
var namearray = [];
NHLPlayerStatsDaily.find ({"created_at": {$gt: new Date(y+"-"+m+"-"+d)}}, function(err,position){
if(!err)
{
for (i=0;i< position.length; i++)
{
var obj = JSON.stringify(position[i]);
var pos = JSON.parse(obj);
var p = pos["player_stats_daily"]["content"]["team_sport_content"]["league_content"]["season_content"]["team_content"]["team"]["id"]
NHLTeam.find({"sdi_team_id": p}, "first_name nick_name short_name sport_id", function(err, team){
if (!err){
var obj = JSON.stringify(team);
var pos = JSON.parse(obj);
namearray.push(team)
}
})
}
return send(namearray);
}
})
})
if i just push "p" it shows the result and when am query "NHLTeam" in separate function it also shows the result. when querying a collection with in collection it return null. how to query a collection within a collection in mongoose. thanks in advance.

This is not a query problem, it is callback issue. The send(namearray) is called before any of the NHLTeam queries in the loop are completed (remember the result of these queries is passed to callback asynchronously).
What you can do is this (basically tracking when all callbacks are completed):
NHLPlayerStatsDaily.find ({"created_at": {$gt: new Date(y+"-"+m+"-"+d)}}, function(err,position){
if(!err)
{
var total = position.length;
for (i=0;i< position.length; i++)
{
var obj = JSON.stringify(position[i]);
var pos = JSON.parse(obj);
var p = pos["player_stats_daily"]["content"]["team_sport_content"]["league_content"]["season_content"]["team_content"]["team"]["id"]
NHLTeam.find({"sdi_team_id": p}, "first_name nick_name short_name sport_id", function(err, team){
total--;
if (!err){
var obj = JSON.stringify(team);
var pos = JSON.parse(obj);
namearray.push(team);
}
if(total == 0)
send(namearray);
})
}
}
})

Related

CosmosDB stored procedure not returning all documents

i have a pretty simple stored procedure that returns 0 to many documents. Here is the code:
function GetAllDocuments(numberOfDays){
var context = getContext();
var response = context.getResponse();
var collection = context.getCollection();
var collectionLink = collection.getSelfLink();
var today = new Date();
today.setDate(today.getDate() - numberOfDays);
var inSeconds = today.getTime() / 1000;
var filterQuery = 'SELECT * FROM c WHERE c._ts >' + inSeconds;
console.log(filterQuery);
collection.queryDocuments(collectionLink, filterQuery, {pageSize:-1},
function(err, documents) {
response.setBody(response.getBody() + JSON.stringify(documents));
}
);
}
The issue I am facing is, if there are many documents to be returned ie 20000 not all documents are returned. I think i am running into the issue where i need to pass a continuationToken. I have read other posts that states that we will need to return to client(c#) then make another call to the sproc passing in a token. I havent been able to find a code sample. Also is that the only way to ensure full return? Is it just a stored procedure issue? Will i be better off to just use client.CreateDocumentQuery that will simply return all records in one call?
Thanks for any feedback!
This is the new code using Jay's sample:
function GetAllDocumentsNew(numberOfDays) {
var collection = getContext().getCollection();
var collectionLink = collection.getSelfLink();
var response = getContext().getResponse();
var docCount = 0;
var counter = 0;
var returnArray = [];
var today = new Date();
today.setDate(today.getDate() - numberOfDays);
var inSeconds = today.getTime() / 1000;
//var filterQuery = 'SELECT * FROM c WHERE c._ts >' + inSeconds;
var filterQuery = "select * from c where c._ts > 1531763849.225 and c._ts <1532637743.261 and c.ProcessTypeID = 1";
console.log(filterQuery);
tryQuery();
function tryQuery(continuation) {
var query = {
query: filterQuery
};
var requestOptions = {
MaxItemCount: 10000,
continuation: continuation
};
var isAccepted =
collection
.queryDocuments(collectionLink,
query,
requestOptions,
function queryCallback(err, documents,responseOptions) {
if (err) throw err;
if (documents.length > 0) {
docCount = documents.length;
console.log(docCount.toString());
for (var i=0; i<docCount; i++){
returnArray.push(documents[i]);
}
getContext().getResponse().setBody(returnArray);
}
else if (responseOptions.continuation) {
// Else if the query came back empty, but with a continuation token;
// repeat the query with the token.
tryQuery(responseOptions.continuation);
} else {
throw new Error("Document not found.");
}
});
if (!isAccepted) {
throw new Error("The stored procedure timed out");
}
}
}
if there are many documents to be returned ie 20000 not all documents
are returned. I think i am running into the issue where i need to pass
a continuationToken.
When the query data is quite huge,of course, you need to consider use continuation token in stored procedure. You could refer to the pseudo-code as below:
function GetAllDocuments(numberOfDays) {
var collection = getContext().getCollection();
var collectionLink = collection.getSelfLink();
var response = getContext().getResponse();
var docCount = 0;
var counter = 0;
var returnArray = [];
var today = new Date();
today.setDate(today.getDate() - numberOfDays);
var inSeconds = today.getTime() / 1000;
var filterQuery = 'SELECT * FROM c WHERE c._ts >' + inSeconds;
tryQuery();
function tryQuery(continuation) {
var query = {
query: "select * from root r"
};
var requestOptions = {
MaxItemCount: 1000
continuation: continuation
};
var isAccepted =
collection
.queryDocuments(collectionLink,
query,
requestOptions,
function queryCallback(err, documents,responseOptions) {
if (err) throw err;
if (documents.length > 0) {
docCount = documents.length;
for (var i=0; i<docCount; i++){
returnArray.push(documents[i]);
}
getContext().getResponse().setBody(returnArray);
}
else if (responseOptions.continuation) {
// Else if the query came back empty, but with a continuation token;
// repeat the query with the token.
tryQuery(responseOptions.continuation);
} else {
throw new Error("Document not found.");
}
});
if (!isAccepted) {
throw new Error("The stored procedure timed out");
}
}
}
I have read other posts that states that we will need to return to
client(c#) then make another call to the sproc passing in a token. I
havent been able to find a code sample.
As I know, stored procedure has 5 seconds running limitation(Is it possible to disable the 5 second time limit for Azure CosmosDB stored procedures, if your stored procedure time out,you need to call [ExecuteStoredProcedureAsync][1] on your client side and pass the continuation token as parameter.
Will i be better off to just use client.CreateDocumentQuery that will
simply return all records in one call?
Per my knowledge,stored procedures are JS code scripts which running on the server side.I presume that performance is more efficient than using client invoke SDK. However, it is limited to the JS syntax and running time.If you can not tolerate these features,i suggest you could complete your requirements via client SDK.
Hope it helps you.Any concern ,please let me know.

How to perform bulk upload in mongoose (node.js)

I am trying to perform bulk upload in node js and mongodb is my db,can anyone suggest me some best articles regarding this.Thanks in advance.
You can use Model.collection.insert or Model.insertMany as below where collections is array of items in bulk.
Model.collection.insert(collections, function (err, models) {
next(err, models);
});
OR,
Model.insertMany(collections, function (err, models) {
next(err, models);
});
Mongoose reference: http://mongoosejs.com/docs/api.html#model_Model.insertMany
Mongo reference: https://docs.mongodb.com/v3.2/reference/method/db.collection.insert/
You can insert multiple records with batch/bulk insert in mongoose.
var arr = [{ name: 'Star Wars' }, { name: 'The Empire Strikes Back' }];
Movies.insertMany(arr, function(error, docs) {});
Let's say i have an excel file employees.xlsx with following data and i want perform bulk write.
There are several libraries out there for converting excel data to json in node, i use xlsx but it's personal taste you can use whatever is convenient for you.
Here is a helper i use for reading "/public/employees.xlsx" file.I found the content from here here.
//** helper/excel-reader.js **//
var excelReader = {};
excelReader.readExcel = function(filePath){
var XLSX = require('xlsx');
var workbook = XLSX.readFile(filePath);
var sheet_name_list = workbook.SheetNames;
var data = [];
sheet_name_list.forEach(function(y) {
var worksheet = workbook.Sheets[y];
var headers = {};
for(z in worksheet) {
if(z[0] === '!') continue;
//parse out the column, row, and value
var tt = 0;
for (var i = 0; i < z.length; i++) {
if (!isNaN(z[i])) {
tt = i;
break;
}
};
var col = z.substring(0,tt);
var row = parseInt(z.substring(tt));
var value = worksheet[z].v;
//store header names
if(row == 1 && value) {
headers[col] = value;
continue;
}
if(!data[row]) data[row]={};
data[row][headers[col]] = value;
}
//drop those first two rows which are empty
data.shift();
data.shift();
});
return data;
}
module.exports = excelReader;
Now the employee model somehow looks like this.
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
var employee = new Schema({
name: String,
adderess: String,
phonenumber: String
});
module.exports = mongoose.model('Employee', employee);
Now let's use the above code so here is my users.js route whenever i type "localhost:3000/users" it write the csv content to database.
var express = require('express');
var router = express.Router();
var excelReader = require('../helpers/excel-reader');
var mongoose = require('mongoose');
var Employee = require('../models/employee');
/* GET users listing. */
router.get('/', function(req, res, next) {
var employeesJsonArray = excelReader.readExcel('./public/employees.xlsx')
Employee.insertMany(employeesJsonArray,function(error, docs) {
if(error){
next(error)
}
else{
res.json(docs);//just rendering the document i got
}
});
});
module.exports = router;
Hope this helps!!

get name from id inside loop in node js

hello all I am new to node js and mongo db so I am facing a problem which is written above want to match an id from an array and get all the record of matching id , but thats not working i have already tried for loop but that is taking too much time so i am looking for some kind of query by which I pass id and get the matching results i have below function also where i can pass id and get the matching result but i don't know how to call that .
exports.getDistrictFromId = function(req, res, next) {
var distId = req.params.id;
districtslib.getDistricts({_id: utils.toObjectId(distId)}, function(err, district) {
if (err) {
return next(err);
}
req.store.district = district;
next();
});
};
Here is my code
exports.getCompleteTeachersList = function(req, res, next) {
var query = req.store.get('query');
var teacherQuery = {enabled: true};
var searchQuery = '';
if (!_.isUndefined(query)) {
// schoolQuery = {'name':new RegExp(query.replace(/([.?*+^$[\]\\(){}|-])/g, "\\$1"),'i')};
//{ $text: { $search: "amit hinduja"} }
//teacherQuery = {enabled: true,$or:[{firstName:new RegExp(query.replace(/([.?*+^$[\]\\(){}|-])/g, "\\$1"),'i')},{lastName:new RegExp(query.replace(/([.?*+^$[\]\\(){}|-])/g, "\\$1"),'i')}]};
if(query.trim() != '') {
teacherQuery = {enabled: true,$text: { $search: query} };
searchQuery = query;
}
}
teacherslib.getTeachers(teacherQuery, function(err, teachers) {
if (err) {
return next(err);
}
var schools = req.store.get('schools');
for(var i = 0; i < teachers.length; i++) {
teachers[i].schoolName = "";
for(var j = 0; j < schools.length; j++) {
if (teachers[i].schoolId.toString() === schools[j]._id.toString()) {
teachers[i].schoolName = schools[j].name;
teachers[i].distId = "";
var districts = req.store.get('districts');
console.log(schools[j].distId);
// i want to get the array of matching district id `schools[j].distId` from the district array from `var districts = req.store.get('districts');` this line
break;
}
}
}
req.store.set('searchQuery', searchQuery);
req.store.set('teachers', teachers);
//req.store.set('districts', districts);
next();
});
};
Collection structure is like this
1) distid is coming in schools collection
using distid get all the matching record from district
2)district array has countyid and from that county id has to get data from the county collection
Instead of looping inside a loop, i would suggest you look into the $lookup operator of the aggregation framework. Here you can perform the lookup server side.

nodejs mongodb $in [array] not working if array is a variable

This is very strange to me. If put the array onlyIds in the aggregation query of my db I get no results. If I however put the content of onlyIds that I get printed from line 5 which looks like:
["52e953942a13df5be22cf792","52e953942a13df5be22cf793","52e953942a13df5be22cf797"...]
Then it works. But not if I use the variable.
This function:
var onlyIds = [];
for (var i = 0; i < users.length; i++) {
onlyIds.push(users[i]._id);
}
console.log("ids: " + JSON.stringify(onlyIds)); <---------- not empty
db.collection('posts', function(err, collection) {
collection.aggregate([
{$match: {user_id: {$in: onlyIds}}}, <------- not working
{$match: {created:{$gte: 0}}},
{$sort:{"created": -1}},
{$skip: req.body.skip},
{$limit: req.body.limit}
],
function(err, posts) {
var errorNo, content, message;
if (err) {
errorNo = resSend.errorDB;
message = JSON.stringify(err);
} else {
errorNo = resSend.errorNo;
content = posts;
message = "";
--> console.log(JSON.stringify(posts));
}
resSend.sendResponse(res, resSend.errorNo, content, message);
});
});
So in short, why does this work:
{$match: {user_id: {$in: ["52e953942a13df5be22cf792","52e953942a13df5be22cf793","52e953942a13df5be22cf797"...]}}}
and this doesn't:
{$match: {user_id: {$in: onlyIds}}}
And the line that does not work, works perfectly in another function. Any ideas or enlightenments?
EDIT:
Switching to find and using the below answer like this:
collection.find({'user_id': {$in: onlyIdsX}}).toArray(function(err, posts)
does not work either.
ANSWER:
As the selected answer indicates below is when the variable you search for is an ObjectId or a string. For anyone else, make sure that the variable in the db is the same type as the one you try to match it with. In my case both were supposed to be strings, but one's in "onlyIds" were ObjectIds.
Try following code to modify your loop:
var ids = ["52e953942a13df5be22cf792","52e953942a13df5be22cf793","52e953942a13df5be22cf797"];
var obj_ids = [];
for (var i = 0; i < users.length; i++) {
obj_ids.push(new ObjectID(users[i]._id.toString()));
var obj_ids.push(users[i]._id); // <== This will not work if your DB has _id : ObjectID("xyz") [i.e. you are not overiding defaults]
}
And you should include var ObjectID = require('mongodb').ObjectID; into your code.
You should use .toArray(function(err,.. (Not in your case since you used aggregation framework).This will also cause issue if you are not using findOne() (For more info on this here is the link)
Following is the example which spots the issue (in comments) & working Code:
var mongo = require('mongodb'),
Server = mongo.Server,
Db = mongo.Db,
ObjectID = require('mongodb').ObjectID;
var BSON = require('mongodb').BSONPure;
var server = new Server('localhost', 27017, {
auto_reconnect: true
});
var MongoClient = require('mongodb').MongoClient
//let id = your _id, smth like '6dg27sh2sdhsdhs72hsdfs2sfs'...
var users = ["52e953942a13df5be22cf792","52cbd028e9f43a090ca0c1af","52e953942a13df5be22cf797"];
var obj_ids = [];
for (var i = 0; i < users.length; i++) {
obj_ids.push(new ObjectID(users[i].toString()));
//obj_ids.push(users[i]._id); // <== This will not work if your DB has _id : ObjectID("xyz") [i.e. you are not overiding defaults]
}
MongoClient.connect('mongodb://127.0.0.1:27017/YourDBName', function(err, db) {
console.log('err' + err);
db.collection('posts', function(error, collection) {
//collection.find({_id:{$in: users}}),function(err, docs) { //This will not work
collection.find({_id:{$in: obj_ids}}).toArray(function(err, docs) {
console.log("Printing docs from Array. count " + docs.length);
docs.forEach(function(doc) {
console.log("Doc from Array ");
console.dir(doc);
});
});
});
});

why update doesn't work

I pass a json variable to a module but I can't do the update of my collection, always I have an error in the updating.
var gestion = function(myJSON) {
var dburl = 'localhost/mongoapp';
var collection = ['clientes'];
var db = require('mongojs').connect(dburl, collection );
function cliente(nombre, estado, nuevo){
this.nombre = nombre;
this.estado = estado;
this.nuevo = nuevo;
}
var cliente1 = new cliente(myJSON.nombre myJSON.estado, myJSON.nuevo);
if (cliente1.estado == "desconectado"){
db.clientes.update(cliente1.nombre, {$set: {estado: "desconectado", nuevo: "no"}}, function(err) {
if (err) console.log("error "+cliente1.nombre);
else console.log("OK");
});
}
}
return 0;
}
I also tried to remove my db and create one more time and I'm sure that my object exist in my db.
The signature you should be using is
update(query, update, callback)
but you're passing a string for query, which doesn't mean anything to Mongo. You may want to look at the docs for an overview, but for this specific instance, it looks like you're trying to find the document where nombre is equal to the string at cliente1.nombre. The query for this is a dictionary { nombre: cliente1.nombre }, so that line should be
db.clientes.update({nombre: cliente1.nombre}, {$set: {estado: "desconectado", nuevo: "no"}}, function(err) {

Resources