Node mongo count query items - node.js

I have a simple GET search. Problem is that when I use some criteria like status or name or email it shows correct results BUT pagination behave like there were no criteria.
For example, there is one user in DB with name John so I search for John and it gives me exactly him as a result. But pagination shows me 10pages and I don't know how to fix it. When I parse count it gives me all results not only one,
Here is my controller
/**
* GET /admin/users/:page
* Find users
*/
exports.findUsers = (req, res, next) => {
const perPage = 13
const page = Number(req.params.page) || 1
var query = {};
const possibleQueryProps = [{
"key": "id",
"queryKey": "_id"
},
{
"key": "email",
"queryKey": "email"
},
{
"key": "firstname",
"queryKey": "profile.firstname"
},
{
"key": "lastname",
"queryKey": "profile.lastname"
},
{
"key": "location",
"queryKey": "profile.location"
},
{
"key": "status",
"queryKey": "profile.status"
}
];
possibleQueryProps.forEach(prop => {
if (req.query[prop.key]) {
query[prop.queryKey] = req.query[prop.key];
}
});
console.log(query)
User
.find(query)
.skip((perPage * page) - perPage)
.limit(perPage)
.exec(function (err, users) {
User.countDocuments().exec(function (err, count) {
if (err) return next(err)
res.render('admin/users', {
layout: "admin",
users: users,
query: req.query,
active: {
users: true
},
current: page,
pages: Math.ceil(count / perPage),
helpers,
pagesArr: Array.from(Array(((Math.ceil(count / perPage)))).keys()).map(i => 1 + i),
pagination: core.pagination(page, (Math.ceil(count / perPage))),
})
})
})
};
I will be thankful for any advice.

But pagination shows me 10pages and I don't know how to fix it. When I parse count it gives me all results not only one,
You need to count only documents that match your search criteria, so rather than User.countDocuments() you'll want something like User.countDocuments(query).

Related

How to count and do jointure like stuff on couchDB

I'm currently trying to get the number of events for one organizer.
This is what my organizer document looks like:
{
"doc_type": "User",
"email": "xxx#gmail.com",
"blebleble: "blebleble",
}
This is what my event document looks like:
{
"doc_type": "Event",
"email": "xxx#gmail.com",
"blablabla: "blablabla",
}
I still couldn't figure out how to do some kind of jointure between both docs and do a count on the number of event that shares the same. I think I can work around the email that both docs shares but I don't know how I can do that. I'm still having trouble with CouchDB. Doesn't seems like a hard thing to do in SQL, but can't find out for nosql.
Thanks you in advance.
"jointure" is not not a term I've encountered in my field so I am left to guess what is meant is join.
Joins are possible with CouchDB views, but what I read from the requirement in the OP is to get counts of events by email. See CouchDB's Joins With Views documentation. For that, I don't see documents with an ancestral relation rather a one-to-many relation, i.e. user ==> events.
Consider this design document:
{
"_id": "_design/SO-68999682",
"views": {
"user_events": {
"map": `function (doc) {
if(doc.doc_type === 'Event') {
emit(doc.email);
}
}`,
"reduce": '_count'
}
}
The view's map function simply adds doc.email to the 'user_events' index when appropriate. Of particular interest the reduce function specifies the built-in reduce function _count.
Given such a view index one may apply the /db/_design/design-doc/_view/view-name endpoint to, for example,
View all events
{
reduce: false,
include_docs: true
}
Get a count of all events
{
reduce: true
}
Get a count of events for every email (summary)
{
reduce: true,
group_level: 1
}
Get a count of events for a specific email
{
reduce: true,
group_level: 1,
key: email
}
Get all events for a specific email
{
reduce: false,
include_docs: true,
key: email
}
The _count reduce built-in provides high performance. The snippet below demonstrates the above using the very handy and compatible PouchDB.
async function showAllEventDocs() {
let result = await db.query('SO-68999682/user_events', {
reduce: false,
include_docs: true
});
//show
gel('user_events_view').innerText = result.rows.map(row => [row.doc.email, row.doc.date].join('\t\t')).join('\n');
}
async function showEventCountTotal() {
let result = await db.query('SO-68999682/user_events', {
reduce: true
});
gel('event_count_total').innerText = result.rows[0].value;
}
async function showEventCountSummary() {
let result = await db.query('SO-68999682/user_events', {
reduce: true,
group_level: 1
});
//show key/value (email, count)
gel('event_count_summary').innerText = result.rows.map(row => [row.key, row.value].join('\t\t')).join('\n');
}
async function showUserEventCount(email, displayElement) {
let result = await db.query('SO-68999682/user_events', {
reduce: true,
group_level: 1,
key: email
});
//show value (count)
gel(displayElement).innerText = result.rows[0].value;
}
async function showUserEvents(email, displayElement) {
let result = await db.query('SO-68999682/user_events', {
reduce: false,
include_docs: true,
key: email
});
//show
gel(displayElement).innerText = result.rows.map(row => [row.doc.email, row.doc.date].join('\t\t')).join('\n');
}
function getDocsToInstall(count) {
const docs = [{
"doc_type": "User",
"email": "Jerry#gmail.com"
},
{
"doc_type": "User",
"email": "Bobby#gmail.com"
},
{
"doc_type": "Event",
"email": "Jerry#gmail.com",
"date": getDocDate().toISOString().slice(0, 10)
}, {
"doc_type": "Event",
"email": "Jerry#gmail.com",
"date": getDocDate().toISOString().slice(0, 10)
}, {
"doc_type": "Event",
"email": "Jerry#gmail.com",
"date": getDocDate().toISOString().slice(0, 10)
}, {
"doc_type": "Event",
"email": "Bobby#gmail.com",
"date": getDocDate().toISOString().slice(0, 10)
}, {
"doc_type": "Event",
"email": "Bobby#gmail.com",
"date": getDocDate().toISOString().slice(0, 10)
},
];
// design document
const ddoc = {
"_id": "_design/SO-68999682",
"views": {
"user_events": {
"map": `function (doc) {
if(doc.doc_type === 'Event') {
emit(doc.email);
}
}`,
"reduce": '_count'
}
}
};
docs.push(ddoc);
return docs;
}
const db = new PouchDB('SO-68999682', {
adapter: 'memory'
});
// install docs and show view in various forms.
(async() => {
await db.bulkDocs(getDocsToInstall(20));
await showAllEventDocs();
await showEventCountTotal();
await showEventCountSummary();
await showUserEventCount('Jerry#gmail.com', 'jerry_event_count');
await showUserEventCount('Bobby#gmail.com', 'bobby_event_count');
await showUserEvents('Jerry#gmail.com', 'jerry_events');
await showUserEvents('Bobby#gmail.com', 'bobby_events');
})();
const gel = id => document.getElementById(id);
function getDocDate() {
const today = new Date();
const day = Math.random() * 100 % today.getDay() + 1; // keep it basic
return new Date(today.getFullYear(), today.getMonth(), day)
}
.bold {
font-weight: bold
}
.plain {
font-weight: normal
}
<script src="https://cdn.jsdelivr.net/npm/pouchdb#7.1.1/dist/pouchdb.min.js"></script>
<script src="https://github.com/pouchdb/pouchdb/releases/download/7.1.1/pouchdb.memory.min.js"></script>
<pre>All user_events (entire view)</pre>
<pre id='user_events_view'></pre>
<hr/>
<pre>Total number of events: <span id='event_count_total'></span> events</pre>
<hr/>
<pre>Event count summary (user, count)</pre>
<pre id='event_count_summary'></pre>
<hr/>
<pre>Event count by email (specific to user)</pre>
<pre>Bobby#gmail.com has <span id='bobby_event_count'></span> events</pre>
<pre>Jerry#gmail.com has <span id='jerry_event_count'></span> events</pre>
<hr/>
<pre>Events by email</pre>
<pre class="bold">Bobby#gmail.com <pre class="plain" id='bobby_events'></pre></pre>
<pre class="bold">Jerry#gmail.com <pre class="plain" id='jerry_events'></pre></pre>
<hr/>
Notice the demo snippet's documents have a date field. If such a field existed in the OPs Event documents, then changing the emit to
emit(doc.email + '/' + doc.date);
would allow all the aforementioned queries plus the option to query by a date or date range, an exercise which I'll leave readers to explore.

Firebase database collection returns empty array when trying to get all documents

I'm trying to get all documents from my database collection "posts" but I'm getting an empty array instead.
The strange thing is that I'm able to get all documents from another collection called "users" that has the same structure and using the exact same code.
I've spent days looking for an answer but I haven't been able to find the solution.
This is the request:
const db = admin.firestore();
exports.getAllPosts = (req, res) => {
db.collection('posts')
.orderBy('createdAt', 'desc')
.get()
.then(snapshot => {
let posts = [];
snapshot.forEach((doc) => {
posts.push({
id: doc.id,
body: doc.data().body,
author: doc.data().author,
createdAt: doc.data().timestamp,
voteScore: doc.data().voteScore
});
});
return res.json(posts);
})
.catch(err => {
console.error(err);
res.status(500).json({ error: err.code });
});
}
And this is the response:
[]
This is what my current collection looks like:
Posts collection screenshot
This the response that I get when I return "snapshot":
{
"_query": {
"_firestore": {
"_settings": {
"projectId": "readable-bf7a6",
"firebaseVersion": "9.6.0",
"libName": "gccl",
"libVersion": "4.10.0 fire/9.6.0"
},
"_settingsFrozen": true,
"_serializer": {
"allowUndefined": false
},
"_projectId": "readable-bf7a6",
"registeredListenersCount": 0,
"bulkWritersCount": 0,
"_backoffSettings": {
"initialDelayMs": 100,
"maxDelayMs": 60000,
"backoffFactor": 1.3
},
"_clientPool": {
"concurrentOperationLimit": 100,
"maxIdleClients": 1,
"activeClients": {},
"failedClients": {},
"terminated": false,
"terminateDeferred": {
"promise": {}
}
}
},
"_queryOptions": {
"parentPath": {
"segments": []
},
"collectionId": "posts",
"converter": {},
"allDescendants": false,
"fieldFilters": [],
"fieldOrders": [
{
"field": {
"segments": [
"createdAt"
]
},
"direction": "DESCENDING"
}
],
"kindless": false
},
"_serializer": {
"allowUndefined": false
},
"_allowUndefined": false
},
"_readTime": {
"_seconds": 1622395245,
"_nanoseconds": 513743000
},
"_size": 0,
"_materializedDocs": null,
"_materializedChanges": null
}
Notice how the request for the collection "users" works successfully:
const db = admin.firestore();
exports.getAllUsers = (req, res) => {
db.collection('users')
.orderBy('createdAt', 'desc')
.get()
.then(snapshot => {
let users = [];
snapshot.forEach((doc) => {
let users = [];
snapshot.forEach((doc) => {
users.push({
id: doc.data().userId,
email: doc.data().email,
handle: doc.data().handle
});
});
return res.json(users);
})
.catch(err => {
console.error(err);
res.status(500).json({ error: err.code });
});
}
And the response:
[
{
"id": "EPoHBxhQFUXbcL3TCVx1LdUG2nO2",
"email": "ruben#gmail.com"
},
{
"id": "RqEa3dEq8TSDcZYeolXafju67rB2",
"email": "user10#gmail.com"
},
{
"id": "dxveb4n2iMQej5Q14uprsKRxFp23",
"email": "user4#gmail.com",
"handle": "user4"
},
{
"id": "YQPzBPcsqlVZk9iJEuZTHKUNuVG2",
"email": "user2#gmail.com",
"handle": "user2"
},
{
"id": "CZ05BJxi3TUOpIrmBaz539OWlbC3",
"email": "user#gmail.com",
"handle": "user"
},
{
"id": "t0t83BVwt4gVgJkDv7HL1r1MaKr1",
"email": "userJose2#gmail.com",
"handle": "Jose"
}
]
This is what the users collection looks like in Firebase:
Users collection screenshot
Why is one collection failing when the other works fine and I'm using the same code? What am I missing here?
Thanks in advance and I hope I've made it as clear as possible. Please let me know if you need me to provide anything else.
Very simple my friend, your posts documents can't be ordered like this:
.orderBy('createdAt', 'desc')
Because the post documents does not have the createdAt property, but they have a timestamp property, you should use that property to order your posts like this:
.orderBy('timestamp', 'desc')
I hope that helps 👍
I am not answering directly to #Ruben Garcia Bri, but for future firebase developers who may run into the problem of getting empty documents, I also ran into the same problem, but I solved it by adding a field to the particular document I am trying to retrieve.
Sometimes the cause is because the documents you are trying to get have no field in them.
I mean that a document must have a field before the server can recognize it as an existing document.
So if you run into this problem, consider adding a field to that document before you can successfully retrieve it.

node pagination in json array

I have implemented the following code for pagination but I am unable to make it work. I need to complete it with the following json data:
following code is for Vt.js file
const express = require('express');
const app = express();
const cors = require('cors');
var MongoClient = require('mongodb').MongoClient;
app.use(cors())
var url = "mongodb://localhost:27017/";
var pagination = require('pagination');
var paginator = pagination.create('search', {
prelink: '/users',
current: 2,
rowsPerPage: 2,
totalResult: 10020
});
console.log(paginator.render());
app.get('/users', function(req, res) {
MongoClient.connect(url, function(err, db) {
if (err)
throw err;
var dbo = db.db("MyNewDatabase");
var data = dbo.collection("VirtualCapitalDB").find({}).toArray(function(err, result) {
if (err)
throw err;
console.log(result);
res.json(result);
db.close();
});
});
})
app.listen(8080, ()=>console.log('Listening on port 8080'));
once you access it without pagination you can get following json array
[{
"_id": "5bcb3c77dc56e939187c13a5",
"firstname": "dumindu",
"lastname": "nagasinghe",
"videocount": 5
}, {
"_id": "5bcb3ce6dc56e939187c13a9",
"firstname": "cha",
"lastname": "advv",
"videocount": 10
}, {
"_id": "5bcb3d4bdc56e939187c13ab",
"firstname": "dvvs",
"lastname": "scvssv",
"videocount": 4
}, {
"_id": "5bcb3d7adc56e939187c13ac",
"firstname": "advav",
"lastname": "dvdvv",
"videocount": 5
}, {
"_id": "5bcb40f7a768f83918480a2b",
"firstname": "advav",
"lastname": "dvdvv",
"videocount": 5
}, {
"_id": "5bcb4103a768f83918480a2c",
"firstname": "advav",
"lastname": "dvdvv",
"videocount": 5
}, {
"_id": "5bcb4106a768f83918480a2d",
"firstname": "advav",
"lastname": "dvdvv",
"videocount": 5
}, {
"_id": "5bcb4125a768f83918480a2e",
"firstname": "advav",
"lastname": "dvdvv",
"videocount": 5
}]
But I need to add pagination and get the 2 data objects for single page. How to implement that in code?
Best way to write pagination of array items
paginator(items, page, per_page) {
var page = page || 1,
per_page = per_page || 10,
offset = (page - 1) * per_page,
paginatedItems = items.slice(offset).slice(0, per_page),
total_pages = Math.ceil(items.length / per_page);
return {
page: page,
per_page: per_page,
pre_page: page - 1 ? page - 1 : null,
next_page: (total_pages > page) ? page + 1 : null,
total: items.length,
total_pages: total_pages,
data: paginatedItems
};
}
Try this module.
// Fetch all running lent loans
module.exports.fetchLends = function (req, res, next) {
var perPage = 5;
var page = req.body.page || 1; // req.body.page pass from client, which page required.
loans
.find({
lenderId: req.user._id,
_disbursed: true,
_approved: true,
_completed: false
})
.select(
"lenderId _disbursed _approved _completed userId disbursed_timestamp status principal lender_interest_amount emi_type completed_timestamp disbursed_timestamp emi.lender_interest_amount emi._disbursed emi._completed emi.principal emi.lender_interest_amount emi.status emi.due_date emi.extension_date_1 emi.extension_date_2 emi.extension_date_3"
)
.populate({
path: "userId",
select: "name"
})
.skip(perPage * page - perPage)
.limit(perPage)
.sort({
disbursed_timestamp: -1
})
.exec(function (err, loan) {
console.log(loan)
)}
}
You want to use the express-paginate module, it's better suited for this case. See the documentation on their github page.
The pagination module you are using needs to create a new http request to get the data, it's not what you need for your application.

Pagination in Mongodb Nodejs

is there any method to get the total count of document in the find operation along with the skip and limit in the query in MongoDB
MongoClient.connect(Config.dbURI, function (err, db) {
if (!err) {
console.log("We are connected");
console.log(uniqueId)
db.collection(dbName).find({'uniqueId': uniqueId, 'isDeleted': false})
.sort({modifiedDateISO: -1})
.limit(parseInt(limit))
.skip(parseInt(limit * page))
.toArray((errFindChat, dataFindChat) => {
console.log(errFindChat, dataFindChat);
});
});
I assume "uniqueId" is not the primary key!
MongoClient.connect(Config.dbURI, function (err, db) {
if (!err) {
console.log("We are connected");
console.log(uniqueId)
db.collection("collname").aggregate(
[
{ "$match": { "uniqueId": uniqueId, 'isDeleted': false} },
{ "$count": "total" },
{ "$sort" : {"modifiedDateISO": -1 },
{ "$limit": parseInt(limit) },
{ "$skip" : parseInt(limit * page) }
]
).toArray((errFindChat, dataFindChat) => {
console.log(errFindChat, dataFindChat);
});
}
});
MongoDB Aggregate Count
You can't filter with skip and limit and have the total count in only one request if you use .find..
If you want to retrieve documents, filter, and perform count operation in only one request you have to use aggregate
db.coll.aggregate([
{$match: your find conditions},
{$group/project: your count operations, etc....},
{$skip: skip}, // pagination skip
{$limit: limit}, // pagination limit
...
]);
var query = {}; // Your condition
var options = {
select: 'title date author',
sort: { date: -1 },
populate: 'author',
lean: true,
offset: 20,
limit: 10
};
Book.paginate(query, options).then(function(result) {
// ...
});
The mongoose-pagination is good

NodeJs with Mongoose - Nested queries async issue

I have to mongodb collections, like this:
UserGroup collection
fields:
name: String
group_id: Number
User collection
fields:
user_name: String
group_id: Number
I want generate a report like this:
ADMINISTRATORS
--------------------------
jlopez
rdiaz
OPERATORS
--------------------------
amiralles
dcamponits
But I get the following report:
ADMINISTRATORS
--------------------------
OPERATORS
--------------------------
jlopez
rdiaz
amiralles
dcamponits
Following is the code to generate the report:
UserGroup.find({}, (err, groups) => {
for(var i in groups){
console.log(groups[i].name)
console.log("--------------------")
User.find( {group_id : groups[i].group_id}, (err, users) =>{
for(var j in users){
console.log(users[j].user_name)
}
})
}
})
Clearly, this is a problem of the NodeJs/Mongoose asynchronicity.
QUESTION: How do I make the first For cycle wait until the internal cycle ends for each UserGrop?
Thanks in advance,
David.
You can run an aggregation pipeline that uses $lookup to do a "left-join" to another collection in the same database to filter in documents from the "joined" collection for processing. With this you won't need any async library:
UserGroup.aggregate([
{
"$lookup": {
"from": "users",
"localField": "group_id",
"foreignField": "group_id",
"as": "users"
}
},
{
"$project": {
"_id": 0,
"name": 1,
"users": {
"$map": {
"input": "$users",
"as": "user",
"in": "$$user.user_name"
}
}
}
}
], (err, groups) => {
if (err) throw err;
console.log(JSON.stringify(groups, null, 4));
})
Sample Output
[
{
"name": "ADMINISTRATORS",
"users": ["jlopez", "rdiaz"]
},
{
"name": "OPERATORS",
"users": ["amiralles", "dcamponits"]
}
]
Add support for promises to mongoose. I use q, but you can use bluebird too.
mongoose.Promise = require('q').Promise;
Then you can use q.all to resolve once all of the user queries have completed.
var promises = [];
var groups = [];
UserGroup.find({}, (err, groups) => {
for(var i in groups){
groups.push(groups[i]);
promises.push(User.find( {group_id : groups[i].group_id}));
}
});
q.all(promises).then( function(usersByGroup){
var indx = 0;
usersByGroup.forEach(function(users){
var grp = groups[indx];
console.log(groups[i].name);
console.log("--------------------");
for(var j in users){
console.log(users[j].user_name)
}
indx++;
});
});
This is a good use case for asyc, you can get a get basic idea from following code. it is based on async each & waterfall. [ Please add proper error handling for the following code yourself.]
UserGroup.find({}, (err, groups) => {
async.each(groups, (group, callback) =>{
async.waterfall([
(wCallback) => {
User.find({group_id : group.group_id}, wCallback)
},
(users, wCallback) => {
console.log(group.name)
console.log("--------------------")
for(var j in users){
console.log(users[j].user_name)
}
wCallback()
}
], callback)
})
})

Resources