server side pagination in node js and mongo db - node.js

please help me with this server side pagination in node js and mongo db
function getServiceQualityAnex(req, res, next) {
if (req.query.code != null) {
ServiceQualityAnex.find({ location: req.query.code }).sort({ _id: -1 }).select('-hash')
.then(serviceQualityAnexC => res.json(serviceQualityAnexC))
.catch(err => {
res.sendStatus(404);
next(err)
});
} else {
ServiceQualityAnex.find({}).sort({ _id: -1 }).select('-hash')
.then(serviceQualityAnexC => res.json(serviceQualityAnexC))
.catch(err => {
res.sendStatus(404);
next(err)
});
}
}

let page = Number(req.query.page);
page = page ? page : 0;
let limit = parseInt(req.query.limit);
const result = {};
let startIndex = page * limit;
if (startIndex > 0) {
result.previous = {
page: page - 1,
limit: limit,
};
}
let receive = await Model.find()
.sort("-_id")
.skip(startIndex)
.limit(limit)
.exec();

i want to do a server side pagination on server side my front end api is http://localhost:3000/getServiceQualityAnexJoin/ the function which is mentioned above is combing 2 tables are returning . my data is very huge and i want to add a server side pagination

You did not specify all the requirements in your question but what i precieve is that you want to do pagination on server side with nodejs in mongodb
Here is what you need to do:
const getServiceQualityAnex = async (request, response) => {
try {
const id = request.params.id;
let { page } = request.query; //this indicates the page you are requesting for in pagination
if (!page)
page = 1; //by default it is one
const result = await ServiceQualityAnex.aggregate([
{
$match: {
"_id": mongoose.Types.ObjectId(id)
}
},
{
$project: {
"_id": 1,
.
.
.
// all the fields you want to get
}
},
{
$facet: { //facet is the aggregation pipeline in mongodb through which you can achieve pagination
metadata: [{ $count: 'total' }],
data: [
{
$skip: (Number(page) - 1) * Number(20)
},
{
$limit: 20 //20 records limit per page
},
]
}
}
]);
console.log("result :", result[0].data);
return response
.status(200)
.json(
{
result: result[0].data,
meta: {
current_page: page,
total_pages: Math.ceil(
(Number(result[0].metadata.length === 0 ? 0 : result[0].metadata[0].total))
/ (Number(20))),
total_count: result[0].metadata.length === 0 ? 0 : result[0].metadata[0].total,
}
}
);
} catch (error) {
console.log(error);
response.status(500).json({
error: "Something went wrong",
});
}
}
If you don't know anything about aggregation then you must visit this site:
MongoDB aggregation framework

Related

How to use Quick sort in MERN Stack project

I have got a task where I have to use Quicksort instead of using the (MongoDB) mongoose default sort method.
I could not find any solution on how to implement it.
Current code:
export const getProduct = expressAsyncHandler(async (req, res) => {
const pageSize = req.headers.referer === DASHBOARD_SCREEN_ROUTE ? 12 : 6;
const page = Number(req.query.pageNumber) || 1;
const sortOrder =
order === 'lowest'
? { price: 1 }
: order === 'highest'
? { price: -1 }
: order === 'toprated'
? { rating: -1 }
: { _id: -1 };
const products = await Product.find({})
.sort(sortOrder)
.skip(pageSize * (page - 1))
.limit(pageSize);
try {
res.send({ products });
} catch (error) {
console.log(error);
res.status(512).send(error.message);
}
});
Instead of .sort method, I have to use quick sort there.

Is there an alternative for $expr in older versions of MongoDB? [duplicate]

This question already has answers here:
MongoDb query condition on comparing 2 fields
(4 answers)
Closed 4 years ago.
In a Node.js backend for a React.js app, I am using
"mongodb" (as db.version returns): "3.4.10",
"mongoose": "5.3.4"
The issue is I can't use $expr with a mongoDB version < 3.6.
It seems like a lot of effort to upgrade the mongoDB version juste because of this issue (deprecated methods and so on).
So I was wondering if there was a way to do what I am trying to do without using $expr ?
Here is the code :
match.$expr = {
$lt: ["$distance", "$range"] // "calculated_distance < tutor_range"
}
Would you know how ? Here is the full method, if ever you want some more details about it.
exports.search = (req, res) => {
let lat1 = req.body.lat;
let lon1 = req.body.lng;
let page = req.body.page || 1;
let perPage = req.body.perPage || 10;
let radius = req.body.radius || 10000;
let levelsIn = req.body.levels && req.body.levels.length !== 0 ? req.body.levels.map(level => {
return ObjectID(level);
}) : null;
let subjectsIn = req.body.subjects && req.body.subjects.length !== 0 ? req.body.subjects.map(subject => {
return ObjectID(subject);
}) : null;
var options = { page: page, limit: perPage, sortBy: { updatedDate: -1 } }
const isAdmin = req.user ? req.user.role === "admin" || req.user.role === "super-admin" : false;
let match = {}
if (levelsIn) match.levels = { $in: levelsIn };
if (subjectsIn) match.subjects = { $in: subjectsIn }
if (typeof req.body.activated !== "undefined") match.profileActivated = req.body.activated;
if (req.body.from) match.createdAt = { $gte: new Date(req.body.from) };
if (req.body.to) {
if (match.createdAt) match.createdAt.$lte = new Date(req.body.to);
else match.createdAt = { $lte: new Date(req.body.to) };
}
var aggregate = null;
if (!isAdmin) {
match.activated = true
match.profileActivated = true
match.profileOnline = true
}
if (lat1 && lon1) {
match.$expr = {
$lt: ["$distance", "$range"] // "calculated_distance < tutor_range"
}
aggregate = Tutor.aggregate([
{
"$geoNear": {
"near": {
"type": "Point",
"coordinates": [lon1, lat1]
},
"distanceField": "distance", // this calculated distance will be compared in next section
"distanceMultiplier": 0.001,
"spherical": true
}
},
{
$match: match
}
]);
} else {
aggregate = Tutor.aggregate([
{
$match: match
}
]);
}
Tutor
.aggregatePaginate(aggregate, options, function (err, result, pageCount, count) {
if (err) {
return res.status(400).send(err);
}
else {
var opts = [
{ path: 'levels', select: 'name' },
{ path: 'subjects', select: 'name' },
{ path: 'assos', select: 'name' }
];
Tutor
.populate(result, opts)
.then(result2 => {
return res.send({
page: page,
perPage: perPage,
pageCount: pageCount,
documentCount: count,
tutors: result2
});
})
.catch(err => {
return res.status(400).send(err);
});
}
})
};
Thank you very much for your answers and help !
You can use $addFields + $match instead of $expr.
Something like
{"$addFields":{"islt":{"$cond":[{"$lt": ["$distance", "$range"]}, true, false]}}},
{"$match":{"islt":true}}
You can use extra project stage to drop the islt variable if you like.
{"$project":{"islt":0}}

How to implement pagination and total with mongoose

I have not found how to return total data included in the resultset. Is there any way to return total data? I need to confirm that the correct JSON data is returned.
What I want is to have something like this returned:
total: 40,
page: 1,
pageSize: 3,
books: [
{
_id: 1,
title: "达·芬奇密码 ",
author: "[美] 丹·布朗",
},
{
_id: 2,
title: "梦里花落知多少",
author: "郭敬明",
},
{
_id: 3,
title: "红楼梦",
author: "[清] 曹雪芹",
}
]
}
Current code:
router.get('/booksquery', (req, res) => {
var page = parseInt(req.query.page) || 1;
var limit = parseInt(req.query.limit) || 3;
Book.find({})
.sort({ update_at: -1 })
.skip((page-1) * limit)
.limit(limit)
.exec((err, doc) => {
if (err) {
res.json(err)
} else {
res.json({
total: doc.total,
page: page,
pageSize: limit,
books:doc,
});
}
})
})
Probably you could do something like this:
Only if current page index is 0 default and pagination starts with next page, that is 1, your could do .skip(page * limit) make sure .skip() and .limit() gets Number.
router.get("/booksquery", (req, res) => {
var page = parseInt(req.query.page) || 0; //for next page pass 1 here
var limit = parseInt(req.query.limit) || 3;
var query = {};
Book.find(query)
.sort({ update_at: -1 })
.skip(page * limit) //Notice here
.limit(limit)
.exec((err, doc) => {
if (err) {
return res.json(err);
}
Book.countDocuments(query).exec((count_error, count) => {
if (err) {
return res.json(count_error);
}
return res.json({
total: count,
page: page,
pageSize: doc.length,
books: doc
});
});
});
});
from above you will get response like below:
{
books: [
{
_id: 1,
title: "达·芬奇密码 ",
author: "[美] 丹·布朗"
},
{
_id: 2,
title: "梦里花落知多少",
author: "郭敬明"
},
{
_id: 3,
title: "红楼梦",
author: "[清] 曹雪芹"
}
],
total: 3,
page: 0,
pageSize: 3
}
if you use any condition and over that you want to make query and get result and total on that basics. do it inside var query ={} .
and the same query will be used for .count() also.
so you can get total count on the basics of that condition.
In case someone wants to see this using async/await in 2020, and some random query instead of an empty object. Remember that estimatedDocumentCount does not work using query filters, it needs to be countDocuments by using the query on it. Please check:
const { userId, page, limit } = req.headers;
const query = {
creator: userId,
status: {
$nin: ['deleted'],
},
};
const page_ = parseInt(page, 10) || 0;
const limit_ = parseInt(limit, 10) || 12;
const books = await BookModel.find(query)
.sort({ update_at: -1 })
.skip(page_ * limit_)
.limit(limit_);
const count = await BookModel.countDocuments(query);
count() is deprecated(form mongoose#5.x). estimatedDocumentCount() is faster than using countDocuments() for large collections because estimatedDocumentCount() uses collection metadata rather than scanning the entire collection.
Reference: https://mongoosejs.com/docs/api/model.html#model_Model.estimatedDocumentCount
So the code is perfect just replace countDocuments to estimatedDocumentCount
router.get("/booksquery", (req, res) => {
var page = parseInt(req.query.page) || 0; //for next page pass 1 here
var limit = parseInt(req.query.limit) || 3;
var query = {};
Book.find(query)
.sort({ update_at: -1 })
.skip(page * limit) //Notice here
.limit(limit)
.exec((err, doc) => {
if (err) {
return res.json(err);
}
Book.estimatedDocumentCount(query).exec((count_error, count) => {
if (err) {
return res.json(count_error);
}
return res.json({
total: count,
page: page,
pageSize: doc.length,
books: doc
});
});
});
});

Buildfire - How to make querying data more reliable

I am making simple calls to the datastore and I am being plagued by "unhandled packets".
When I make calls to the datastore sometimes I'll get all my data, sometimes I'll get half, sometimes I'll get nothing like in the screenshot above. I can see the data that I'm looking for in that error message but it does not make it to my callback function.
Here is an example of a function that I run to get all of my data:
getData(searchObj, list) {
let search = searchObj ? searchObj : {};
search = { ...search, ...{ "sort": { "name": 1 }} };
const page = search.page || 0;
search = { ...search, ...{ pageSize: 10, page }};
buildfire.datastore.search({}, 'some-tag', (err, res) => {
if (err) {
console.log(err);
}
else {
if (res.length === 0 || res.length !== 10) {
const dataList = [ ...list, ...res ];
this.setState({
loading: false,
dataList: _.chunk(dataList, 10)
});
} else {
const dataList = [ ...list, ...res ];
search.page = search.page + 1;
this.getData(search, dataList);
}
}
})
}
getData(null, []);

How to get json data from multiple url-pages Node.js?

I am quite new to Node.js and haven't been working with json data before so really hope that you can help me.
I am trying to get all event information from Ticketmaster's API and add specific variables to mongoDB. However, the APIs' that I am currently using are limited to 200 events per page. It is therefore not possible for me to connect the event information with venue information since these are added seperately to mongoDB and are not exhaustive of all event and venue information (not able to connect on ids because of missing event and venue data).
My question is therefore in regards to how I can get all pages into my database at once?
The code that I have written so far looks something like below:
app.get('/tm', (req, res) => {
axios // getting venues
.get('https://app.ticketmaster.com/discovery/v2/venues.json?apikey=myApiKey&page=0&size=200&countryCode=DK')
.then(response => {
const venuesToBeInserted = response.data._embedded.venues.map(venue => { // preparing venues
return {
sourceID: venue.id,
venue: venue.name,
postalCode: venue.postalCode,
city: venue.city.name,
country: venue.country.name,
countryCode: venue.country.countryCode,
address: !!venue.address ? venue.address.line1 : null,
longitude: !!venue.location ? venue.location.longitude : null,
latitude: !!venue.location ? venue.location.latitude : null,
source: 'ticketmaster'
}
})
// Create all venues at once
Venue.create(venuesToBeInserted).then(venues => {
console.log("venues inserted")
axios // getting events and shows - note the page parameter in the api link
.get('https://app.ticketmaster.com/discovery/v2/events.json?apikey=myApiKey&countryCode=DK&size=200&page=0')
.then(response => {
const eventsToBeInserted = response.data._embedded.events.map(events => { // preparing events
const event = events._embedded.attractions[0]
return {
sourceID: event.id,
name: event.name,
slug: slugify(event.name).toLowerCase(),
tags: !!event.classifications ? [event.classifications[0].genre.name, event.classifications[0].subGenre.nam] : [], // duplicate genres occur
// possible tags from ticketmaster: type and subtype
}
})
// Create all events at once
Event.create(eventsToBeInserted).then(events => {
console.log("events inserted")
const showsToBeInserted = response.data._embedded.events.map(show => {
const event = events.find(event => event.sourceID == show._embedded.attractions[0].id);
const venue = venues.find(venue => venue.sourceID == show._embedded.venues[0].id);
if (!!event && !!venue) {
return {
event: event._id,
venue: venue._id,
timezone: show.dates.timezone,
dateStart: !!show.dates.start.dateTime ? show.dates.start.dateTime : show.dates.start.localDate,
tickets: !!show.priceRanges ? {
minPrice: show.priceRanges[0].min,
maxPrice: show.priceRanges[0].max,
currency: show.priceRanges[0].currency
}: {}
}
}
})
// Let's see what we have created in the database
Venue.find({}).select({
name: 1,
slug: -1
}).limit(10).populate('event').populate('venue').then(events => {
console.log(util.inspect(events));
}).catch(err => {
console.error(err);
});
}).catch( err => {
console.error(err)
})
}).catch( err => {
console.error(err)
})
}).catch(err => {
console.error(err)
});
}).catch(err => {
console.error(err)
})
})
EDIT
Using the approach that Jake suggested gave me an error (Error: Requested failed with status code 401). I have tried to search for it online but I cannot figure out why the error happens.. See picture below of part of the error message in my console.log.
error message
You can do this using promises, which you already are using, you just need to chain them together using recursion.
function getVenues(page, size, venues) {
page = page || 0;
size = size || 200;
venues = venues || [];
return axios
.get(`https://app.ticketmaster.com/discovery/v2/venues.json?apikey=myApiKey&page=${page}&size=${size}&countryCode=DK`)
.then(response => response.data._embedded.venues)
.then(rawVenues => {
rawVenues.forEach(venue => venues.push(venue));
if (rawVenues.length < size) {
// All done return the compiled list.
return venues;
}
// Recurse over the next set of venues by adding another promise to the chain.
return getVenues(page + 1, size, venues);
});
}
function getEvents(page, size, events) {
page = page || 0;
size = size || 200;
events = events || [];
return axios
.get(`https://app.ticketmaster.com/discovery/v2/events.json?apikey=myApiKey&countryCode=DK&size=${size}&page=${page}`)
.then(response => response.data._embedded.events)
.then(rawEvents => {
rawEvents.forEach(event => events.push(event));
if (rawEvents.length < size) {
// All done return the compiled list.
return events;
}
// Recurse over the next set of events by adding another promise to the chain.
return getEvents(page + 1, size, events);
});
}
app.get('/tm', (req, res) => {
getVenues().then(rawVenues => {
const venuesToBeInserted = rawVenues.map(venue => {
return {
sourceID: venue.id,
venue: venue.name,
postalCode: venue.postalCode,
city: venue.city.name,
country: venue.country.name,
countryCode: venue.country.countryCode,
address: !!venue.address ? venue.address.line1 : null,
longitude: !!venue.location ? venue.location.longitude : null,
latitude: !!venue.location ? venue.location.latitude : null,
source: 'ticketmaster'
};
});
// Return promise so errors bubble up the chain...
return Venue.create(venuesToBeInserted).then(venues => {
console.log("venues inserted");
// Return promise so errors bubble up the chain...
return getEvents().then(rawEvents => {
const eventsToBeInserted = rawEvents.map(rawEvent => {
const event = events._embedded.attractions[0];
return {
sourceID: event.id,
name: event.name,
slug: slugify(event.name).toLowerCase(),
tags: !!event.classifications ? [event.classifications[0].genre.name, event.classifications[0].subGenre.nam] : []
};
});
// Return promise so errors bubble up the chain...
return Event.create(eventsToBeInserted).then(events => {
console.log("events inserted");
const showsToBeInserted = rawEvents.map(show => {
const event = events.find(event => event.sourceID == show._embedded.attractions[0].id);
const venue = venues.find(venue => venue.sourceID == show._embedded.venues[0].id);
if (!!event && !!venue) {
return {
event: event._id,
venue: venue._id,
timezone: show.dates.timezone,
dateStart: !!show.dates.start.dateTime ? show.dates.start.dateTime : show.dates.start.localDate,
tickets: !!show.priceRanges ? {
minPrice: show.priceRanges[0].min,
maxPrice: show.priceRanges[0].max,
currency: show.priceRanges[0].currency
} : {}
}
}
});
// Do something with the found shows...
});
});
});
}).then(() => { // This then is fired after all of the promises above have resolved...
return Venue.find({}).select({
name: 1,
slug: -1
}).limit(10).populate('event').populate('venue').then(events => {
console.log(util.inspect(events));
res.send(events);
});
}).catch(err => { // Catches any error during execution.
console.error(err);
res.status(500).send(err);
});;
});

Resources