I am making simple calls to the datastore and I am being plagued by "unhandled packets".
When I make calls to the datastore sometimes I'll get all my data, sometimes I'll get half, sometimes I'll get nothing like in the screenshot above. I can see the data that I'm looking for in that error message but it does not make it to my callback function.
Here is an example of a function that I run to get all of my data:
getData(searchObj, list) {
let search = searchObj ? searchObj : {};
search = { ...search, ...{ "sort": { "name": 1 }} };
const page = search.page || 0;
search = { ...search, ...{ pageSize: 10, page }};
buildfire.datastore.search({}, 'some-tag', (err, res) => {
if (err) {
console.log(err);
}
else {
if (res.length === 0 || res.length !== 10) {
const dataList = [ ...list, ...res ];
this.setState({
loading: false,
dataList: _.chunk(dataList, 10)
});
} else {
const dataList = [ ...list, ...res ];
search.page = search.page + 1;
this.getData(search, dataList);
}
}
})
}
getData(null, []);
Related
please help me with this server side pagination in node js and mongo db
function getServiceQualityAnex(req, res, next) {
if (req.query.code != null) {
ServiceQualityAnex.find({ location: req.query.code }).sort({ _id: -1 }).select('-hash')
.then(serviceQualityAnexC => res.json(serviceQualityAnexC))
.catch(err => {
res.sendStatus(404);
next(err)
});
} else {
ServiceQualityAnex.find({}).sort({ _id: -1 }).select('-hash')
.then(serviceQualityAnexC => res.json(serviceQualityAnexC))
.catch(err => {
res.sendStatus(404);
next(err)
});
}
}
let page = Number(req.query.page);
page = page ? page : 0;
let limit = parseInt(req.query.limit);
const result = {};
let startIndex = page * limit;
if (startIndex > 0) {
result.previous = {
page: page - 1,
limit: limit,
};
}
let receive = await Model.find()
.sort("-_id")
.skip(startIndex)
.limit(limit)
.exec();
i want to do a server side pagination on server side my front end api is http://localhost:3000/getServiceQualityAnexJoin/ the function which is mentioned above is combing 2 tables are returning . my data is very huge and i want to add a server side pagination
You did not specify all the requirements in your question but what i precieve is that you want to do pagination on server side with nodejs in mongodb
Here is what you need to do:
const getServiceQualityAnex = async (request, response) => {
try {
const id = request.params.id;
let { page } = request.query; //this indicates the page you are requesting for in pagination
if (!page)
page = 1; //by default it is one
const result = await ServiceQualityAnex.aggregate([
{
$match: {
"_id": mongoose.Types.ObjectId(id)
}
},
{
$project: {
"_id": 1,
.
.
.
// all the fields you want to get
}
},
{
$facet: { //facet is the aggregation pipeline in mongodb through which you can achieve pagination
metadata: [{ $count: 'total' }],
data: [
{
$skip: (Number(page) - 1) * Number(20)
},
{
$limit: 20 //20 records limit per page
},
]
}
}
]);
console.log("result :", result[0].data);
return response
.status(200)
.json(
{
result: result[0].data,
meta: {
current_page: page,
total_pages: Math.ceil(
(Number(result[0].metadata.length === 0 ? 0 : result[0].metadata[0].total))
/ (Number(20))),
total_count: result[0].metadata.length === 0 ? 0 : result[0].metadata[0].total,
}
}
);
} catch (error) {
console.log(error);
response.status(500).json({
error: "Something went wrong",
});
}
}
If you don't know anything about aggregation then you must visit this site:
MongoDB aggregation framework
I am trying to do a call which retrieves a list of categories. Inside this call I want to loop through the categories and retrieve the items for each category and return them all together. My call retrieves the categories perfectly before I added the loop to retrieve the items.
To double check my call to another controller works, I added a proof of concept block of code which you can see below is commented out. So I know it isn't the call to an external class.
Here is my code:
'use strict';
var mongoose = require('mongoose'),
MenuCategory = mongoose.model('MenuCategory');
module.exports = function(menuItemController) {
var mod = {
listEntireMenu(req, res) {
return new Promise(function(resolve, reject) {
var entireMenu = [];
MenuCategory.find({}, function(err, menuCategories) {
if (err) {
return reject(err)
} else {
//---------------------------
// PROOF OF CONCEPT THAT CALL TO OTHER CONTROLLER WORKS
//---------------------------
//
// var categoryWithItems = menuCategories[0].toObject();
// req.body.menuCategoryID = categoryWithItems._id;
// menuItemController.listAllMenuItemsByCategory(req, res).then((menuItems) => {
// if(menuItems)
// {
// return resolve(menuItems);
// }
// else
// {
// return { success: false }
// }
// });
//-----------------------------
for (var i = 0; i < menuCategories.length; i++) {
var categoryWithItems = menuCategories[i].toObject();
var subItems = [];
req.body.menuCategoryID = categoryWithItems._id;
menuItemController.listAllMenuItemsByCategory(req, res).then((menuItems) => {
if(menuItems)
{
subItems = menuItems;
}
else
{
return { success: false }
}
});
categoryWithItems.tester = { "itemsList" : subItems };
entireMenu.push(categoryWithItems);
}
return resolve(entireMenu)
}
});
}).then((menuCategories) => {
if(menuCategories)
{
return menuCategories
}
else
{
return { success: false }
}
});
},
}
return mod;
};
What I actually get returned is this :
[
{
"_id": "5ed16fxxxxxxxx95676e37",
"locationID": "5ed16xxxxxxxx7295676e36",
"menuCategoryName": "Category One",
"Created_date": "2020-05-29T20:26:34.991Z",
"__v": 0,
"tester": {
"itemsList": []
}
},
{
"_id": "5ed170xxxxxx95676e38",
"locationID": "5ed16xxxxxxxx7295676e36",
"menuCategoryName": "Category Two",
"Created_date": "2020-05-29T20:26:48.799Z",
"__v": 0,
"tester": {
"itemsList": []
}
}
]
Here is the call from the route.js :
app.get('/api/listEntireMenu', (req, res) => {
menuCategoryController.listEntireMenu(req, res).then(menuCategories => res.json(menuCategories));
})
It never writes the subItems into the object. Is this an async issue or something else? I am not sure how to solve this.
Thanks in advance.
i believe the reason the result of your call to resolve is being returned before the requests are able to complete...for this you need to wait until all the promises or requests have finished properly and returned.
There are two ways you can do this: you could either run them one by one and wait for each one to finish first or run them all concurrently until all of them are done.
Ofcourse the fastest way to do it would be to run them all concurrently so lets go for that way:
so to start, let us not use the for loop and instead remap the iterable array menuCategories to promises of the request, we will use your proof of concept code to make the array of promises
//...
Promise.all(
menuCategories.map((category) => {
let category_with_items = category.toObject();
req.body.menuCategoryID = category_with_items._id;
// here we need to return this since its the promise we are remapping to
return menuItemController.listAllMenuItemsByCategory(req, res)
.then((menuitems) => {
if(menuItems) {
return menuitems;
}
throw 'No menu items found'
});
});
)
// each promise will return menuitems so we have to wait for all the promises to complete
// then with the results of each promise we push the items into the entire menu
.then((itemslist) => {
itemslist.forEach((items) => entireMenu.push(items));
return entireMenu;
})
// lastly we need to handle any errors from the promises
.catch((error) => { success: false });
//...
So now we have...
listEntireMenu(req, res) {
return MenuCategory.find({}, function(err, menuCategories) {
if (err) {
throw err
} else {
entireMenu = [];
return /* the promise all call from above will go right here */;
}
}
I hope it works out, thanks...
Can somebody help me with this?
I am trying to scrape a website and store the collected data in a Json file. I'm using cheerios and request-promise.
The Json structure goes like that: companys > packages > cities
"companies": [
{
"id": 0,
"name": "companyName",
"url": "https://www.url-company.net/",
"packages": [
{
"id": 0,
"name": "general",
"url": "https://www.url-company.net/package",
"cities": [
{
"id": 0,
"name": "cityName",
"url": "https://www.url-company.net/package/city",
},
...]
}
...]
}
..]
I have extracted the array of companies from this site.
Each COMPANY has a specific url --> from every url I scraped the
packages for each company.
Each PACKAGE has a specific url --> from
every url I want to scrape the cities for each package but I am NOT
able to do it.
I am only able to populate companies and packagesByCompany, but I'm lost when trying to populate citiesByPackage:
const rp = require('request-promise');
const cheerio = require('cheerio');
const jsonfile = require('jsonfile');
const baseUrl = 'https://www.base-url-example.net';
scrapeAll();
function scrapeAll() {
return scrapeCompanies().then(function (dataCompanys) {
//Map every endpoint so we can make a request with each URL
var promises = dataCompanys.map(function (company) {
return scrapePackagesByCompany(company) // Populate each company with all the array of packages from this company
});
return Promise.all(promises);
})
.then(function(promiseArray) { // Need help here!!!!
var promise4all = Promise.all(
promiseArray.map(function(company) {
return Promise.all( // This is NOT working, I do not know how to get promises from nested arrays
company.packages.map(function(package) {
return Promise.all(
scrapeCitiesByPackage(package) // Try to populate each package with all the array of cities from this package
);
})
);
})
);
return promise4all;
})
.then(function (data) {
saveScrapedDateIntoJsonFile(data);
return data;
})
.catch(function (err) {
return Promise.reject(err);
});
}
function scrapeCompanies() {
return rp(baseUrl)
.then(function(html){
const data = [];
let companysImg = '#content section .elementor-container > .elementor-row > .elementor-element.elementor-top-column .elementor-widget-wrap .elementor-widget-image >.elementor-widget-container > .elementor-image';
let $ = cheerio.load(html);
$(companysImg).each(function(index, element){
const urlCompany = $(element).find('a').attr('href');
const imgCompany = $(element).find('img').data('lazy-src');
if (urlCompany && imgCompany) {
const nameCompany = urlCompany;
const company = {
id : index,
name: nameCompany,
url : baseUrl + urlCompany,
img: imgCompany,
};
data.push(company);
}
});
return data;
})
.catch(function(err){
//handle error
console.error('errorrr2', err);
});
}
function scrapePackagesByCompany(company) {
return rp(company.url)
.then(function(html){
company.packages = [];
let packagesImg = '#content section .elementor-container > .elementor-row > .elementor-element.elementor-top-column .elementor-widget-wrap .elementor-widget-image >.elementor-widget-container > .elementor-image';
let $ = cheerio.load(html);
$(packagesImg).each(function(index, element){
const urlPackage = $(element).find('a').attr('href');
const imgPackage = $(element).find('img').data('lazy-src');
if (urlPackage && imgPackage) {
const namePackage = urlPackage.text();
const package = {
id : index,
name: namePackage,
url : urlPackage,
img: imgPackage,
};
company.packages.push(package);
}
});
return company;
})
.catch(function(err){
//handle error
console.error('errorrr2', err);
});
}
function scrapeCitiesByPackage(insurancePackage) {
return rp(insurancePackage.url)
.then(function(html){
insurancePackage.cities = [];
let citiesLinks = '#content section .elementor-container > .elementor-row > .elementor-element .elementor-widget.elementor-widget-posts .elementor-posts-container article';
let $ = cheerio.load(html);
$(citiesLinks).each(function(index, element) {
const $linkCity = $(element).find('a');
const urlCity = $linkCity.attr('href');
const nameCity = $linkCity.text();
if (urlCity && nameCity) {
const city = {
id : index,
name: nameCity,
url : urlCity,
};
insurancePackage.cities.push(city);
}
});
return insurancePackage;
})
.catch(function(err){
//handle error
console.error('errorrr2', err);
});
}
function saveScrapedDateIntoJsonFile(data) {
jsonfile.writeFile(
'./data/company.json',
{companies : data },
//data,
{spaces: 2},
function(err) {
console.error('errorrr', err);
});
}
Thanks in advance :)
What you are trying could be made to work but it's arguably better for scrapePackagesByCompany() and scrapeCitiesByPackage() simply to deliver data, and to perform all the "assembly" work (ie bundling the delivered arrays into higher level objects) in scrapeAll().
You can write something like this:
scrapeAll()
.catch(function(err) {
console.log(err);
});
function scrapeAll() {
return scrapeCompanies()
.then(function(companies) {
return Promise.all(companies.map(function(company) {
return scrapePackagesByCompany(company)
.then(function(packages) {
company.packages = packages; // assembly
return Promise.all(packages.map(function(package) {
return scrapeCitiesByPackage(package)
.then(function(cities) {
package.cities = cities; // assembly
});
}));
});
}))
.then(function() {
return saveScrapedDateIntoJsonFile(companies);
});
});
}
Then it's fairly trivial to simplify scrapePackagesByCompany() and scrapeCitiesByPackage(package) such that they deliver packages array and cities array respectively.
I am using mongodb find query inside loop as I need to run the find query 5 times. And I used below code for that:
let result = {};
let miles = ['5','10','15','25'];
let i = 0;
while (i < miles.length) {
Shops.find({ 'shopInfo.address':{ $geoWithin:{ $centerSphere: [ [ 75.83183541365247, 30.902146005639267 ], miles[i] / 3959 ] } } }).then(response=>{
if(i==4){
result[miles[i]] = response.length;
res.json(result);
}else{
result[miles[i]] = response.length;
i++;
}
})
.catch(err=>{
console.log(err)
});
}
And when I hit the api on browser. It's not returning with anything and getting below error in console:
Please help me, How can I solve the issue?
The following happens:
Your while loop runs and starts an async action. However as that async action is not finished yet, the i++ will not be executed, therefore the loop runs forever, creates more and more async actions that fill up your memory, and finally NodeJS crashes because it runs out of memory. To prevent that, you should not synchronously iterate with an asynchronous task. Either await the asynchronous task inside a loop, or .map the miles to the asynchronous tasks then await them all:
const entries = Promise.all(miles.map(mile =>
Shops.find({ 'shopInfo.address':{ $geoWithin:{ $centerSphere: [ [ 75.83183541365247, 30.902146005639267 ], mile / 3959 ] } } })
.then(entry => ([mile, entry.length]))
));
entries.then(entries => {
const result = Object.fromEntries(entries);
//...
});
// Polyfill: Object.fromEntries
if(!Object.fromEntries)
Object.defineProperty(Object, "fromEntries", {
value(entries) {
const result = {};
for(const [k, v] of entries)
result[k] = v;
return result;
}
});
Normal loops have issues with looping over async code. You can implement an asyncForEach helper function:
async function asyncForEach(array, callback) {
if (array)
for (let index = 0; index < array.length; index++) {
await callback(array[index], index, array);
}
}
And call it executing your mongoQueries:
await asyncForEach(miles, async mile => {
const response = await Shops.find({ 'shopInfo.address':{ $geoWithin:{ $centerSphere: [ [ 75.83183541365247, 30.902146005639267 ], miles[i] / 3959 ] } } })
// do soming with the response
})
Use async/await to wait mongo promise get resolved and then increment variable. Try this :
async function test() {
let result = {};
let miles = ["5", "10", "15", "25"];
let i = 0;
while (i < miles.length) {
try {
let response = await Shops.find({
"shopInfo.address": {
$geoWithin: {
$centerSphere: [
[75.83183541365247, 30.902146005639267],
miles[i] / 3959
]
}
}
});
if (i == 4) {
result[miles[i]] = response.length;
res.json(result);
} else {
result[miles[i]] = response.length;
i++;
}
} catch (err) {
console.log(err);
}
}
}
Below is the code which worked for me:
let miles = ['5','10','15','25','50'];
async.map(miles,getDistance , function(err, results) {
if(err){
console.log(err);
}
res.json(results);
});
function getDistance(mile, callback) {
Shops.count({ 'shopInfo.address':{ $geoWithin:{ $centerSphere: [ [ 75.83183541365247, 30.902146005639267 ], mile / 3959 ] } } }).then(response=>{
if(response){
callback(null,response);
}
})
}
I Hope It will work for you as well.
One of the biggest issue we face now with parse-server is duplication. Although we have implemented a Parse cloud code to prevent such event through beforeSave and afterSave methods at the same time added external middleware to check for existing object before saving still we face duplication over and over specially on concurrent operations.
Here is our code to prevent duplication for a specific class:
Parse.Cloud.beforeSave("Category", function(request, response) {
var newCategory = request.object;
var name = newCategory.get("name");
var query = new Parse.Query("Category");
query.equalTo("name", name);
query.first({
success: function(results) {
if(results) {
if (!request.object.isNew()) { // allow updates
response.success();
} else {
response.error({errorCode:400,errorMsg:"Category already exist"});
}
} else {
response.success();
}
},
error: function(error) {
response.success();
}
});
});
Parse.Cloud.afterSave("Category", function(request) {
var query = new Parse.Query("Category");
query.equalTo("name", request.object.get("name"));
query.ascending("createdAt");
query.find({
success:function(results) {
if (results && results.length > 1) {
for(var i = (results.length - 1); i > 0 ; i--) {
results[i].destroy();
}
}
else {
// No duplicates
}
},
error:function(error) {
}
});
});
This code above is able to prevent some duplicate but most still goes through, example:
What is the "ultimate way" to prevent duplication with Parse server?
You can always create a unique index in mongodb for the field that should be unique in your document.
This way any save that conflicts with that index, will be aborted
Maybe you should write something with Promises like :
Parse.Cloud.beforeSave("Category", function (request, response) {
return new Promise((resolve, reject) => {
var query = new Parse.Query("Category");
query.equalTo("name", "Dummy");
return query.first().then(function (results) {
resolve(); // or reject()
});
})
});
Parse.Cloud.beforeSave("Category", async (request) => {
(...)
await results = query.first();
// then your logic here
response.success();
response.error({ errorCode: 400, errorMsg: "Category already exist" })
})
Here is my Solution:
Parse.Cloud.beforeSave( 'ClassName', async ( request ) => {
const columnName = 'columnName'
const className = 'ClassName'
if( request.object.isNew() ) {
var newCategory = request.object
var name = newCategory.get( columnName )
var query = new Parse.Query( className )
query.equalTo( columnName, name )
const results = await query.count()
if( results === 0 ) {
// no response.success needed
// https://github.com/parse-community/parse-server/blob/alpha/3.0.0.md
} else {
throw 'Is not unique';
}
}
} )