How to optimize query for super user? - node.js

I am doing the following query in pseudo code:
Give me all users in age range (provided by user input)
User's who have been seen the least recent first (10 at a time)
User's interest must include one of my interests
User's who have not been swiped left or right by me
If I swiped through 1,000 users and then someone swiped all the same 1,000 then a new user is added making it 1,001. Since all the other 1,000 have been least recent since each new user gets the most recent being the date they create their account, then it has to go through the 1,000 before getting to 1,001 leading to a 3-4 minute query call which is not ok.
My current stack includes Javascript for my web, Backend uses NodeJs with Mongodb.
The three queries look like this:
var minAge = (req.body.min);
var maxAge = (req.body.max);
var swipersCount = 0;
verifyLoginFilter(req, function(loggedIn, userObj, uid) {
if (loggedIn && !userObj.accountFrozen) {
//doing -1 on min and +1 on max for inclusive
var minAgeMinusOne = minAge - 1;
if (minAge === 0) {
minAgeMinusOne = 0;
}
var maxAgePlusOne = maxAge + 1;
var theSwipers, swipersInterests;
var x = 0;
var y = 0;
var skipCount = 0;
grabSwiperLastSwiped();
function grabSwiperLastSwiped() {
User.find({
"$and": [
{ accountFrozen: false },
{ creditsAvailable: { $gt: minAgeMinusOne } },
{ userId: { $ne: uid } }, {
"$or": [
{ "minAge": { "$lt": maxAgePlusOne, "$gt": minAgeMinusOne } },
{ "maxAge": { "$lt": maxAgePlusOne, "$gt": minAgeMinusOne } }
]
}
]
})
.limit(10).sort({ dateLastSwiped: 1 }).skip(skipCount).select({ userId: 1, activeInterests: 1, creditsAvailable: 1, dateLastSwiped: 1 })
.exec(function(err, swipers) {
if (err) {
return res.json({ statusCode: alertErrors.mongodb });
} else {
if (swipers.length > 0) {
theSwipers = swipers;
grabSwiperInterests();
} else {
return res.json({ statusCode: alertErrors.noSwipers });
}
}
});
}
function grabSwiperInterests() {
var today = new Date();
var finalMax;
if (theSwipers[x].creditsAvailable > maxAgePlusOne) {
finalMax = maxAgePlusOne;
} else {
finalMax = theSwipers[x].creditsAvailable;
}
Interests.
find({
"$and": [
{ userId: theSwipers[x].userId },
{ paused: false },
{ _id: { $nin: userObj.personSwiped } }
]
}).limit(10)
.sort({ dailyCredits: 1 })
.exec(function(err, interests) {
if (err) {
return res.json({ statusCode: alertErrors.mongodb });
} else {
if (interests.length > 0) {
swipersInterests = interests;
checkInterst();
} else {
nextSwiper();
}
}
});
}
function nextSwiper() {
swiperCount++;
console.log("nextSwiper " + theSwipers[x].userId + " number " + swiperCount);
if (x === theSwipers.length - 1) {
x = 0;
skipCount = skipCount + theSwipers.length;
grabSwiperLastSwiped();
} else {
console.log("nextSwiper " + theSwipers.length + " x " + x);
x++;
grabSwiperInterests();
}
}
function nextInterest() {
console.log("nextInterest");
if (y === swipersInterests.length - 1) {
nextSwiper();
} else {
y++;
checkInterst();
}
}

Related

Loading jquery in chome extension service worker manifest v3

I am having a extension where it can notify about new items added to RSS feed reader
All works in v2, but v3 I am unable to load jquery into service worker, since chrome doesnt allow.
As a workaround I have added as module
"background": {
"service_worker": "js/background.js","type": "module"
},
But still its a issue and says ReferenceError: $ is not defined
at Object.parseFeed
Or is there a way I can tweak my code to read xml without jquery?
import $ as module from 'js/jquery-2.1.4.min.js';
var Storage = (function() {
return {
getFeeds: function(callback, returnArray, sortArray) {
returnArray = typeof returnArray !== 'undefined' ? returnArray : true;
sortArray = typeof sortArray !== 'undefined' ? sortArray : true;
chrome.storage.sync.get(function(dataObject) {
var result = dataObject;
if (returnArray) {
var feedArray = this.parseDataObjectIntoArray(dataObject);
result = sortArray ? this.sortArrayOfObjects(feedArray, 'position') : feedArray;
} else {
delete result['RssR:Settings'];
}
callback(result)
}.bind(this));
},
getFeedByUrl: function(feedUrl, callback) {
chrome.storage.sync.get(feedUrl, function(feedData) {
callback(feedData[feedUrl]);
});
},
removeFeedByUrl: function(feedUrl) {
chrome.storage.sync.remove(feedUrl);
},
saveFeedData: function(feedData) {
var saveFeed = {};
saveFeed[feedData.url] = feedData;
this.setDataObject(saveFeed);
},
parseDataObjectIntoArray: function(object) {
var array = [];
Object.keys(object).forEach(function(objectKey) {
if (objectKey.indexOf('RssR:Settings') !== 0) {
array.push(object[objectKey]);
}
});
return array;
},
sortArrayOfObjects: function(array, sortKey) {
array.sort(function(a, b) {
if (typeof a[sortKey] === 'undefined') {
return true;
} else if (typeof b[sortKey] === 'undefined') {
return false;
}
return a[sortKey] - b[sortKey];
});
return array;
},
clearAllData: function() {
chrome.storage.sync.clear();
},
setDataObject: function(dataObject) {
chrome.storage.sync.set(dataObject);
}
}
}());
Array.prototype.max = function() {
return Math.max.apply(null, this);
};
function msToTime(ms) {
let seconds = (ms / 1000).toFixed(1);
let minutes = (ms / (1000 * 60)).toFixed(1);
let hours = (ms / (1000 * 60 * 60)).toFixed(1);
let days = (ms / (1000 * 60 * 60 * 24)).toFixed(1);
if (seconds < 60) return seconds + " Sec";
else if (minutes < 60) return minutes + " Min";
else if (hours < 24) return hours + " Hrs";
else return days + " Days"
}
var FeedService = (function() {
return {
downloadFeed: function(feed) {
if (!feed.url) {
return;
}
console.log(feed.url)
fetch(feed.url)
.then(response => response.text())
.then(xmlString => {
console.log(xmlString)
this.parseFeed(xmlString, feed);
}
)
.then(data => console.log(data))
},
parseFeed: function(rawData, existingFeedData) {
var newFeedData = {};
var $feedEntries = $(rawData).find("entry").length > 0 ? $(rawData).find("entry") : $(rawData).find('item');
var lastUpdate = Date.now()
console.log("Now = " + lastUpdate)
lastUpdate = (existingFeedData && existingFeedData.lastUpdate) || Date.now()
console.log("Last Update = " + lastUpdate)
console.log("existingFeedData = " + JSON.stringify(existingFeedData))
pubDates = [];
$feedEntries.each(function(index, elm) {
pubDates.push(new Date($(elm).find("pubDate").text()).getTime())
if (lastUpdate < new Date($(elm).find("pubDate").text()).getTime()) {
console.log($(elm).find("title").text().substring(0, 20));
chrome.notifications.create(
$(elm).find("link").text(), {
type: "basic",
iconUrl: "../icons/254.png",
title: $(elm).find("title").text(),
message: "(" + msToTime((Date.now() - new Date($(elm).find("pubDate").text()).getTime())) + ") ",
requireInteraction: true
},
function() {}
);
}
}.bind(this));
console.log("Max date = " + Math.max.apply(null, pubDates))
existingFeedData.lastUpdate = Math.max.apply(null, pubDates);
Storage.saveFeedData(existingFeedData);
},
getUrlForFeed: function($feed, feedSettings) {
if (feedSettings.linkType && $feed.find(feedSettings.linkType).length > 0) {
return $feed.find(feedSettings.linkType).text();
} else if ($feed.find('link').length == 1) {
return $feed.find('link').text();
} else {
if ($feed.find('link[rel="shorturl"]').length > 0) {
return $feed.find('link[rel="shorturl"]').attr('href');
} else if ($feed.find('link[rel="alternate"]').length > 0) {
return $feed.find('link[rel="alternate"]').attr('href');
} else if ($feed.find('link[rel="related"]').length > 0) {
return $feed.find('link[rel="related"]').attr('href');
} else {
return $feed.find('link').first();
}
}
},
saveFeed: function(existingFeedData, newFeedData) {
;
},
updateReadStatusOfNewItems: function(oldItems, newItems) {
if (typeof oldItems === 'undefined' || Object.keys(oldItems).length == 0 || Object.keys(newItems).length == 0) {
return newItems;
}
Object.keys(newItems).forEach(function(url) {
if (oldItems[url]) {
newItems[url].unread = oldItems[url].unread;
}
});
return newItems;
},
setNewBadge: function() {
chrome.browserAction.setBadgeText({
text: 'NEW'
});
}
};
}());
chrome.alarms.onAlarm.addListener(function(alarm) {
if (alarm.name == 'updateRssFeeds') {
console.log("Called!")
updateRssFeeds();
}
});
chrome.notifications.onClicked.addListener(function(notificationId) {
chrome.tabs.create({
url: notificationId
});
});
chrome.runtime.onInstalled.addListener(onInstall);
function onInstall() {
chrome.runtime.openOptionsPage();
chrome.alarms.create('updateRssFeeds', {
when: 0,
periodInMinutes: 1
});
}
function updateRssFeeds() {
Storage.getFeeds(function(feeds) {
feeds.forEach(function(feed) {
FeedService.downloadFeed(feed);
});
}, true, false)
}

Is there an alternative for $expr in older versions of MongoDB? [duplicate]

This question already has answers here:
MongoDb query condition on comparing 2 fields
(4 answers)
Closed 4 years ago.
In a Node.js backend for a React.js app, I am using
"mongodb" (as db.version returns): "3.4.10",
"mongoose": "5.3.4"
The issue is I can't use $expr with a mongoDB version < 3.6.
It seems like a lot of effort to upgrade the mongoDB version juste because of this issue (deprecated methods and so on).
So I was wondering if there was a way to do what I am trying to do without using $expr ?
Here is the code :
match.$expr = {
$lt: ["$distance", "$range"] // "calculated_distance < tutor_range"
}
Would you know how ? Here is the full method, if ever you want some more details about it.
exports.search = (req, res) => {
let lat1 = req.body.lat;
let lon1 = req.body.lng;
let page = req.body.page || 1;
let perPage = req.body.perPage || 10;
let radius = req.body.radius || 10000;
let levelsIn = req.body.levels && req.body.levels.length !== 0 ? req.body.levels.map(level => {
return ObjectID(level);
}) : null;
let subjectsIn = req.body.subjects && req.body.subjects.length !== 0 ? req.body.subjects.map(subject => {
return ObjectID(subject);
}) : null;
var options = { page: page, limit: perPage, sortBy: { updatedDate: -1 } }
const isAdmin = req.user ? req.user.role === "admin" || req.user.role === "super-admin" : false;
let match = {}
if (levelsIn) match.levels = { $in: levelsIn };
if (subjectsIn) match.subjects = { $in: subjectsIn }
if (typeof req.body.activated !== "undefined") match.profileActivated = req.body.activated;
if (req.body.from) match.createdAt = { $gte: new Date(req.body.from) };
if (req.body.to) {
if (match.createdAt) match.createdAt.$lte = new Date(req.body.to);
else match.createdAt = { $lte: new Date(req.body.to) };
}
var aggregate = null;
if (!isAdmin) {
match.activated = true
match.profileActivated = true
match.profileOnline = true
}
if (lat1 && lon1) {
match.$expr = {
$lt: ["$distance", "$range"] // "calculated_distance < tutor_range"
}
aggregate = Tutor.aggregate([
{
"$geoNear": {
"near": {
"type": "Point",
"coordinates": [lon1, lat1]
},
"distanceField": "distance", // this calculated distance will be compared in next section
"distanceMultiplier": 0.001,
"spherical": true
}
},
{
$match: match
}
]);
} else {
aggregate = Tutor.aggregate([
{
$match: match
}
]);
}
Tutor
.aggregatePaginate(aggregate, options, function (err, result, pageCount, count) {
if (err) {
return res.status(400).send(err);
}
else {
var opts = [
{ path: 'levels', select: 'name' },
{ path: 'subjects', select: 'name' },
{ path: 'assos', select: 'name' }
];
Tutor
.populate(result, opts)
.then(result2 => {
return res.send({
page: page,
perPage: perPage,
pageCount: pageCount,
documentCount: count,
tutors: result2
});
})
.catch(err => {
return res.status(400).send(err);
});
}
})
};
Thank you very much for your answers and help !
You can use $addFields + $match instead of $expr.
Something like
{"$addFields":{"islt":{"$cond":[{"$lt": ["$distance", "$range"]}, true, false]}}},
{"$match":{"islt":true}}
You can use extra project stage to drop the islt variable if you like.
{"$project":{"islt":0}}

"MongoDB: unknown top level operator: $expr"

In a Node.js backend for a React.js app, I am using
"mongodb" (as db.version returns): "3.4.10",
"mongoose": "5.3.4"
I looked into the issue and I discovered that it maybe was because of the mongoDb version I am using.
So I tried to switch mongoDb to 3.4.X but I still get the same error message :
"MongoError: unknown top level operator: $expr"
because of this line of code:
match.$expr = {
$lt: ["$distance", "$range"] // "calculated_distance < tutor_range"
}
Would you know why ? Here is the full method, if ever you want some more details about it.
exports.search = (req, res) => {
let lat1 = req.body.lat;
let lon1 = req.body.lng;
let page = req.body.page || 1;
let perPage = req.body.perPage || 10;
let radius = req.body.radius || 10000;
let levelsIn = req.body.levels && req.body.levels.length !== 0 ? req.body.levels.map(level => {
return ObjectID(level);
}) : null;
let subjectsIn = req.body.subjects && req.body.subjects.length !== 0 ? req.body.subjects.map(subject => {
return ObjectID(subject);
}) : null;
var options = { page: page, limit: perPage, sortBy: { updatedDate: -1 } }
const isAdmin = req.user ? req.user.role === "admin" || req.user.role === "super-admin" : false;
let match = {}
if (levelsIn) match.levels = { $in: levelsIn };
if (subjectsIn) match.subjects = { $in: subjectsIn }
if (typeof req.body.activated !== "undefined") match.profileActivated = req.body.activated;
if (req.body.from) match.createdAt = { $gte: new Date(req.body.from) };
if (req.body.to) {
if (match.createdAt) match.createdAt.$lte = new Date(req.body.to);
else match.createdAt = { $lte: new Date(req.body.to) };
}
var aggregate = null;
if (!isAdmin) {
match.activated = true
match.profileActivated = true
match.profileOnline = true
}
if (lat1 && lon1) {
match.$expr = {
$lt: ["$distance", "$range"] // "calculated_distance < tutor_range"
}
aggregate = Tutor.aggregate([
{
"$geoNear": {
"near": {
"type": "Point",
"coordinates": [lon1, lat1]
},
"distanceField": "distance", // this calculated distance will be compared in next section
"distanceMultiplier": 0.001,
"spherical": true
}
},
{
$match: match
}
]);
} else {
aggregate = Tutor.aggregate([
{
$match: match
}
]);
}
Tutor
.aggregatePaginate(aggregate, options, function (err, result, pageCount, count) {
if (err) {
return res.status(400).send(err);
}
else {
var opts = [
{ path: 'levels', select: 'name' },
{ path: 'subjects', select: 'name' },
{ path: 'assos', select: 'name' }
];
Tutor
.populate(result, opts)
.then(result2 => {
return res.send({
page: page,
perPage: perPage,
pageCount: pageCount,
documentCount: count,
tutors: result2
});
})
.catch(err => {
return res.status(400).send(err);
});
}
})
};
Thank you very much for your answers and help !

NodeJS Callback not returning completed array outside for loop

I've got an empty object array "storing" and I'm currently pushing objects in a nested for loop. This issue is when I call the callback(null, storing) outside the for loop it returns an empty array. Should I be running a promise call? Not sure where to go from here
var params = {
TableName: USER_TABLE,
KeyConditions: {
orgid: {
ComparisonOperator: 'EQ',
AttributeValueList: [org_uuid]
},
orgkey: {
ComparisonOperator: 'EQ',
AttributeValueList: [secret_key]
}
}
};
// query table for devices tied to user
dynamodb.query(params, function(err, data) {
if (err) {
badRequest();
} else {
// check whether correct org_uuid & org_key has been passed
var checked_org = data.Items[0].org_key;
if (checked_org == secret_key) {
var storing = [];
var value = data.Items[0].read.values;
for (var x = 0; x < value.length; x++) {
var query_params = {
TableName: DEVICE_TABLE,
KeyConditions: {
device_id: {
AttributeValueList: {
S: value[x]
},
ComparisonOperator: 'EQ'
}
}
};
dynamodb.query(query_params, function(err, data) {
if (err) {
console.log(err);
} else {
for (var i = 0; i < data.Count; i++) {
storing.push(data.Items[i].device_id);
}
}
});
}
callback(null, storing)
}
}
});
}

How can get two collection documents and calculate points using express.js?

exports.show = = function(req, res) {
var userdata = [{
"productcode": "9563456789",
"cost": "1000"
}, {
"productcode": "8756348947",
"cost": "5600"
}]
var parameterObject = [];
Parameter.find().exec(function(err, Parameters) {
if (err) {
return handleError(res, err);
}
// i want to push Parameters[0].value to parameterObject
parameterObject.push({
pointvalue: Parameters[0].value
});
});
for (var i = 0; i < userdata.length; i++) {
Product.find({
'productcode': userdata[i].productcode
}).exec(function(err, Products) {
if (err) {
return handleError(res, err);
}
var point = 0;
if (!Products) {
point = 0;
} else if (Products[0].product.point > 0) {
point = Products[0].product.point;
}
if (point > 0) {
// here i am not getting userdata[i].cost
//parameterObject.pointvalue value also not getting
totalprice = userdata[i].cost / parameterObject.pointvalue * point;
}
});
}
};
Here i have written function for calculating totalprice. i have mentioned userdata(this is my req.body).
Expectation :
i need to store Parameters objects in some variable to access where ever i want.
i want to pass userdata object in Product.find() function
how can i calculate this
totalprice= userdata[i].cost/parameterObject.pointvalue) * point);
exports.show = = function(req, res) {
var userdata = [{
"productcode": "9563456789",
"cost": "1000"
}, {
"productcode": "8756348947",
"cost": "5600"
}]
var parameterObject = [];
Parameter.find().exec(function(err, Parameters) {
if (err) {
return handleError(res, err);
}
// i want to push Parameters[0].value to parameterObject
parameterObject.push({
pointvalue: Parameters[0].value
});
return FindProducts(parameterObject, function(data) {
console.log(data);
});
});
function FindProducts(parameterObject, callback) {
for (var i = 0; i < userdata.length; i++) {
var totalprice = 0;
findProduct(i, parameterObject, function(i, price) {
totalprice += price;
if (i <= userdata.length) {
return callback({
"userid": "myuserid",
"total": totalprice
});
}
});
}
}
function findProduct(i, parameterObject, callback) {
Product.find({
'productcode': userdata[i].productcode
}).exec(function(err, Products) {
if (err) {
return handleError(res, err);
}
var point = 0;
if (!Products) {
point = 0;
} else if (Products[0].product.point > 0) {
point = Products[0].product.point;
}
if (point > 0) {
// here you can now get the value of userdata[i].cost
// here you can now get the value of parameterObject
totalprice = userdata[i].cost / parameterObject[0].pointvalue * point;
return callback(i, totalprice);
}
});
}
};
You can use promises when you want to use the result of two functions and later use it for further computation.
In your case, you can execute the two asynchronous functions in parallel. It can look like this.
Promise.all([
asyncFunc1(),
asyncFunc2(),
])
.then(function(result){
// result is an array and has the response of the functions which is
// result[0] and result[1]
···
// you can manipulate the result of the functions here
})
.catch(function(err){
// Receives rejection/error among the Promises
···
});
Here asyncFunc1() will be your first find function
asyncFunc2() will be your second find function.
The result[0] and result[1] will be the results of the functions respectively.
Later you can use the result to do further computations.
Hope this helps.

Resources