How to cache a mongoose query in memory? - node.js

I have the following queries, which starts with the GetById method firing up, once that fires up and extracts data from another document, it saves into the race document.
I want to be able to cache the data after I save it for ten minutes. I have taken a look at cacheman library and not sure if it is the right tool for the job. what would be the best way to approach this ?
getById: function(opts,callback) {
var id = opts.action;
var raceData = { };
var self = this;
this.getService().findById(id,function(err,resp) {
if(err)
callback(null);
else {
raceData = resp;
self.getService().getPositions(id, function(err,positions) {
self.savePositions(positions,raceData,callback);
});
}
});
},
savePositions: function(positions,raceData,callback) {
var race = [];
_.each(positions,function(item) {
_.each(item.position,function(el) {
race.push(el);
});
});
raceData.positions = race;
this.getService().modelClass.update({'_id' : raceData._id },{ 'positions' : raceData.positions },callback(raceData));
}

I have recently coded and published a module called Monc. You could find the source code over here. You could find several useful methods to store, delete and retrieve data stored into the memory.
You may use it to cache Mongoose queries using simple nesting as
test.find({}).lean().cache().exec(function(err, docs) {
//docs are fetched into the cache.
});
Otherwise you may need to take a look at the core of Mongoose and override the prototype in order to provide a way to use cacheman as you original suggested.
Create a node module and force it to extend Mongoose as:
monc.hellocache(mongoose, {});
Inside your module you should extend the Mongoose.Query.prototype
exports.hellocache = module.exports.hellocache = function(mongoose, options, Aggregate) {
//require cacheman
var CachemanMemory = require('cacheman-memory');
var cache = new CachemanMemory();
var m = mongoose;
m.execAlter = function(caller, args) {
//do your stuff here
}
m.Query.prototype.exec = function(arg1, arg2) {
return m.execAlter.call(this, 'exec', arguments);
};
})
Take a look at Monc's source code as it may be a good reference on how you may extend and chain Mongoose methods

I will explain with npm redis package which stores key/value pairs in the cache server. keys are queries and redis stores only strings.
we have to make sure that keys are unique and consistent. So key value should store query and also name of the model that you are applying the query.
when you query, inside the mongoose library, there is
function Query(conditions, options, model, collection) {} //constructor function
responsible for query. inside this constructor,
Query.prototype.exec = function exec(op, callback) {}
this function is responsible executing the queries. so we have to manipulate this function and have it execute those tasks:
first check if we have any cached data related to the query
if yes respond to request right away and return
if no we need to respond to request and update our cache and then respond
const redis = require("client");
const redisUrl = "redis://127.0.0.1:6379";
const client = redis.createClient(redisUrl);
const util = require("util");
//client.get does not return promise
client.get = util.promisify(client.get);
const exec = mongoose.Query.prototype.exec;
//mongoose code is written using classical prototype inheritance for setting up objects and classes inside the library.
mongoose.Query.prototype.exec = async function() {
//crate a unique and consistent key
const key = JSON.stringify(
Object.assign({}, this.getQuery(), {
collection: this.mongooseCollection.name
})
);
//see if we have value for key in redis
const cachedValue = await redis.get(key);
//if we do return that as a mongoose model.
//the exec function expects us to return mongoose documents
if (cachedValue) {
const doc = JSON.parse(cacheValue);
return Array.isArray(doc)
? doc.map(d => new this.model(d))
: new this.model(doc);
}
const result = await exec.apply(this, arguments); //now exec function's original task.
client.set(key, JSON.stringify(result),"EX",6000);//it is saved to cache server make sure capital letters EX and time as seconds
};
if we store values as array of objects we need to make sure that each object is individullay converted to mongoose document.
this.model is a method inside the Query constructor and converts object to a mongoose document.
note that if you are storing nested values instead of client.get and client.set, use client.hset and client.hget
Now we monkey patched
Query.prototype.exec
so you do not need to export this function. wherever you have a query operation inside your code, mongoose will execute above code

Related

what is the argument "arguments" passed to exec function in mongoose?

i've been trying to use cache with redis in my nodejs mongodb application but i didn't find any tutorial on how to do that except few which are using logic that seems to not be explained in mongoose documentation
const exec = mongoose.Query.prototype.exec;
mongoose.Query.prototype.exec = async
function (){
// our caching logic
return await exec.apply(this, arguments);
}
where does arguments come from ? because it seems to be undefined yet it is used
mongoose.Query.prototype.exec = async function(){
const collectionName = this.mongooseCollection.name;
if(this.cacheMe){
// You can't insert json straight to redis needs to be a string
const key = JSON.stringify({...this.getOptions(),
collectionName : collectionName, op : this.op});
const cachedResults = await redis.HGET(collectionName,key);
// this.op is the method which in our case is "find"
if (cachedResults){
// if you found cached results return it;
const result = JSON.parse(cachedResults);
return result;
}
//else
// get results from Database then cache it
const result = await exec.apply(this,arguments);
redis.HSET(collectionName, key, JSON.stringify(result) , "EX",this.cacheTime);
//Blogs - > {op: "find" , ... the original query} -> result we got from database
return result;
}
clearCachedData(collectionName, this.op);
return exec.apply(this,arguments);
}
and what is this.getOptions()?
i would be thankful if any one can explain me this logic, because i did not find any help in the documentations nor internet blogs and articls
The arguments object is a local variable that is available inside every function in JavaScript and contains the values of the arguments passed to the function.
this.getOptions() is the local method that returns the options to the query.
// A key example for mongoose Redis integration
const key = JSON.stringify({
collectionName: this.mongooseCollection.name,
op: this.op,
options: this.getOptions(),
filter: this.getFilter(),
projection: this.projection(),
populatedPaths: this.getPopulatedPaths(),
});
There are a lot of similar packages on NPM, but I highly recommend standard mongoose and redis ones to get up and running. I assumed your initialization point was similar to this post. This can also be a relevant source.

generate hashed password on findOneAndUpdate [duplicate]

I'm trying to update counts on a pre hook. The issue is that for some unknown reason the findOneAndUpdate hook doesn't have access to the document, as far as I can tell.
I would like to do this:
source.pre('findOneAndUpdate', function (next) {
console.log('------------->>>>>> findOneAndUpdate: ');
this.objects = this.objects || [];
this.people = this.people || [];
this.events = this.events || [];
this.objectCount = this.objects.length;
this.peopleCount = this.people.length;
this.eventCount = this.events.length;
next();
});
But for some reason the this in the hook isn't the document, its a Query object which seems about useless.
What am I missing? How do I use a pre hook to update counts on a findOneAndUpdate?
You can do smthng like that ->
source.pre('findOneAndUpdate', function (next) {
console.log('------------->>>>>> findOneAndUpdate: ');
this._update.$set.objects = [];
this._update.$set.people = [];
this._update.$set.events = [];
next();
});
pay attention to _update.$set because in the context "this" will be a query. So you can easily add anything you want!
The documentation states:
Query middleware differs from document middleware in a subtle but important way: in document middleware, this refers to the document being updated. In query middleware, mongoose doesn't necessarily have a reference to the document being updated, so this refers to the query object rather than the document being updated.
An update action generally updates a document that only exists in the database (it tells the MongoDB server: "find document X and set property X to value Z"), so the full document isn't available to Mongoose and, hence, you can't update the counts (which requires access to at least the arrays whose length you want to determine).
As an aside: why do you need separate *Count properties in your schema anyway? If you want to query for arrays matching a certain size, you can use the $size operator on the arrays directly.
If you really do need the count properties, then for each update, you need to track the number of changes you made to each of the arrays (in terms of the number of items added/deleted) and use the $inc operator to adjust the counts.
I had a similar issue when I used the updateOne method and was also going to use the updateOne pre hook to make intermittent update before saving to the database. couldn't find a way for it to work. I ended up using the findOneAndUpdate pre hook and doing the updateOne in it.
schema.pre('findOneAndUpdate', async function(next){
const schema = this;
const { newUpdate } = schema.getUpdate();
const queryConditions = schema._condition
if(newUpdate){
//some mutation magic
await schema.updateOne(queryConditions, {newUpdate:"modified data"});
next()
}
next()
})
Another solution is to use the official MongoDB documentation on middleware. They explain why "this" does not refer to the document itself. You may try something in that sense:
source.pre('findOneAndUpdate', async function(next) {
const docToUpdate = await this.model.findOne(this.getFilter());
//modify the appropriate objects
docToUpdate.save(function(err) {
if(!err) {
console.log("Document Updated");
}
});
console.log(docToUpdate);
// The document that `findOneAndUpdate()` will modify
next();
});
This worked for me
SCHEMA.pre('findOneAndUpdate', function(next){
this._update.yourNestedElement
next();
});
schema.pre(['findOneAndUpdate'], async function(next) {
try {
const type = this.get('type')
const query = this.getQuery()
const doc = await this.findOne(query)
if (type) {
this.set('type', doc.type)
}
next()
} catch (e) {
next(new BaseError(e))
}
})
mongoose Documentation:
You cannot access the document being updated in pre('updateOne') or pre('findOneAndUpdate') query middleware. If you need to access the
document that will be updated, you need to execute an explicit query
for the document.
schema.pre('findOneAndUpdate', async function() {
const docToUpdate = await this.model.findOne(this.getQuery());
console.log(docToUpdate); // The document that `findOneAndUpdate()` will modify
});

Non-Blocking MongoDB + NodeJS

I am trying to find a document in a MongoDB collection in a NodeJS environment. Is there any way to do the following?
This is not working:
var foo = function (id) {
// find document
var document = database.find(id);
// do whatever with the document
...
}
This way creates a block :
var foo = function (id) {
// find document
var document = database.find(id);
while (!database.find.done) {
//wait
}
// do whatever with the document
...
}
What I want to do :
var foo = function (id) {
// find document
var document = database.find(id);
// pause out of execution flow
// continue after find is finished
// do whatever with the document
...
}
I know I can use a callback but is there a simpler way of just "pausing" and then "continuing" in NodeJS/JavaScript? Sorry, I am still pretty new to web development.
This is not possible. If you are concerned about the readability of callbacks you may consider using a language that compiles to JavaScript. LiveScript for example has so called “Backcalls”, they make the code appear to be pausing, but compile to a callback function:
For example:
result <- mongodb.find id
console.log result
compiles to:
mongodb.find(id, function(result){
return console.log(result);
});

mongoose model inheritance within javascript prototypal object

Suppose you have a route initialization like this required in your main:
module.exports = function(app) {
for (var name in names) {
var schema = new Schema({}) // schema that accepts anything
, m = mongoose.model(name, schema)
, controller = new TextController(m)
app.get('/path', controller.create.bind(controller))
// etc, etc
And TextController is defined externally as:
var TextController = function(Model) {
this.Model = Model
}
TextController.prototype.create = function(req, res) {
var aDoc = this.Model({ // this is the problematic bit
title: req.body.title
, content: req.body.content})
aDoc.save(function(err) {...})
}
For some reason, mongo saves this as an empty document even though the title and content params are the expected strings. As expected, this.Model is some sort of mongoose object, but it seems to be rejecting the save or the instantiation. Any ideas or suggestions?
Note: I added the controller.method.bind(controller) because it was the only way (I knew of) to get access to this.Model.
Edit: I've also tried the following:
var TextController = function(myCollection) {
this.myCollection = myCollection
this.list = function(req, res) {
this.myCollection.find({}, function { ... })
}
}
And also tried passing in the name and initializing the model within the scope of the function function(name) { this.myCollection = mongoose.model(name) ... }
This turns out to be unrelated to javascript prototypes and completely due to how mongoose does Mixed Type Schemas:
In order to tell mongoose the document has changed you need to markModified(field)
example here: http://mongoosejs.com/docs/schematypes.html#mixed

In node.js, how to use node.js and mongodb to store data in multiple levels

I met a wired problem, that when i use mongodb to store data, some data is missing, which I think it is because of its asynchronous feature
So for this list the timetable, I would use re
/* Here is the a application, in which by using a train_uid and today,
*/
var today = new Date();
var day = today.getDay();
scheduleModel.findByTrainAndTime(train_uid,today,function(err, doc){
var a = new Object();
if(err){}
else{
if(doc != null)
{
//mongodb database can give me some data about the train_id ,uid
a.train_uid = doc.train_uid;
a.train_id = train_id;
and most importantly a train schedule time table, the train schedule time table is a list ( doc.time_schedule )of json objects like arrival, departure and tiploc. However, I need to change the tiploc to sanox number, which referenceModel can help find sanox by providing tiploc number.
//doc.time_schedule
// here is to add a array
so I use async, for each item in the list, I use referenceModel to query sanox and construct a array - a.timeline to store each b, at last when async each operation is finished, trainModel is to store a object with an array of sanox object. However when it comes to the mongodb database, only the array of sanox objects are empty, I guess it is because of asynchronous operation, but since I used async , why it doesn't work
a.train_uid = doc.train_uid; //works
a.train_id = train_id; works
a.timeline = [] // doesn't work
a.timeline = new Array();
var b ;
async.forEachSeries(doc.time_schedule,
function(item,callback){
referenceModel.findStanoxByTicloc(item.tiploc_code,function(err,sanox){
try{
b = new Object();
b.sanox = sanox;
a.time.push(b);
}catch(err2){
}
});
callback();
},
function(err){
trainModel.createNewTrain(a,function(){});
}
}
});
You're calling callback after you fire off the asynchronous find, but before it actually comes back. You need to wait until after you've gotten the data to do that. The following should work better:
async.forEachSeries(doc.time_schedule,
function(item,callback){
referenceModel.findStanoxByTicloc(item.tiploc_code,function(err,sanox){
try{
b = new Object();
b.sanox = sanox;
a.time.push(b);
}catch(err2){
}
callback();
});
},
function(err){
trainModel.createNewTrain(a,function(){});
}

Resources