In loopback I need to handle Patch in loop.
Whenever I try to patch a property, I may have to update the property in one or all the other records in the table/collection. I've been using before save hook and calling updateInLoop as shown below to update the other records.
module.exports = function(Model) {
Model.observe('before save', async(ctx, next) => {
if(ctx.Data) {
updateInLoop(ctx.Data, Model);
}
});
}
const updateInLoop = async function(data, Model) {
var modelDbData = await Model.find();
for(let entity in modelDbData) {
if(entity.property > threshold){
entity.property = entity.property + 1;
await Model.upsert(model);
}
}
}
This seems to be working fine if loop in updateInloop doesn't run more than once. Whenever the loop need to be run more than once, the code doesn't seem to be working as expected.
I found that the unexpected behavior is due to the before save hook being executed for every update action in loop. How can this be resolved?
This behavior is definitely expected, if you make your increment statement easily recognizable you can prevent it from calling itself.
module.exports = function(Model) {
Model.observe('before save', async(ctx) => {
// if(ctx.Data) {
let keys = ctx.Data ? Object.keys(ctx.Data) : undefined;
// It is an increment if the length of its keys is 1 and that the key is property
let isIncrement = (keys && keys.length === 1 && keys[0] === 'property');
if (ctx.Data && !isIncrement) {
updateInLoop(ctx.Data, Model);
}
});
}
const updateInLoop = async function(data, Model) {
var modelDbData = await Model.find();
for(let entity in modelDbData) {
if(entity.property > threshold){
// entity.property = entity.property + 1;
// await Model.upsert(model);
await Model.updateById(entity.id, {property: entity.property + 1});
}
}
}
Related
This is the code I have written for implementing redis caching in queries:
mongoose.Query.prototype.exec = async function () {
if (!this.__useCache || !client.__connected) {
return exec.apply(this);
}
const key = JSON.stringify(
Object.assign({}, this.getQuery(), {
collection: this.mongooseCollection.name,
})
);
// see if key exists
const cachedValue = await client.HGET(this.__hashKey, key);
// if yes, return it
if (cachedValue) {
const doc = JSON.parse(cachedValue);
const result = Array.isArray(doc)
? doc.map((d) => new this.model(d))
: new this.model(doc);
console.log('result', result);
return result;
}
// if no, store it in redis and return
const result = await exec.apply(this);
if (typeof result === 'object') {
await client.HSET(this.__hashKey, key, JSON.stringify(result));
// cache will expire after 24 hrs by default
client.expire(this.__hashKey, this.__duration || 86400);
}
return result;
};
This works perfectly fine for all queries that doesn't 'populate'. But with populate it gets stuck at await exec.apply(this). Why does it happen and how do I fix it ?
I solved it by saving the reference to original exec() inside the Query object instead of just a local variable and calling that instead of the local variable. Like this :
mongoose.Query.prototype._exec = mongoose.Query.prototype.exec;
mongoose.Query.prototype.exec = async function () {
//redis functionality
.
.
.
return await mongoose.Query.prototype._exec.apply(this, arguments);
}
I am building my first web application with node v12.18.1 and express v4.17.1. Since the start of development, I have the same error on all routes: when i quickly click a link multiple times the server crashes with this error: screenshot of the error. It can be fixed in the front-end by disabling events on user input after a click, but I prefer to know what is wrong with my code.
Route of the index page :
let express = require('express');
let router = express.Router();
let controller_index = require("../controller/controller_index.js")
router.get('/', controller_index.get_index);
router.get('/rubrique/:category', controller_index.get_category);
module.exports = router;
Controller of the index page :
const Query = require("../lib/dbs.js");
const ObjectId = require('mongodb').ObjectId;
const utils = require('../lib/utils');
exports.get_index = async (req, res, next) => {
try {
let user = await Query.findOne("users", "_id", ObjectId(req.user_id));
let notification;
if (user) {
notification = await Query.findSortToArray("notifications", "for_user", ObjectId(user._id));
notification.count = notification.filter((notif => !notif.hasOwnProperty("readedAt"))).length;
}
if (!req.query.q) {
let page = req.query.page > 1 ? (req.query.page * 10) - 10 : req.query.page <= 0 ? undefined : 0;
let current_page = req.query.page ? Number(req.query.page) : 1;
let [countDocuments, docs] = await Promise.all([Query.countAll("articles"), Query.findAll(page)]);
let nb_pages = Math.ceil(countDocuments / 10);
if (!docs.length && current_page !== 1) {
next();
}
else {
docs = await documents_processing(docs, user, req);
res.render("../views/index.pug", { docs: docs, user: user, notification: notification, nb_pages: nb_pages, current_page: current_page })
};
}
else if (req.query.q) {
let page = req.query.page > 1 ? (req.query.page * 10) - 10 : req.query.page <= 0 ? undefined : 0;
let current_page = req.query.page ? Number(req.query.page) : 1;
let [countDocuments, docs] = await Promise.all([Query.countAllBySearch("articles", req.query.q), Query.findAllBySearch(req.query.q, page)]);
let nb_pages = Math.ceil(countDocuments / 10);
if (!docs.length && current_page !== 1) {
next();
}
else {
docs = await documents_processing(docs, user, req);
res.render("../views/index.pug", { docs: docs, search: req.query.q, user: user, notification: notification, nb_pages: nb_pages, current_page: current_page })
};
};
}
catch (err) {
console.error(err);
return next(err);
};
};
Example of static function of the query object :
const connect = require("../index.js")
module.exports = class Query {
static async findOne(collection, field, item) {
const result = await connect.client.db("blog_db").collection(collection).findOne({ [`${field}`]: item })
return result;
};
static async findOneAndUpdateOrInsertOnUser(collection, field, itemToSearch, updateItem) {
const result = await connect.client.db("blog_db").collection(collection).findOneAndUpdate({ [`${field}`]: itemToSearch }, { $set: updateItem }, { upsert: true, returnOriginal: false });
return result;
};
static async findSortToArray(collection, field, item) {
const results = await connect.client.db("blog_db").collection(collection).find({ [`${field}`]: item }).sort({ date: -1 }).toArray()
return results;
};
};
I'm fairly new to programming so any advice is welcome, thank you in advance!
----- EDIT -----
Kind of solution :
I have found people who have talked about this error on node v12 and newer, with a downgrade to v10 the issue was resolved without any clear explanation yet.
The error in the screenshot you've shared shows that the error is "Callback called multiple times", but I don't see anywhere obvious in the code you've shared where this is happening. As you're saying this bug only happens when multiple requests are made rapidly one after the other, it suggests there might be a global variable which is being shared between requests, which is something that should be avoided.
To debug the error you're seeing I would recommend commenting out all of the code in the get_index function and gradually uncommenting it in small chunks until you see the error happen again. You will probably want to do the same with the code that is called by the get_index controller function e.g. documents_processing, as the issue might possibly lie there.
Express only support callback-style, and you're using async function to handle the logic.
Basically, with async function, you call a function without waiting for it to resolve the logics. Hence, it creates too many callbacks in the event loop when that route has a large amount of concurrent coming requests.
function asyncWrapper(fn) {
return (req, res, next) => {
return Promise.resolve(fn(req))
.then((result) => res.send(result))
.catch((err) => next(err))
}
};
router.get('/', asyncWrapper(controller_index.get_index));
I am trying to call one async function from inside a loop run by another async function. These functions call APIs and I am using request-promise using nodeJS.
functions.js file
const rp = require("request-promise");
// function (1)
async email_views: emailId => {
let data = {};
await rp({
url: 'myapiurl',
qs: { accessToken: 'xyz', emailID: emailId },
method: 'GET'
})
.then( body => { data = JSON.parse(body) })
.catch( error => { console.log(error} );
return data;
};
The above JSON looks like this:
...
data:{
records: [
{
...
contactID: 123456,
...
},
{
...
contactID: 456789,
...
}
]
}
...
I am running a loop to get individual record, where I am getting a contactID associated with each of them.
// function#2 (also in functions.js file)
async contact_detail: contactId => {
let data = {};
await rp({
url: 'myapiurl2',
qs: { accessToken: 'xyz', contactID: contactId },
method: 'GET'
})
.then( body => { data = JSON.parse(body) })
.catch( error => { console.log(error} );
return data;
};
The above function takes one contactId as parameter and gets that contact's detail calling another API endpoint.
Both functions work fine when they are called separately. But I am trying to do it inside a loop like this:
...
const result = await email_views(99999); // function#1
const records = result.data.records;
...
let names = "";
for( let i=0; i<records.length; i++) {
...
const cId = records[i].contactID;
const contact = await contact_detail(cId); // function#2
names += contact.data.firstName + " " + contact.data.lastName + " ";
...
}
console.log(names);
...
The problem is I am only getting the first contact back from the above code block, i.e. even I have 20 records from function#1, in the loop when I am calling contact_detail (function#2) for each contactID (cId), I get contact detail once, i.e. for the first cId only. For rest I get nothing!
What is the correct way to achieve this using nodeJs?
UPDATE:
const { App } = require("jovo-framework");
const { Alexa } = require("jovo-platform-alexa");
const { GoogleAssistant } = require("jovo-platform-googleassistant");
const { JovoDebugger } = require("jovo-plugin-debugger");
const { FileDb } = require("jovo-db-filedb");
const custom = require("./functions");
const menuop = require("./menu");
const stateus = require("./stateus");
const alexaSpeeches = require("./default_speech");
const app = new App();
app.use(new Alexa(), new GoogleAssistant(), new JovoDebugger(), new FileDb());
let sp = "";
async EmailViewsByContactIntent() {
try {
const viewEmailId =
this.$session.$data.viewEmailIdSessionKey != null
? this.$session.$data.viewEmailIdSessionKey
: this.$inputs.view_email_Id_Number.value;
let pageIndex =
this.$session.$data.viewEmailPageIndex != null
? this.$session.$data.viewEmailPageIndex
: 1;
const result = await custom.email_views_by_emailId(
viewEmailId,
pageIndex
);
const records = result.data.records;
if (records.length > 0) {
const totalRecords = result.data.paging.totalRecords;
this.$session.$data.viewEmailTotalPages = totalRecords;
sp = `i have found a total of ${totalRecords} following view records. `;
if (totalRecords > 5) {
sp += `i will tell you 5 records at a time. for next 5 records, please say, next. `;
this.$session.$data.viewEmailIdSessionKey = this.$inputs.view_email_Id_Number.value;
this.$session.$data.viewEmailPageIndex++;
}
for (let i = 0; i < records.length; i++) {
const r = records[i];
/* Here I want to pass r.contactID as contactId in the function contact_detail like this: */
const contact = await custom.contact_detail(r.contactID);
const contact_name = contact.data.firstName + " " + contact.data.lastName;
/* The above two lines of code fetch contact_name for the first r.contactID and for the rest I get an empty string only. */
const formatted_date = r.date.split(" ")[0];
sp += `contact ID ${spellOut_speech_builder(
r.contactID
)} had viewed on ${formatted_date} from IP address ${
r.ipAddress
}. name of contact is, ${contact_name}. `;
}
if (totalRecords > 5) {
sp += ` please say, next, for next 5 records. `;
}
} else {
sp = ``;
}
this.ask(sp);
} catch (e) {
this.tell(e);
}
}
I am building an alexa skill using JOVO framework and nodeJS.
UPDATE #2
As a test, I only returned the contactId which I am passing to the contact_detail function and I am getting the correct value back to the above code under my first UPDATE.
async contact_detail: contactId => {
return contactId;
}
It seems even after getting the value right, the function is somehow failing to execute. However, the same contact_detail function works perfectly OK, when I am calling it from another place. Only doesn't not work inside a loop.
What could be the reason?
I must be missing something but don't know what!
You are mixing async await and promises together which is causing you confusion. You typically would use one of the other(as async await effectivly provides syntax sugar so you can avoid dealing with the verbose promise code) in a given location.
Because you mixed the two you are in a weird area where the behavior is harder to nail down.
If you want to use async await your functions should look like
async contact_detail: contactId => {
try {
const body = await rp({
url: 'myapiurl2',
qs: { ... }
});
return JSON.parse(body);
} catch(e) {
console.log(e);
//This will return undefined in exception cases. You may want to catch at a higher level.
}
};
or with promises
async contact_detail: contactId => {
return rp({
url: 'myapiurl2',
qs: { ... }
})
.then( body => JSON.parse(body))
.catch( error => {
console.log(error);
//This will return undefined in exception cases. You probably dont want to catch here.
});
};
Keep in mind your current code executing the function will do each call in series. If you want to do them in parallel you will need to call the function a bit differently and use something like Promise.all to resolve the result.
Here you go:
...
const result = await email_views(99999); // function#1
const records = result.data.records;
...
let names = "";
await Promise.all(records.map(async record => {
let cId = record.contactID;
let contact = await contact_detail(cId);
names += contact.data.firstName + " " + contact.data.lastName + " ";
});
console.log(names);
...
I'm posting this as an answer only because I need to show you some multi-line code as part of throubleshooting this. Not sure this solves your issue yet, but it is a problem.
Your contact_detail() function is not properly returning errors. Instead, it eats the error and resolves with an empty object. That could be what is causing your blank names. It should just return the promise directly and if you want to log the error, then it needs to rethrow. Also, there's no reason for it to be declared async or to use await. You can just return the promise directly. You can also let request-promise parts the JSON response for you too.
Also, I notice, there appears to be a syntax error in your .catch() which could also be part of the problem.
contact_detail: contactId => {
return rp({
url: 'myapiurl2',
qs: { accessToken: 'xyz', contactID: contactId },
json: true,
method: 'GET'
}).catch( error => {
// log error and rethrow so any error propagates
console.log(error);
throw error;
});
};
Then, you would call this like you originally were (note you still use await when calling it because it returns a promise):
...
const result = await email_views(99999); // function#1
const records = result.data.records;
...
let names = "";
for( let i=0; i<records.length; i++) {
...
const cId = records[i].contactID;
const contact = await contact_detail(cId);
names += contact.data.firstName + " " + contact.data.lastName + " ";
...
}
console.log(names);
...
It is 2016, Node has had nearly full ES6 support since v4, and Promises have been around since 0.12. It's time to leave callbacks in the dust IMO.
I'm working on a commander.js-based CLI util which leverages a lot of async operations - http requests and user input. I want to wrap the Commander actions in async functions so that they can be treated as promises, and also to support generators (useful for the co-prompt library I'm using for user input).
I've tried wrapping the CB with co in two ways:
1)
program.command('myCmd')
.action(program => co(function* (program) {...})
.catch(err => console.log(err.stack)) );
and
2) program.command('myCmd').action(co.wrap(function* (program) { .. }));
The problem with 1) is that the program parameter isn't passed
The problem with 2) is that errors are swallowed...
I'd really like to get this working as it yields much nicer code in my use case - involving a lot of http requests and also waiting for user input using the co-prompt library..
Is it a better option altogether perhaps to wrap program.Command.prototype.action somehow?
thanks!
I've used a bespoke version of something like co to get a db.exec function which uses yield to do database request. You can pass parameters into a generator function (I pass in a connection object - see the comment where I do it).
Here is by db.exec function that is very similar to what co does
exec(generator) {
var self = this;
var it;
debug('In db.exec iterator');
return new Promise((accept,reject) => {
debug('In db.exec Promise');
var myConnection;
var onResult = lastPromiseResult => {
debug('In db.exec onResult');
var obj = it.next(lastPromiseResult);
if (!obj.done) {
debug('db.exec Iterator NOT done yet');
obj.value.then(onResult,reject);
} else {
if (myConnection) {
myConnection.release();
debug('db.exec released connection');
}
accept(obj.value);
debug('db.exec Promise Resolved with value %d',obj.value);
}
};
self._connection().then(connection => {
debug('db.exec got a connection');
myConnection = connection;
it = generator(connection); //This passes it into the generator
onResult(); //starts the generator
}).catch(error => {
logger('database', 'Exec Function Error: ' + error.message);
reject(error);
});
});
}
the connection object also wraps by database connection object and provides a generator function ability to process the rows of the results from the database, but I won't post that here (although the example below is using it to process the rows).
Here is an example of using the exec function to run a sequence of sql
db.exec(function*(connection) {
if (params.name === ADMIN_USER) {
debug('Admin Logon');
user.name = ADMIN_DISPLAY;
user.keys = 'A';
user.uid = 0;
let sql = 'SELECT passwordsalt FROM Admin WHERE AdminID = 0';
connection.request(sql);
yield connection.execSql(function*() {
let row = yield;
if (row) {
user.nopass = (row[0].value === null);
} else {
user.nopass = false;
}
debug('Admin Password bypass ' + user.nopass.toString());
});
} else {
debug('Normal User Logon');
let sql = `SELECT u.UserID,PasswordSalt,DisplayName,AccessKey,l.LogID FROM Users u
LEFT JOIN UserLog l ON u.userID = l.userID AND DATEDIFF(D,l.LogDate,GETDATE()) = 0
WHERE u.UserName = #username`;
let request = connection.request(sql);
request.addParameter('username',db.TYPES.NVarChar,params.name);
let count = yield connection.execSql(function*() {
let row = yield;
if (row) {
user.uid = row[0].value;
user.name = row[2].value;
user.keys = (row[3].value === null) ? '' : row[3].value;
user.nopass = (row[1].value === null) ;
user.lid = (row[4].value === null) ? 0 : row[4].value;
debug('Found User with uid = %d and lid = %d, keys = %s',
user.uid, user.lid, user.keys);
}
});
if (count === 0) {
debug('Not Found User');
// couldn't find name in database
reply(false,false);
return;
}
}
if (!user.nopass) {
debug('Need a Password');
//user has a password so we must check it
passGood = false; //assume false as we go into this
let request = connection.request('CheckPassword');
request.addParameter('UserID',db.TYPES.Int,user.uid);
request.addParameter('password',db.TYPES.VarChar,params.password);
yield connection.callProcedure(function*() {
let row = yield;
if (row) {
//got a valid row means we have a valid password
passGood = true;
}
});
} else {
passGood = true;
}
if (!passGood) {
debug('Not a Good Pasword');
reply(false,true);
} else {
if (user.uid !== 0 && user.lid === 0) {
let sql = `INSERT INTO UserLog(UserID,LogDate,TimeOn,UserName) OUTPUT INSERTED.logID
VALUES(#uid,GETDATE(),GETDATE(),#username)`;
let request = connection.request(sql);
request.addParameter('uid',db.TYPES.Int,user.uid);
request.addParameter('username',db.TYPES.NVarChar,user.name);
yield connection.execSql(function*() {
let row = yield;
if (row) {
user.lid = row[0].value;
debug('Users Log Entry = %d',user.lid);
}
});
}
reply(true,user);
}
})
.catch((err) => {
logger('database','Error on logon: ' + err.message);
reply(false,false);
});
});
There is a quite simple way to do async function in Commander.js
async function run() {
/* code goes here */
}
program
.command('gettime')
.action(run);
program.parse(process.argv);
Now I've working on NodeJS and Sequelize to query and process an database data.
I've call findAll from Table1 and I want to query each rows to apply some data to Table2 then I want to add all data to array before send output, I did like this
var last_promise;
var output_results = {};
Table1Model.findAll()
.then(function(results1)
{
for (var i = 0; i < results1.length; ++i)
{
var result1 = results1[i];
output_results[result1.id] = result1;
var add_promise = Table2Model
.create({
id_from_table1: result1.id,
data_from_table1: result1.data
});
.then(function(result2) {
output_results[result2.id_from_table1].data2 = result2;
});
if (last_promise)
{
last_promise.then(function()
{
return add_promise;
});
} else {
last_promise = add_promise;
}
}
}
}
last_promise.then(function() {
return output_results;
}
I want to know that there any better way to execute promises sequentially in a loop like this ?
It looks like you can do that with .all() method:
Table1Model
.findAll()
.then(function(results1) {
return Promise.all(results1.map(function(result) {
return Table2Model
.create({
id_from_table1: result1.id,
data_from_table1: result1.data
})
.then(function(result2) {
...
});
}));
})
.then(function(output_results) {
});