How do I use promises and loops together without promise.defer? - node.js

I'm fairly new to promises, and I'm having a problem avoiding some of the things I see described as promise anti-patterns (like Q.defer()). I have a loop that is mostly synchronous, but may occasionally need to make an asynchronous call. The code was very simple before I added the asynchronous calls, but the changes I had to make in order to keep it working with asynchronous calls is very messy.
I would like some advice on how to refactor my code. The code is trying to take selected properties of one object and add them to another. A simplified version is as follows:
function messyFunction(user, fieldArray) {
return Promise.fcall(() => {
var deferred = Promise.defer();
if (!fieldArray) {
deferred.resolve(user);
} else {
var temp = {};
var fieldsMapped = 0;
for (var i = 0; i < fieldArray.length; i++) {
var field = fieldArray[i];
if (field === 'specialValue') {
doSomethingAsync().then((result) => {
temp.specialField = result;
fieldsMapped++;
if (fieldsMapped === fieldArray.length) {
deferred.resolve(temp);
}
});
} else {
temp[field] = user[field];
fieldsMapped++;
if (fieldsMapped === fieldArray.length) {
deferred.resolve(temp);
}
}
}
}
return deferred.promise;
});
}

Here is how I would refactor it. I'm not super familiar with q, so I just used native Promises, but the principal should be the same.
To hopefully make things more clear, I've de-generalized your code a bit to turn messyFunction into getUserFields, asynchronously calculating the age.
Instead of using a for loop and using a counter to keep track of how many fields have been collected, I put them in an array and then pass it into Promise.all. Once all of the values for the fields are collected, the then on the Promise.all promise resolves to the new object.
// obviously an age can be calculated synchronously, this is just an example
// of a function that might be asynchronous.
let getAge = (user) => {
return new Promise((resolve, reject) => {
setTimeout(function () {
resolve((new Date()).getFullYear() - user.birthYear);
}, 100);
});
};
function getUserFields(user, fieldArray) {
if (!fieldArray) {
// no field filtering/generating, just return the object as
// a resolved promise.
return Promise.resolve(user);
// Note: If you want to make sure this is copy of the data,
// not just a reference to the original object you could instead
// use Object.assign to return it:
//
// return Promise.resolve(Object.assign({}, user));
} else {
// Each time we grab a field, we are either store it in the array right away
// if it is synchronous, if it is asyncronous, create a thenable representing
// its eventual value and store that.
// This array will hold them so we can later pass them into Promise.all
let fieldData = [];
// loop over all the fields we want to collect
fieldArray.forEach(fieldName => {
// our special 'age' field doesn't exist on the object but can
// be generated using the .birthYear
if (fieldName === 'age') {
// getAge returns a promise, we attach a then to it and store
// that then in fieldData array
let ageThenable = getAge(user).then((result) => {
// returning a value inside of then will resolve this
// then to that value
return [fieldName, result];
});
// add our thenable to the promise array
fieldData.push(ageThenable);
} else {
// if we don't need to do anything asyncronous, just add the field info
// to the array
fieldData.push([fieldName, user[fieldName]]);;
}
});
// Promise.all will wait until all of the thenables to be resolved before
// firing it's then. This means if none of the fields we were looking for
// is asyncronous, it will fire right away. If some of them were
// asyncronous, it will wait until they all return a value before firing.
// We are returning the then, which will transform the collected data into
// an object.
return Promise.all(fieldData).then(fields => {
// fields is an array of 2-element arrays containing the field name
// and field value for each field we are collecting. We will loop over
// this array with reduce and transform them into an object containing
// all of the properties we collected.
let newUserObj = fields.reduce((acc, [key, value]) => {
acc[key] = value;
return acc;
}, {});
// Above I am using destructuring to get the key/value if the browsers
// you are targeting don't support destructuring, you can instead just
// give it a name as an array and then get the key/value from that array:
//
// let newUserObj = fields.reduce((acc, fieldData) => {
// let key = fieldData[0],
// value = fieldData[1];
// acc[key] = value;
// return acc;
// }, {});
// return new object we created to resolve the then we returned
return newUserObj;
});
}
}
let users = [
{
name: 'Tom',
birthYear: 1986
},
{
name: 'Dick',
birthYear: 1976
},
{
name: 'Harry',
birthYear: 1997
}
];
// generate a new user object with an age field
getUserFields(users[0], ['name', 'birthYear', 'age']).then(user => {
console.dir(user);
});
// generate a new user object with just the name
getUserFields(users[1], ['name']).then(user => {
console.dir(user);
});
// just get the user info with no filtering or generated properties
getUserFields(users[2]).then(user => {
console.dir(user);
});

Related

Handling Callbacks in Classes

I'm building classes to find and quickly operate actions on mongodb documents. This is the UserCursor class. (Not talking about MongoDB's cursor)
exports { UserCursor };
class UserCursor {
private __id: object;
constructor(query: { _id?: object, otherId?: number }) {
let { _id, otherId } = query; // Shortens the vars' name
if (!_id && !otherId) return; // Checks if 1 identifier is provided
if (_id) { // If a _id is provided
Users.findOne({ _id }, (err, user) => {
this.__id = user._id;
});
} else if (otherId) { // If a otherId is provided
Users.findOne({ otherId }, (err, user) => {
console.log(1); // Debug, you'll see later
this.__id = user._id;
});
}
}
// Returns this.__id (which should have been initialized in the constructor)
get _id() {
console.log(2)
return this.__id;
}
}
When run, the console returns
2
1
I think you got the problem: the mongo callback in the constructor gets on after _id operates. How could I manage that, since the constructor gets activated each time the class is used?
It's not entirely clear to me, what exactly you want to happen and how you use this class but I assume you want to instantiate it and then be able to get _id instantaneously. If it's not the case, you may still get some useful info from my answer. Feel free to provide more details, I will update it.
So mongodb operations are asynchronous, if you do
const cursor = new UserCursor(...)
console.log(cursor._id)
(I assume you want this), first all operations in this thread will run, including the call to get _id(), then the callback code will. The problem with such asynchronous things is that now to use this _id you will have to make all of your code asynchronous as well.
So you will need to store a Promise that resolves with _id from mongodb and make a method getId that returns this promise like this:
private __id: Promise<object>
constructor(...) {
// ...
if(_id) {
this.__id = new Promise((resolve, reject) => {
Users.findOne({ _id }, (err, user) => {
if(err) return reject(err)
resolve(user._id)
});
})
} else {
// Same here
}
}
public getId() {
return this.__id;
}
Then use it:
const cursor = new UserCursor(...)
cursor.getId().then(_id => {
// Do smth with _id
})
Or
async function doStuff() {
const cursor = new UserCursor()
const _id = await cursor.getId()
}
doStuff()
If you now do it inside some function, you'll also have to make that function async
Also you could leave a getter like you have now, that will return a promise, but I find it less readable than getId():
cursor._id.then(_id => { ... })
const _id = await cursor._id

NodeJS constructing array from asynchronious callbacks before returning

I'm writing a function that's returning and array of values. Some of the values are calculated in a callback. But I don't know how to make the program asynchronious so all of my results are in the array, and not added after they're returned.
let array = []
for (stuff : stuffs) {
if (condition) {
array.add(stuff)
} else {
api.compute(stuff, callback(resp) {
array.add(resp.stuff)
}
}
}
res.json({ "stuff": array })
In this example the array is written to the response before the async calls have finished.
How can I make this work asynchronously?
You have to use one of the approaches:
async library
Promise.all
coroutines/generators
async/await
The most cool yet, I think, is async/await. First we modify your function, so it returns a promise:
const compute = function(stuff) {
return new Promise( (resolve, reject) => {
api.compute(stuff, callback(resp){
resolve(resp.stuff)
});
});
};
Then we modify your route with async handler:
app.get('/', async function(req, res, next) {
const array = [];
for (const stuff of stuffs) {
if (condition) {
array.add(stuff);
} else {
const stuff = await compute(stuff);
array.push(stuff);
}
}
res.json({ stuff: array });
});
Note: You might need to update node version to latest.
UPDATE:
Those who are not awared, how event loop works, execute this snippet, and finish with that:
const sleep = async function(ms) {
console.log(`Sleeping ${ms}ms`);
return new Promise( resolve => setTimeout(resolve, ms));
};
async function job() {
console.log('start');
for (let t = 0; t < 10; t++) {
await sleep(100);
}
}
job();
console.log('oops did not expect that oO');
You will be surprised.
Here is an answer without package using callbacks
Create a function that's gonna recursively treat all your stuffs.
getArray(stuffs, callback, index = 0, array = []) {
// Did we treat all stuffs?
if (stuffs.length >= index) {
return callback(array);
}
// Treat one stuff
if (condition) {
array.add(stuffs[index]);
// Call next
return getArray(stuffs, callback, index + 1, array);
}
// Get a stuff asynchronously
return api.compute(stuffs[index], (resp) => {
array.add(resp.stuff);
// Call next
return getArray(stuffs, callback, index + 1, array);
});
}
How to call it?
getArray(stuffs, (array) => {
// Here you have your array
// ...
});
EDIT: more explanation
What we want to do to transform the loop you had into a loop that handle asynchronous function call.
The purpose is that one getArray call gonna treat one index of your stuffs array.
After treating one index, the function will call itself again to treat the next index, until all get treated.
-> Treat index 0 -> Treat index 1 -> Treat index 2 -> Return all result
We are using parameters to pass the infos through the process. Index to know which array part we have to treat, and array to keep a tract of what we did calculate.
EDIT: Improvement to 100% asynchronous soluce
What we have done here it's a simple transposition of your initial for loop into an asynchronous code. it can be improved so by making it totally asynchronous, which make it better but slightly more difficult.
For example :
// Where we store the results
const array = [];
const calculationIsDone = (array) => {
// Here our calculation is done
// ---
};
// Function that's gonna aggregate the results coming asynchronously
// When we did gather all results, we call a function
const gatherCalculResult = (newResult) => {
array.push(newResult);
if (array.length === stuffs.length) {
callback(array);
}
};
// Function that makes the calculation for one stuff
const makeCalculation = (oneStuff) => {
if (condition) {
return gatherCalculResult(oneStuff);
}
// Get a stuff asynchronously
return api.compute(oneStuff, (resp) => {
gatherCalculResult(resp.stuff);
});
};
// We trigger all calculation
stuffs.forEach(x => x.makeCalculation(x));

Async Push data to Array in Node

I'm calling mongoose query inside of another mongoose query. When I push results to a array, When ever I check it at last it is empty. After seacrching a lot, Found that issue is this executes asynchronously. But can't find how to fix the issue. My code is as follows.
Bus.find().exec(function(err,buses) {
if(err)
console.log(err);
if (buses[0] != null){
const cords = [];
buses.forEach( function (bus) {
// console.log(bus);
Position.findOne({"busId": bus.busId},{}, {sort : {'time' : -1}}, function (err, position) {
cords.push(position);
console.log(cords);
// console.log(position);
});
console.log(cords);
},
function (err) {
if (err){
console.log(err,"Errrrrrrrrrrrrr");
}
});
console.log(cords);
res.json({cords: cords});
}
Well, there are a number of problems with your code, but chief among them is the fact that you cannot save or act upon values you receive inside a callback to anything outside of that callback. Your example has (rewritten for clarity):
var result = []
arry.forEach(function(opt) {
async.call(args, function(err,value) {
result.push(value)
})
})
// result is empty here!
Which cannot work as you expect because you can not know when the inner callbacks complete.
By definition, callbacks are triggered some time in the future, and since you cannot know when, you must do all computation using the result passed to a callback in the callback itself!
Doing otherwise will give you inconsistent results.
UPDATED - Re comment
(Note: hastily typed on iPad during train ride, will fix later if needed.)
The best way would be to use Promises to aggregate the results. Here's a naive example:
/*
* given a value and an optional array (accum),
* pass the value to the async func and add its result to accum
* if accum is not an array, make it one
* return accum
*/
var do_something = (value, accum) => {
// on first pass, accum will be undefined, so make it an array
accum = Array.isArray(accum) ? accum : []
return new Promise((resolve, reject) => {
async_func(value, (err, res) => {
if(err) {
reject(err)
}
accum.append(res)
resolve(accum)
})
})
}
/*
* for each member of input array, apply do_something
* then operate on accumulated result.
*/
Promise.map(input, do_something)
.then(results => {
// results will contain the accumulated results from all
// the mapped operations
})
.catch(err => {
throw err
})
UPDATED - per comment
Using callbacks only, you can achieve the same result using:
const inputs = [...] // array of inputs to pass to async function
const expected_num_of_results = inputs.length
let results = []
const onComplete = (results) => {
// do something with completed results here
console.log(`results: ${results}`);
}
const onResult = (err, res) => { // callback to async_func
if(err) {
throw new Error(`on iteration ${results.length+1}: ${err}`)
}
results.push(res) // save result to accumulator
if( results.length >= expected_num_of_results) { // are we done?
onComplete(results) // process results
}
}
// example async func - REPLACE with actual async function
const async_func = (val,cb) => {
// call callback with no error and supplied value multiplied by 2
cb(null,val*2)
}
// wrapper that takes one value
// and calls async_func with it and predefined callback
const do_async = (value) => {
async_func(value, onResult)
}
// process inputs
inputs.forEach(do_async)
So with:
const inputs = [1,2,3,4,5]
will print:
results: 2,4,6,8,10

NodeJs async sql to variable

I want a result like this
var rolecheck = ['289773584216358912','281531832938266625'];
Only fetched from a database, so I can compare it to another array with Id's (and yes it's supposed to be a string)
The purpoose of this is to check, before executing a command, if the user has a specific role with permission for that role. So it needs to be a function able to be called.
I've never worked with NodeJs async functions, so i have no clue how to convert this sql to an array:
The content of the .then is just some code of me trying to find out how it works, so ignore the consolelogs etc. Note: the logs do return the correct roles, but i just need them to return them to use them in my compare function.
sql.all("SELECT roleId FROM roles WHERE punish = 'true' and guildId = '"+guildids+"'").then(row => {
if (row) {
var rolecheck = [];
row.forEach(function(row){
rolecheck.push(row.roleId);
});
console.log(rolecheck);
}
});
returning does not work, so I need a workaround.
Here's where i compare it: (this works fine as long as rolecheck and role.id are defined correctly, which they aren't. It does work when i hardcode the rolecheck array.
member.forEach(function(role){
if(HasRole(rolecheck, role.id)){
console.log('user has role: '+role.name);
return true;
}
});
In case you can use a node version with async/await support, like version 7, here is a way to write promise-based code in a synchronous manner. This makes it simpler to pass around the row value.
async function myFunction () {
try {
let member = ''; // whatever member should be
let row = await sql.all("SELECT roleId FROM roles WHERE punish = 'true' and guildId = '"+guildids+"'");
// now you have the row available, outside of 'then' blocks
var rolecheck = [];
if (row) {
row.forEach(function(row){
rolecheck.push(row.roleId);
});
console.log(rolecheck);
}
member.forEach(function(role){
if (HasRole(rolecheck, role.id)) {
console.log('user has role: '+role.name);
return true;
}
});
} catch (error) {
consol.log(error.stack);
}
}
If sql.all did not return a promise you could instead do something like this which would also work with callback based functions
async function myFunction () {
try {
let row = await runSql();
// everything else same as first example above
}
async function runSql () {
try {
return new Promise(function (resolve, reject) {
sql.all("SELECT roleId FROM roles WHERE punish = 'true' and guildId = '"+guildids+"'")
.then(row => {
if (row) {
var rolecheck = [];
row.forEach(function(row){
rolecheck.push(row.roleId);
});
console.log(rolecheck);
resolve(row);
} else {
reject('Row not found');
}
});
});
} catch (error) {
console.log('erro')
}
};

Using a generator to call an API multiple times and only resolve when all requests are finished?

I'm making a simple NodeJS app and I'm refactoring it out of my callback hell.
I've realised generators could be used but I'm struggling to grasp exactly how to use them.
Here's the basic flow of my function (I'm using the request-promise module):
// Iterate through keys to get values for
Object.keys(sourceData).forEach(function(key){
makeRequest(key);
})
makeRequest is a function that basically does this (it's incomplete):
// Make Request
function makeRequest(key) {
rp(apiEndpoint)
.then((data) => {
staticDictionary[key] = data.value;
})
}
I want to synchronously make a call to the endpoint, wait until it's finished getting the data, then move on to the next key in the loop using generators.
Can someone help?
You can sequence your requests one after the other, using .reduce() and promises.
// Make Request
function makeRequest(key) {
// use the key to construct the apiEndpoint here
return rp(apiEndpoint).then((data) => {
return data.value;
});
}
Object.keys(sourceData).reduce(function(p, key){
p = p.then(function(dictionary) {
return makeRequest(key).then(function(data) {
dictionary[key] = data;
return dictionary;
});
}
}, Promise.resolve({})).then(function(dictionary) {
// dictonary contains all your keys here
});
But, since none of your requests depend upon the prior requests, you could also run them all in parallel:
// Make Request
function makeRequest(key) {
// use the key to construct the apiEndpoint here
return rp(apiEndpoint).then((data) => {
return data.value;
});
}
// Iterate through keys to get values for
var staticDictionary = {};
Promise.all(Object.keys(sourceData).map(function(key){
return makeRequest(key).then(function(data) {
staticDictionary[key] = data;
});
})).then(function() {
// staticDictionary is fully available here
});
Using the Bluebird promise library, you could use Promise.props() which takes an object with promises as values and returns an object with the same keys, but resolved values as values (you pass in a map object and get back a map object):
// Make Request
function makeRequest(key) {
// use the key to construct the apiEndpoint here
return rp(apiEndpoint).then((data) => {
return data.value;
});
}
var promiseMap = {};
Object.keys(sourceData).forEach(function(key) {
promiseMap[key] = makeRequest(key);
})
return Promise.props(promiseMap).then(function(dictionary) {
// dictionary is available here
});
You can use Array.reduce function to do so.
Object.keys(sourceData).reduce(function(results, key){
makeRequest(key)
.then(function(data) {
results[key] = data;
});
}, {});
You can take a look at Bluebird, it has a lot of features related to what you are trying to acomplish.
Checkout co
co(function*(){
// Iterate through keys to get values for
for(const key in Object.keys(sourceData)){
// yield the promise
yield makeRequest(key);
}
});
// Make Request
function makeRequest(key) {
// return the promise
return rp(apiEndpoint)
.then((data) => {
staticDictionary[key] = data.value;
});
}
A better way is figuring out how does the ES6 generator work and probably do the things by yourself. Although co is a good library. :)

Resources