Or quite possibly I am doing it wrong, in fact, more than likely I am doing it wrong.
Have a table which contains a "tree" of skill, starting at the root level and may be as deep as ten levels (only two so far), but I want to return it as one big fat JSON structure, so I want to ask the database for each set of data, build my structure then ask for the next level.
Of course if I just send of my requests using mongoose, they will come back at any time, as they are all nice asyncronous calls. Normally a good things.
Looking at the documentation for Mongoose(using 4.1.1) it seems like it has a promise built in, but whenever I try to use it the api call throws a hissy fit and I get a 500 back.
Here is my simple function:
exports.getSkills = function(req,res) {
console.log("Will return tree of all skills");
for (var i = 0; i<10; i++){
var returnData = [];
console.log("Lets get level " + i );
var query = Skill.find({level: i });//The query function
var promise = query.exec; //The promise?
promise.then(function(doc) { //Totally blows up at this point
console.log("Something came back")
return "OK";
});
}
}
The Mongoose documentation on the subject can be found here
http://mongoosejs.com/docs/api.html#promise_Promise
var promise = query.exec;
// =>
var promise = query.exec()
exports.getSkills = function(req,res) {
console.log("Will return tree of all skills");
var p;
for (var i = 0; i < 10; i ++) {
if (i == 0 ) {
p = Skill.find({level:i}).exec();
} else {
p.then(function (){
return Skill.find({level:i}).exec()
})
}
p.then(function (data) {
//deal with your data
})
}
p.then(function () {
// deal with response
})
}
Related
I have a Node/Express partial that is being called with AJAX, and is supposed to send a status update back to the view, after 2 subsequent API calls are made. This workflow relies on the csv-to-array module to read a ship-orders.csv file, and determine if the second API call (POST to Shipments) has already occured. It is supposed to do this by matching the OrderNumber in the csv file to the returned OrderNumber from the FindOrders endpoint (the first API).
The problem is that I am creating 2 arrays of order numbers to compare, but matching the first set of order numbers to the second set either always returns true or always returns false, and it very clearly should show "true" for the first record in the csv, and "false" for every other.
Before getting into the bulk of the code, here's the promise that reads the csv file into an array:
csv-to-array:
var csvShipPromise = new Promise(function(resolve, reject){
var csvColumns = ['ChannelName', 'OrderNumber', 'LineNumber', 'WarehouseCode', 'Qty', 'Carrier', 'TrackingNumber', 'Shipdate', 'ShipMethod'];
var csvShipArr;
var csvArr;
csvArray({
file: shipLog,
columns: csvColumns
}, function(err, array){
csvShipArr = array;
resolve(csvShipArr);
});
});
Next I have a long promise that gets executed when the request to the partial is made. The comparison between logged OrderNumbers and OrderNumbers that need to be posted to Shipments is the 5th "then" block (and it's commented in the code below).
router.get and chained promise:
router.get('/', function(req, res, next) {
findPromise.then(function(findData){
//Properly format xml string
var foundData = replaceAll(findData, '<', '<');
foundData = replaceAll(foundData, '>', '>');
return foundData;
}).then(function(foundData){
//Parse xml to JSON and stringify
var parsedFound;
parseString(foundData, function(err, result){ //uses an xml to json module
parsedFound = JSON.stringify(result);
});
return(parsedFound);
}).then(function(parsedStr){
//Parse JSON and return an array of objects
var parsedJson = JSON.parse(parsedStr);
var orders = parsedJson['soap:Envelope']['soap:Body'][0]['FindOrders'][0]['orders'][0]['order'];
return orders;
}).then(function(orders){
//Get only orders with a tracking number.
var trackArray = [];
var ord;
for(ord in orders){
var postObj = orders[ord];
if(postObj.TrackingNumber[0].length > 1){
trackArray.push(postObj);
}
}
return trackArray; //array of orders that contain tracking numbers
}).then(function(trackArray){
/**** This is the block that is causing problems. *****/
var tItm;
var loggedOrders = [];
for(tItm in trackArray){
var alreadyLogged = false;
var trackedItm = trackArray[tItm];
var trackedOrderNum = trackedItm.ReferenceNum;
csvShipPromise.then(function(csvOrders){
var csv;
var loggedOrderArr = [];
for (csv in csvOrders){
var csvItm = csvOrders[csv];
var csvOrderNum = csvItm.OrderNumber; //gets the OrderNumber as expected
loggedOrderArr.push(csvOrderNum);
}
return loggedOrderArr; //Return a simple array of all OrderNumbers
}).then(function(loggedOrderArr){
console.log(loggedOrderArr);
console.log(trackedOrderNum);
var ord;
for (ord in loggedOrderArr){
if(trackedOrderNum == loggedOrderArr[ord]){
console.log('found');
alreadyLogged = true;
}
else {
console.log('not found');
alreadyLogged = false;
}
}
return loggedOrderArr; //Simply returning this value because the alreadyLogged test isn't working.
});
/* Here is where the test fails.
It shouldn't, because there are, say, 4 OrderNumbers in the result of the first API call,
and only 1 Order number logged in the CSV.
So it should be true once, and false 3 times.
But it is true all the time.
*/
if(alreadyLogged){
console.log('found'); //Always logs true/found.
} else {
console.log('not found');
}
}
return trackArray; //Just passing the array to the view, for now.
}).then(function(obj){
res.send(obj);
return(obj);
}).catch(function(err){
console.log(err);
});
});
When I console.log the values of trackArray and loggedOrderArr, I see that there should be an intersection between an array of 4 values and an array of 1 value, but for some reason the comparison, if(trackedOrderNumber == loggedOrderArr[ord]) isn't working.
Alright, I'm gonna be honest, your code made my eyes swim. but as far as I can tell, a few things pop up:
move var alreadyLogged = false; to before the loop;
then add alreadyLogged = false; after if(alreadyLogged) statement
I think it has to do with scope. You are basically checking bool value of a var that has not changed yet because your promises has not resolved at the point of if(alreadyLogged)
Might I suggest a different approach?
why not make use of array.indexOf() ?
lets say you have two arrays to compare arrA & arrB; you can see if an item exists like so:
var index = arrA.indexOf(arrB[0]);
if(index == -1){
console.log('No Match');
}
else{
console.log('Match found');
}
no need for any preset flags to see if one array contains an element.
Hope it helps.
A bit more context:
var index = loggedOrderArray.indexOf(trackedOrderNum);
if(index == -1){
console.log('No Match');
// -1 basicaly means that there is not instance of trackedOrderNum in loggedOrderArray
}
else{
console.log('Match found');
}
What you are attempting appears to be reasonably simple. You are just overwhelming yourself with awkward flow control and bulky code.
As it stands, asynchronous flow isn't quite right chiefly due to parseString() not being promisified. A value returned from a raw nodeback won't propagate down a .then chain.
In addition, asynchronous flow will improve with :
application of Promise.all() up front to aggregate the two essential data-delivering promises csvShipPromise and findPromise.
the realisation that wholly synchronous steps in a promise chain can be merged with next step.
And, the bulk of the synchronous code will reduce by employing several Array methods:
Array.prototype.filter()
Array.prototype.map()
Array.prototype.includes()
Boiling it down to somewhere near the bare minimum, I get the following router.get() expression:
router.get('/', function(req, res, next) {
return Promise.all([csvShipPromise, findPromise])
.then([csvOrders, findData] => { // destructuring
let loggedOrderArr = csvOrders.map(order => order.OrderNumber);
let foundData = replaceAll(findData, '<', '<');
foundData = replaceAll(foundData, '>', '>');
return new Promise((resolve, reject) => { // promisify parseString() on the fly
parseString(foundData, (err, result) => {
if(err) reject(err);
else resolve(result['soap:Envelope']['soap:Body'][0].FindOrders[0].orders[0].order); // does this expression really return `orders` (plural)?
});
})
.then(orders => {
let trackArray = orders.filter(postObj => postObj.TrackingNumber[0].length > 1); // filter orders to eliminate those without a tracking number.
let loggedOrders = trackArray.filter(trackedItm => loggedOrderArr.includes(trackedItm.ReferenceNum));
// let unloggedOrders = trackArray.filter(trackedItm => !loggedOrderArr.includes(trackedItm.ReferenceNum));
res.send(loggedOrders); // or res.send(unloggedOrders), depending on what you want.
});
})
.catch(err => {
console.log(err);
res.error(err); // or similar
});
});
untested - I may have made mistakes, though hopefully ones that are simple to correct
I have below snmp call in my code, and am executing this in api call and need to get array of values and return it to the client side. but i am facing issue in executing this lines synchronously, my res.json executing first before getting vales from session.subtree
var shelf = new Array();
function doneCb (error) {
console.log("donecb");
console.log("shelf -----",shelf);
}
function feedCb (varbinds) {
console.log("feed cb");
shelf = [];
for (var i = 0; i < varbinds.length; i++) {
if (snmp.isVarbindError (varbinds[i]))
console.error (snmp.varbindError (varbinds[i]));
else {
var temp = varbinds[i].oid.trim(".");
shelf.push({'id':temp[temp.length-1], 'name':shelfmap[varbinds[i].value.toString()]});
console.log (varbinds[i].oid + "|" + shelfmap[varbinds[i].value.toString()]);
}
}
}
router.get('/getShelves/:r',function(req,res){
shelf = [];
session.subtree (oid, maxRepetitions, feedCb, doneCb);
res.json(shelf);
});
since feedcb and donecb are internal methods of net-snmp module i am not able to rewrite the functions to return the values, I tried with sync and async waterfall model, but it is not working as excepted, kinldy someone suggest some way to execute the method synchronously in node JS. Thanks in advance.
I am trying to run a PostgreSQL query with Knex and then use the results to run another query.
exports.buildBuoyFeaturesJSON = function (conditionA, conditionB) {
var query = null;
var selectedFields = knex.select
(
knex.raw('t_record.id AS id'),
...
knex.raw('t_record.latitude AS latitude'),
knex.raw('t_record.longitude AS longitude')
)
.from('t_record')
.then(function (response) {
var geometry_array = [];
var rows = response.rows;
var keys = [];
for (var key = 0; key <= rows.length - 1; key++) {
var geometry =
{
"id" : rows[key].id,
"type" : "Feature",
"geometry" : rows[key].geometry,
"properties" : {
...
"sensors" : []
}
};
keys.push(rows[key].id);
geometry_array.push(geometry);
}
getMeasurementsAndSensors(keys, geometry_array);
});
};
The latter function uses some of the results from the previous function. Due to asynchronous nature of Knex, I need to call the second function from inside the first function's .then() statement:
function getMeasurementsAndSensors (keys, geometry_array) {
var query = knex
.select
(
't_record_id',
'i_sensor_id',
'description',
'i_measurement_id',
't_sensor_name',
't_measurement_name',
'value',
'units'
)
.from('i_record')
...
.whereRaw('i_record.t_record_id IN (' + keys + ')')
.orderByRaw('t_record_id, i_sensor_id ASC')
.then(function (response) {
var rows = response.rows;
var t_record_id = 0;
var i_sensor_id = 0;
var record_counter = -1;
var sensor_counter = -1;
for (var records = 0; records <= rows.length -1; records++) {
if (t_record_id !== rows[records].t_record_id) {
t_record_id = rows[records].t_record_id;
record_counter++;
sensor_counter = -1;
}
if (i_sensor_id !== rows[records].i_sensor_id) {
i_sensor_id = rows[records].i_sensor_id;
geometry_array[record_counter].properties.sensors[++sensor_counter] =
{
'i_sensor_id' : rows[records].i_sensor_id,
't_sensor_name' : rows[records].t_sensor_name,
'description' : rows[records].description,
'measurements' : []
};
}
geometry_array[record_counter].properties.sensors[sensor_counter].measurements.push
({
'i_measurement_id': rows[records].i_measurement_id,
'measurement_name': rows[records].t_measurement_name,
'value': rows[records].value,
'units': rows[records].units
});
}
//wrapping features with metadata.
var feature_collection = GEOGRAPHY_METADATA;
feature_collection.features = geometry_array;
JSONToFile(feature_collection, 'buoy_features');
});
}
Currently I save end result to a JSON file because I couldn't get Promises to work. JSON is later used to power a small OpenLayers application, hence the JSON-ification after getting results.
I am quite sure that getting the data from a database, saving it to file, then accessing it from another process and using it for OpenLayers is a very redundant way to do it, but so far, it is the only one that works.
I know there are a lot of ways to make these functions work better, but I am new to promises and don't know how to work with them outside of most basic functions. Any suggestions how to make this code better are welcome.
All you appear to be missing is a bunch of returns.
Here are skeletonized versions of the two functions, including the necessary returns :
exports.buildBuoyFeaturesJSON = function(conditionA, conditionB) {
return knex.select (...)
^^^^^^
.from(...)
.then(function(response) {
// synchronous stuff
// synchronous stuff
return getMeasurementsAndSensors(keys, geometry_array);
^^^^^^
}).then(function(geometry_array) {
var feature_collection = GEOGRAPHY_METADATA;
feature_collection.features = geometry_array;
return feature_collection;
^^^^^^
});
};
function getMeasurementsAndSensors(keys, geometry_array) {
return knex.select(...)
^^^^^^
.from(...)
...
.whereRaw(...)
.orderByRaw(...)
.then(function(response) {
// heaps of synchronous stuff
// heaps of synchronous stuff
// heaps of synchronous stuff
return geometry_array;
^^^^^^^^^^^^^^^^^^^^^
});
}
I moved the feature_collection collection part into buildBuoyFeaturesJSON() on the basis that it seems to sit there more logically. If not, then it would be very simple to move it back into getMeasurementsAndSensors().
I have not tried to fix the additional issue highlighted by #vitaly-t.
I know there are a lot of good examples over the web and I read a lot of them, but currently I'm stucked with resolving promises with the new functionality of generators in nodejs 0.11.x.
For e.g. I have the following function:
SolrBaseDomain.prototype.promisedQuery = function(query, callback) {
var solrClient = solr.createClient(this.configuration);
var defer = Q.defer();
solrClient.search(query, function(err,obj){
if (!err) {
if (obj.response.numFound > 0) {
defer.resolve(obj.response.docs);
} else {
defer.resolve(null);
}
} else {
defer.reject(err);
}
});
var promise = defer.promise;
return Q.async(function* (){
var result = yield promise;
return result;
});
};
I expected that every call to this method will wait until the promise is fullfilled and the return-statement gives back the result of the promise.
But currently it seems that instead the code inside "Q.async..." will not be executed or the async call arrives after the return statement of the method was executed.
It's strange, in every example I know, this is one of the recommended ways in order to wait for async calls in nodejs but currently it does not work for me.
I've tried a lot of different variations of the above example, but the result is everytime the same, I get not back a valid result.
I have nodejs installed in version 0.11.10 and the --harmony-flag is set, when the code is executede.
Can anyone point me to right direction? I'm wondering if I oversee something ... :)
Thanks for your feedback.
Best regards
Udo
I expected that every call to this method will wait until the promise is fullfilled and the return-statement gives back the result of the promise.
No. Generators will not make functions synchronous, you cannot (and don't want to) block while waiting for a result. When calling a generator function and running sequentially through the async steps that it yields, the result you will get back in the end is still asynchronous - and therefore a promise. Only inside of the generator, your code can use synchronous control flow and yield.
This means that the (then-) callback-based code
SolrBaseDomain.prototype.promisedQuery = function(query) {
var promise = Q.ninvoke(solr.createClient(this.configuration), "search", query);
return promise.then(function(obj) {
if (obj.response.numFound > 0) {
return obj.response.docs;
} else {
return null;
}
});
};
becomes
SolrBaseDomain.prototype.promisedQuery = Q.async(function* (query) {
var promise = Q.ninvoke(solr.createClient(this.configuration), "search", query);
var obj = yield promise;
// ^^^^^
if (obj.response.numFound > 0) {
return obj.response.docs;
} else {
return null;
}
});
Try this
SolrBaseDomain.prototype.promisedQuery = Q.async(function*(query) {
var solrClient = solr.createClient(this.configuration);
var obj = yield Q.ninvoke(solrClient, "search", query);
return obj.response.numFound > 0 ? obj.response.docs : null;
});
This does the same thing for promises as this does for callbacks:
SolrBaseDomain.prototype.query = function (query, callback) {
var solrClient = solr.createClient(this.configuration);
solrClient.search(query, function(err, obj) {
if (err) return callback(err);
callback(null, obj.response.numFound > 0 ? obj.response.docs : null);
});
};
Therefore if the first return a promise that resolves to undefined so will the callback version call the callback with undefined.
according to your suggestions, my code looks now like this:
...
SolrBaseDomain.prototype.query = Q.async(function* (query) {
var solrClient = solr.createClient(this.configuration);
var obj = yield Q.ninvoke(solrClient, "search", query);
return obj.response.numFound > 0 ? obj.response.docs : null;
});
...
I share the above query-function over all data access layers in order to have a central method which is querying the different indexes in an asynchronous way.
For e.g. in the domain data access layer, the code which deals with that function looks like this:
SolrHostDomain.prototype.getByName = Q.async(function* (domain) {
var queryObject = {
"domain": domain
};
var query = this.getQuery("byName", queryObject);
var docs = yield this.query(query);
var domain = null;
if (docs != null && docs.length > 0) {
domain = this.dataMapper.merge(docs[0]);
}
return domain;});
Currently I'm not sure if the generator in the "getByName"-function is necessary, but it seems to work. Dealing with promises is some unclear concept for me, since I'm new to nodejs.
So maybe, if you can help me on that topic and point me in the right direction, this would be helpfull.
The main question for me is, how can I ensure, that a synchronous method can call an asynchronous method and get back not a promise but the final result of this promise.
I've searched a long time, but I could not find a good documentation which describes the use of generator functions or promises in conjunction with synchronous calls. Even examples are focusing only of using the mechanism but not working together with synchronous function.
Best regards and many thanks for your help
Udo
Got it!!!
After a few trial and errors, I think I got it now and I have a working solutions:
Query function:
SolrBaseDomain.prototype.query = Q.async(function* (query) {
var solrClient = solr.createClient(this.configuration);
var obj = yield Q.ninvoke(solrClient, "search", query);
return obj.response.numFound > 0 ? obj.response.docs : null;
});
Calling method:
SolrHostDomain.prototype.getByName = function(domain) {
var queryObject = {
"domain": domain
};
var query = this.getQuery("byName", queryObject);
var docsPromise = this.query(query);
var _self = this;
return docsPromise.then(function(docs) {
var domain = null;
if (docs != null && docs.length > 0) {
domain = _self.dataMapper.merge(docs[0]);
}
return domain;
});
};
The solution was to understand, that the "query"-method still returns a promise instead of the concrete result even if yield is used.
So I have to add every code which is working on the result of the promise within the "then"-functions (or "done" if no other caller up in the calling hierarchy of methods will follow).
After the settlement of the promise, each code which is set within the "then"-functions will be processed.
BR
Udo
Hi I have a backbone web app using Jquery and NodeJs/mongo as the server side framework. I'm having problems with making a http get call with a foreah loop and the results of the get call being iteratively added to each row of the loop.
var eventid = this.model.get("_id");
var inPromise = $.get("/registrants/list?eventid="+eventid,null,null,"json").then(
function (result){
var temp;
var finalVal = '';
var tempfinalVal = "";
var loop = 0
percentage = 0;
$.each(result.registrants,function(index,registrant){
temp = JSON.parse(registrant.fields);
for (var key in temp) {
if(key =="Email"){
if(temp[key] != ""){
$.get("/stats/registrant?userid="+temp[key]+"&eventid="+eventid,null,null,"json").then(function(result2){
percentage = (result2.Stats.type ===undefined || result2.Stats.type ==null) ? "0": result2.Stats.type;
finalVal +=percentage+"\n";
}).fail(function(){
percentage = "0";
});
}
}else if(key =="eventid"){
loop++;
finalVal = finalVal.slice(0, - 1);
finalVal +='\n';
}
finalVal +=temp[key] + ',';
}
});
//promises.push(inPromise);
}
).done(function(finalVal){
$("#webcast-download-registrants-tn").attr("href",'data:text/csv;charset=utf-8;filename=registration.csv",'+encodeURIComponent(finalVal));
console.log("DONE");
}).fail(function(){
console.log("fail");
});
// promise.done(function () {
// console.log(" PROMISE DONE");
// });
So I have the loop through a collection and the last item of the docuemnt gets a content froma nother http call and when all is fone it will create a CSV file. The problem is that THE "DONE" text echos firts then the "CALL" text is displayed
Rick, your problem is not the simplest due to :
the need for nested asynchronous gets
the need to build each CSV data row partly synchronously, partly asynchronously.
the need for a mechanism to handle the fulfilment of multiple promises generated in the inner loop.
From what you've tried, I guess you already know that much.
One important thing to note is that you can't rely on for (var key in temp) to deliver properties in any particular order. Only arrays have order.
You might try something like this :
var url = "/stats/registrant",
data = { 'eventid': this.model.get('_id') },
rowTerminator = "\n",
fieldNames = ['firstname','lastname','email','company','score'];
function getScore(email) {
return $.get(url, $.extend({}, data, {'userid':email}), null, "json").then(function(res) {
return res.Stats ? res.Stats.type || 0 : 0;
}, function() {
//ajax failure - assume score == 0
return $.when(0);
});
}
$.get("/registrants/list", data, null, "json").then(function(result) {
var promises = [];//An array in which to accumulate promises of CSV rows
promises.push($.when(fieldNames)); //promise of CSV header row
if(result.registrants) {
$.each(result.registrants, function(index, registrant) {
if(registrant.fields) {
// Synchronously initialize row with firstname, lastname, email and company
// (omitting score for now).
var row = fieldNames.slice(0,-1).map(function(fieldName, i) {
return registrant.fields[fieldName] || '';
});
//`row` remains available to inner functions due to closure
var promise;
if(registrant.fields.Email) {
// Fetch the registrant's score ...
promise = getScore(registrant.fields.Email).then(function(score) {
//... and asynchronously push the score onto row
row.push(score);
return row;
});
} else {
//or synchronously push zero onto row ...
row.push(0);
//... and create a resolved promise
promise = $.when(row);
}
promises.push(promise);//Accumulate promises of CSV data rows (still in array form), in the correct order.
}
});
}
return $.when.apply(null, promises).then(function() {
//Join all the pieces, in nested arrays, together into one long string.
return [].slice.apply(arguments).map(function(row) {
return row.join(); //default glue is ','
}).join(rowTerminator);
});
}).done(function(str) {
$("#webcast-download-registrants-tn").attr("href",'data:text/csv;charset=utf-8;filename=registration.csv",'+encodeURIComponent(str));
console.log("DONE");
}).fail(function() {
console.log("fail");
});
partially tested
See comments in code for explanation and please ask if there's anything you don't follow.