I am trying to run a PostgreSQL query with Knex and then use the results to run another query.
exports.buildBuoyFeaturesJSON = function (conditionA, conditionB) {
var query = null;
var selectedFields = knex.select
(
knex.raw('t_record.id AS id'),
...
knex.raw('t_record.latitude AS latitude'),
knex.raw('t_record.longitude AS longitude')
)
.from('t_record')
.then(function (response) {
var geometry_array = [];
var rows = response.rows;
var keys = [];
for (var key = 0; key <= rows.length - 1; key++) {
var geometry =
{
"id" : rows[key].id,
"type" : "Feature",
"geometry" : rows[key].geometry,
"properties" : {
...
"sensors" : []
}
};
keys.push(rows[key].id);
geometry_array.push(geometry);
}
getMeasurementsAndSensors(keys, geometry_array);
});
};
The latter function uses some of the results from the previous function. Due to asynchronous nature of Knex, I need to call the second function from inside the first function's .then() statement:
function getMeasurementsAndSensors (keys, geometry_array) {
var query = knex
.select
(
't_record_id',
'i_sensor_id',
'description',
'i_measurement_id',
't_sensor_name',
't_measurement_name',
'value',
'units'
)
.from('i_record')
...
.whereRaw('i_record.t_record_id IN (' + keys + ')')
.orderByRaw('t_record_id, i_sensor_id ASC')
.then(function (response) {
var rows = response.rows;
var t_record_id = 0;
var i_sensor_id = 0;
var record_counter = -1;
var sensor_counter = -1;
for (var records = 0; records <= rows.length -1; records++) {
if (t_record_id !== rows[records].t_record_id) {
t_record_id = rows[records].t_record_id;
record_counter++;
sensor_counter = -1;
}
if (i_sensor_id !== rows[records].i_sensor_id) {
i_sensor_id = rows[records].i_sensor_id;
geometry_array[record_counter].properties.sensors[++sensor_counter] =
{
'i_sensor_id' : rows[records].i_sensor_id,
't_sensor_name' : rows[records].t_sensor_name,
'description' : rows[records].description,
'measurements' : []
};
}
geometry_array[record_counter].properties.sensors[sensor_counter].measurements.push
({
'i_measurement_id': rows[records].i_measurement_id,
'measurement_name': rows[records].t_measurement_name,
'value': rows[records].value,
'units': rows[records].units
});
}
//wrapping features with metadata.
var feature_collection = GEOGRAPHY_METADATA;
feature_collection.features = geometry_array;
JSONToFile(feature_collection, 'buoy_features');
});
}
Currently I save end result to a JSON file because I couldn't get Promises to work. JSON is later used to power a small OpenLayers application, hence the JSON-ification after getting results.
I am quite sure that getting the data from a database, saving it to file, then accessing it from another process and using it for OpenLayers is a very redundant way to do it, but so far, it is the only one that works.
I know there are a lot of ways to make these functions work better, but I am new to promises and don't know how to work with them outside of most basic functions. Any suggestions how to make this code better are welcome.
All you appear to be missing is a bunch of returns.
Here are skeletonized versions of the two functions, including the necessary returns :
exports.buildBuoyFeaturesJSON = function(conditionA, conditionB) {
return knex.select (...)
^^^^^^
.from(...)
.then(function(response) {
// synchronous stuff
// synchronous stuff
return getMeasurementsAndSensors(keys, geometry_array);
^^^^^^
}).then(function(geometry_array) {
var feature_collection = GEOGRAPHY_METADATA;
feature_collection.features = geometry_array;
return feature_collection;
^^^^^^
});
};
function getMeasurementsAndSensors(keys, geometry_array) {
return knex.select(...)
^^^^^^
.from(...)
...
.whereRaw(...)
.orderByRaw(...)
.then(function(response) {
// heaps of synchronous stuff
// heaps of synchronous stuff
// heaps of synchronous stuff
return geometry_array;
^^^^^^^^^^^^^^^^^^^^^
});
}
I moved the feature_collection collection part into buildBuoyFeaturesJSON() on the basis that it seems to sit there more logically. If not, then it would be very simple to move it back into getMeasurementsAndSensors().
I have not tried to fix the additional issue highlighted by #vitaly-t.
Related
I have some dynamic data that needs to have work performed on it. The work must happen sequentially. Using the Q Library, I'd like to create an array of functions and execute the code sequentially using sequences. I can't seem to quite figure out the syntax to achieve this.
const fruits = ["apple", "cherry", "blueberry"]
function makeFruitPie (fruit) {
return Q.Promise((resolve, reject) => {
// Do some stuff here
resolve(fruit+" pie")
// Error handling here
reject(new Error(""))
})
}
const fruitFuncs = new Array(fruits.length)
for(var i = 0; i < fruits.length; i++) {
fruitFuncs[i] = makeFruitPie(fruits[i])
}
// Stole this example from the github docs but can't quite get it right.
i = 0
var result = Q(fruits[i++])
fruitFuncs.forEach((f) => {
result = result(fruits[i++]).then(f)
})
With these lines
for(var i = 0; i < fruits.length; i++) {
fruitFuncs[i] = makeFruitPie(fruits[i])
}
you already run the functions and, hence, their processing will begin.
Assuming you want the execution of the functions in sequence, the following would be more appropriate:
// construct the pipeline
const start = Q.defer();
let result = start.promise; // we need something to set the pipeline off
fruits.forEach( (fruit) => {
result = result.then( () => makeFruitPie( fruit ) );
});
// start the pipeline
start.resolve();
Sidenote: There is a native Promise implementation supported by almost all environments. Maybe consider switching from the library backed version.
You can use Promise.all
Promise.all(fruits.map(fruit=>makeFruitPie(fruit).then(res=> res) )).
then(final_res => console.log(final_res))
final_res will give you array of results
you could use for..of and do things sequentially. something like this
const Q = require("q");
const fruits = ["apple", "cherry", "blueberry"];
function makeFruitPie(fruit) {
return Q.Promise((resolve, reject) => {
// Do some stuff here
resolve(`${fruit} pie`);
// Error handling here
reject(new Error(""));
});
}
for (const fruit of fruits) {
const result = await makeFruitPie(fruit);
console.log(result);
}
By the way also worth considering native Promise insteead of using q
Or quite possibly I am doing it wrong, in fact, more than likely I am doing it wrong.
Have a table which contains a "tree" of skill, starting at the root level and may be as deep as ten levels (only two so far), but I want to return it as one big fat JSON structure, so I want to ask the database for each set of data, build my structure then ask for the next level.
Of course if I just send of my requests using mongoose, they will come back at any time, as they are all nice asyncronous calls. Normally a good things.
Looking at the documentation for Mongoose(using 4.1.1) it seems like it has a promise built in, but whenever I try to use it the api call throws a hissy fit and I get a 500 back.
Here is my simple function:
exports.getSkills = function(req,res) {
console.log("Will return tree of all skills");
for (var i = 0; i<10; i++){
var returnData = [];
console.log("Lets get level " + i );
var query = Skill.find({level: i });//The query function
var promise = query.exec; //The promise?
promise.then(function(doc) { //Totally blows up at this point
console.log("Something came back")
return "OK";
});
}
}
The Mongoose documentation on the subject can be found here
http://mongoosejs.com/docs/api.html#promise_Promise
var promise = query.exec;
// =>
var promise = query.exec()
exports.getSkills = function(req,res) {
console.log("Will return tree of all skills");
var p;
for (var i = 0; i < 10; i ++) {
if (i == 0 ) {
p = Skill.find({level:i}).exec();
} else {
p.then(function (){
return Skill.find({level:i}).exec()
})
}
p.then(function (data) {
//deal with your data
})
}
p.then(function () {
// deal with response
})
}
What I'm trying to do is this, I have 2 users an attacker and a defender. I want the call hit() function until one of the runs out of Health, hit() should be called on turns, first attacker, then defender, then attacker and so on until one reaches 0 hp.
My idea was to do it with a while loop, with current code all I get is the console.log from hit(), an infinite loop. The data from hit() is returned correctly if it's not inside a loop ... I could simply just work in the while loop and not use the hit function but it would mean repeating a lot of code, since there will be a few things to consider before a hit actually happens.
If you have an alternative to the while loop I'm open to ideas, also I should mention I'm new at node so keep it as simple as possible please. Thank you.
This is the relevant part of the code:
var prepareAttack = function(attacker,defender) {
connectdb().done(function(connection) {
query(connection,'SELECT * FROM members WHERE name = ?', attacker).done(function(result) {
var attackerData = result[0][0]
query(connection,'SELECT * FROM members WHERE name = ?', defender).done(function(result) {
var defenderData = result[0][0]
var attackerHp = attackerData.hp
var defenderHp = defenderData.hp
while(attackerHp > 0 && defenderHp > 0) {
hit(attackerData,defenderData).done(function(result){
defenderHp = result;
console.log(defenderHp)
})
hit(defenderData, attackerData).done(function(result) {
attackerHp = result;
console.log(attackerHp)
})
}
})
})
})
}
var hit = function (attackerData, defenderData) { // simplified code just relevant parts inside.
console.log('hitting')
var defer = Q.defer()
var newHp = 0;
defer.resolve(newHp)
return defer.promise
}
Hi I have a backbone web app using Jquery and NodeJs/mongo as the server side framework. I'm having problems with making a http get call with a foreah loop and the results of the get call being iteratively added to each row of the loop.
var eventid = this.model.get("_id");
var inPromise = $.get("/registrants/list?eventid="+eventid,null,null,"json").then(
function (result){
var temp;
var finalVal = '';
var tempfinalVal = "";
var loop = 0
percentage = 0;
$.each(result.registrants,function(index,registrant){
temp = JSON.parse(registrant.fields);
for (var key in temp) {
if(key =="Email"){
if(temp[key] != ""){
$.get("/stats/registrant?userid="+temp[key]+"&eventid="+eventid,null,null,"json").then(function(result2){
percentage = (result2.Stats.type ===undefined || result2.Stats.type ==null) ? "0": result2.Stats.type;
finalVal +=percentage+"\n";
}).fail(function(){
percentage = "0";
});
}
}else if(key =="eventid"){
loop++;
finalVal = finalVal.slice(0, - 1);
finalVal +='\n';
}
finalVal +=temp[key] + ',';
}
});
//promises.push(inPromise);
}
).done(function(finalVal){
$("#webcast-download-registrants-tn").attr("href",'data:text/csv;charset=utf-8;filename=registration.csv",'+encodeURIComponent(finalVal));
console.log("DONE");
}).fail(function(){
console.log("fail");
});
// promise.done(function () {
// console.log(" PROMISE DONE");
// });
So I have the loop through a collection and the last item of the docuemnt gets a content froma nother http call and when all is fone it will create a CSV file. The problem is that THE "DONE" text echos firts then the "CALL" text is displayed
Rick, your problem is not the simplest due to :
the need for nested asynchronous gets
the need to build each CSV data row partly synchronously, partly asynchronously.
the need for a mechanism to handle the fulfilment of multiple promises generated in the inner loop.
From what you've tried, I guess you already know that much.
One important thing to note is that you can't rely on for (var key in temp) to deliver properties in any particular order. Only arrays have order.
You might try something like this :
var url = "/stats/registrant",
data = { 'eventid': this.model.get('_id') },
rowTerminator = "\n",
fieldNames = ['firstname','lastname','email','company','score'];
function getScore(email) {
return $.get(url, $.extend({}, data, {'userid':email}), null, "json").then(function(res) {
return res.Stats ? res.Stats.type || 0 : 0;
}, function() {
//ajax failure - assume score == 0
return $.when(0);
});
}
$.get("/registrants/list", data, null, "json").then(function(result) {
var promises = [];//An array in which to accumulate promises of CSV rows
promises.push($.when(fieldNames)); //promise of CSV header row
if(result.registrants) {
$.each(result.registrants, function(index, registrant) {
if(registrant.fields) {
// Synchronously initialize row with firstname, lastname, email and company
// (omitting score for now).
var row = fieldNames.slice(0,-1).map(function(fieldName, i) {
return registrant.fields[fieldName] || '';
});
//`row` remains available to inner functions due to closure
var promise;
if(registrant.fields.Email) {
// Fetch the registrant's score ...
promise = getScore(registrant.fields.Email).then(function(score) {
//... and asynchronously push the score onto row
row.push(score);
return row;
});
} else {
//or synchronously push zero onto row ...
row.push(0);
//... and create a resolved promise
promise = $.when(row);
}
promises.push(promise);//Accumulate promises of CSV data rows (still in array form), in the correct order.
}
});
}
return $.when.apply(null, promises).then(function() {
//Join all the pieces, in nested arrays, together into one long string.
return [].slice.apply(arguments).map(function(row) {
return row.join(); //default glue is ','
}).join(rowTerminator);
});
}).done(function(str) {
$("#webcast-download-registrants-tn").attr("href",'data:text/csv;charset=utf-8;filename=registration.csv",'+encodeURIComponent(str));
console.log("DONE");
}).fail(function() {
console.log("fail");
});
partially tested
See comments in code for explanation and please ask if there's anything you don't follow.
I'm having an issue when trying to query based on the result of another query on mongodb.
I'm trying to make an initial query and then do another query for each one of the result of the first query. The reason I'm doing it like this is because I have two different collections and I need to join some data from one collection with the data of the other collection. In a SQL world I could easily do this with a JOIN, but as I'm using mongodb in this one I can't really use JOINs, so I guessed doing a for loop inside the first query's callback function would be the way to go.
Here's the code I'm using...
var resultSet = [];
db.get('busstopcollection').find({id_bus_stop: parseInt(req.body.busstopid)}, function(e, docs){
if(e || docs.length === 0) {
console.log("Sorry, wrong id.");
return e;
}
for(var m=0; m<docs.length; m++){
var auxRes = {};
auxRes.id_bus = docs[m].id_bus;
auxRes.id_bus_stop = docs[m].id_bus_stop;
auxRes.coord_x = docs[m].coord_x;
auxRes.coord_y = docs[m].coord_y;
auxRes.id_bus_variation = docs[m].id_bus_variation;
db.get('buscollection').find({id_bus: parseInt(docs[m].id_bus)}, function(e, busDocs){
auxRes.s_origin_description = busDocs[0].s_origin_description;
auxRes.s_destination_description = busDocs[0].id_destination_description;
resultSet.push(auxRes);
});
res.send(JSON.stringify(resultSet));
}
});
I need to res.send the resultSet array after all the values have been added.
I've tried some other ways of doing this, but the thing is that when the res.send line is reached the second query hasn't finished at all. I also tried doing that inside the inner query's callback, but I need to check if it's the last in the for loop, and checking the value o m won't do it as it always is equivalent to docs.length.
As far as I know there's no such thing as a synchronous query in mongodb, but maybe I'm wrong.
What's the right way of doing this?
EDIT
I found a way around it, but I'm sure there's got to be a better way. Here's how I'm doing it...
db.get('busstopcollection').find({id_bus_stop: parseInt(req.body.busstopid)}, function(e, docs){
if(e || docs.length === 0) {
console.log("Ha ocurrido un error, no existe esa parada");
return e;
}
var busIDs = [];
for(var m=0; m<docs.length; m++){
busIDs.push(parseInt(docs[m].id_bus));
var auxRes = {};
auxRes.id_bus = docs[m].id_bus;
auxRes.id_bus_stop = docs[m].id_bus_stop;
auxRes.coord_x = docs[m].coord_x;
auxRes.coord_y = docs[m].coord_y;
auxRes.id_bus_variation = docs[m].id_bus_variation;
resultSet.push(auxRes);
}
db.get('buscollection').find({id_bus: {$in: busIDs}}, function(e, busDocs){
for(var n = 0; n<busDocs.length; n++){
for(var k=0; k<resultSet.length; k++){
if(resultSet[k].id_bus == busDocs[n].id_bus){
resultSet[k].s_origin_description = busDocs[n].s_origin_description;
resultSet[k].s_destination_description = busDocs[n].id_destination_description;
}
}
}
res.send(JSON.stringify(resultSet));
});
});
Node.js behavior is asynchronous , programmer has to code taking consideration of this behavior. Use callbacks or promises or a flow control library . In your your program , you
have put mongo query inside loop , which is a bad approach of querying . Instead if querying multiple times , use $in operator . It will optimize your code performance and
solves your response sending problem also.
var resultSet = [];
db.get('busstopcollection').find({id_bus_stop: parseInt(req.body.busstopid)}, function(e, docs){
if(e || docs.length === 0) {
console.log("Sorry, wrong id.");
return e;
}
var bus_ids = [];
for(var m=0; m<docs.length; m++){
var auxRes = {};
auxRes.id_bus = docs[m].id_bus;
bus_ids.push(parseInt(docs[m].id_bus)); // collect all ids
auxRes.id_bus_stop = docs[m].id_bus_stop;
auxRes.coord_x = docs[m].coord_x;
auxRes.coord_y = docs[m].coord_y;
auxRes.id_bus_variation = docs[m].id_bus_variation;
resultSet.push(auxRes);
}
// Query at one time for all document
db.get('buscollection').find({id_bus: {$in : bus_ids}}).toArray( function(e, busDocs){
// Now find and merge in one go
busDocs.forEach(function(eachBusDoc){
for(var i=0,len = resultSet.length;i< len;i++){
if(resultSet[i].id_bus == busDocs.id_bus ){
resultSet[i].s_origin_description = eachBusDoc.s_origin_description;
resultSet[i].s_destination_description = eachBusDoc.id_destination_description;
}
}
});
res.send(JSON.stringify(resultSet));
});
});
Your updated solution in your question is generally fine, as using $in is an excellent way of fetching a set of results (you'll want to make sure that you've indexed the id_bus property).
Here are a few tweaks (with a bit of cleanup and optimization):
db.get('busstopcollection')
.find({id_bus_stop: parseInt(req.body.busstopid)}).toArray(function(e, docs){
var auxById = {}; // store a dictionary of all the results for later
if(e || docs === null || docs.length === 0) {
console.log("Ha ocurrido un error, no existe esa parada");
return e;
}
var busIDs = [];
docs.forEach(function(doc) {
busIDs.push(parseInt(doc.id_bus));
// consider just using the doc directly rather than copying each property
// especially if you're not manipulating any of the data as it passes
var auxRes = {
id_bus : doc.id_bus,
id_bus_stop : doc.id_bus_stop,
coord_x : doc.coord_x,
coord_y : doc.coord_y,
id_bus_variation : doc.id_bus_variation
};
// you could just use each doc directly ...
// var auxRes = doc; ??
// ** importantly, store off the id_bus for each one so you can
// ** avoid a costly loop trying to match an id below.
auxById[doc.id_bus] = auxRes;
resultSet.push(auxRes);
});
// might want to consider using a cursor ... here's an example
db.get('buscollection')
.find({id_bus: {$in: busIDs}}).each(function(e, busDoc){
// the last item in the cursor will be null
if (busDoc === null) {
res.send(JSON.stringify(resultSet));
return;
}
var res = auxById[busDoc.id_bus];
if (res) { // did we find it in our dictionary of results?
// yes, we did == copy the necessary field data
res.s_origin_description = busDoc.s_origin_description;
res.s_destination_description = busDoc.id_destination_description;
}
});
});