I'm building an app using firebase and Node.js. I need to get data from nested foreach. How to do it correctly? Need to return the results of all iterations simultaneously.
exports.userParty = function (userInfo, cb) {
var userID = userInfo.userID;
var clubID = userInfo.clubID;
var refUserParty = ref.child(userID).child('my_party_id');
var party = {};
refUserParty.orderByValue().once("value", function (snapshot) {
var party = {};
snapshot.forEach(function (partyID) {
var refParty = dbb.ref('clubs').child(clubID).child('party').child(partyID.val());
refParty.once('value', function (partyBody) {
party[partyID.val()] = partyBody.val();
//console.log(party);
});
});
cb(party); // {}
});
};
You need to call the callback after all the async functions in the forEach block have completed. You can do this using a simple counter to check all async functions are complete:
...
let completedSnapshots = 0;
snapshot.forEach(function (partyID) {
var refParty = dbb.ref('clubs').child(clubID).child('party').child(partyID.val());
refParty.once('value', function (partyBody) {
party[partyID.val()] = partyBody.val();
completedSnapshots++;
if (completedSnapshots === snapshot.val().length) {
cb(party);
}
});
});
...
Related
I have a web request which gets some data and returns the response. I am looking to see how to store the response in a variable so it can be used later on in the code.
This is for node js running in a lambda function
/**
* Performs operations for vehicle management actions interfacing primiarly with
* Amazon DynamoDB table.
*
* #class vehicle
*/
/**
* Registers a vehicle to and owner.
* #param {JSON} ticket - authentication ticket
* #param {JSON} vehicle - vehicle object
* #param {createVehicle~callback} cb - The callback that handles the response.
*/
vehicle.prototype.createVehicle = function(ticket, vehicle, cb) {
let vehicle_data = [];
vehicle_data.push(vehicle);
let vin_data = _.pluck(vehicle_data, 'vin');
let vin_number = vin_data[0];
console.log(vin_number);
var options = {
url: 'https://vindecoder.p.mashape.com/decode_vin?' + 'vin=' + vin_number,
headers: {"X-Mashape-Key": "XXXXXXXXXXXXXXXXXXXXXXXXXX","Accept": "application/json"}
};
var data;
function callback(error, response, body) {
if (!error && response.statusCode == 200) {
var result = JSON.parse(body);
var data = result['specification'];
//console.log(data);
}
}
request(options, callback);
var year = data['year'];
var make = data['make'];
var model = data['model'];
var trim_level = data['trim_level'];
var engine = data['engine'];
var body_style = data['style'];
var made_in = data['made_in'];
var steering_type = data['steering_type'];
var anti_brake_system = data['anti_brake_system'];
var fuel_tank = data['tank_size'];
var overall_height = data['overall_height'];
var overall_length = data['overall_length'];
var overall_width = data['overall_width'];
var standard_seating = data['standard_seating'];
var optional_seating = data['optional_seating'];
var highway_mileage = data['highway_mileage'];
var city_mileage = data['city_mileage'];
vehicle.owner_id = ticket['cognito:username'];
// vehicle.vehicle_year = year;
// vehicle.make = make;
// vehicle.model = model;
// vehicle.trim_level = trim_level;
// vehicle.engine = engine;
// vehicle.body_style = style;
// vehicle.made_in = made_in;
// vehicle.steering_type = steering_type;
// vehicle.anti_brake_system = anti_brake_system;
// vehicle.fuel_tank = fuel_tank;
// vehicle.overall_height = overall_height;
// vehicle.overall_length = overall_length;
// vehicle.overall_width = overall_width;
// vehicle.standard_seating = standard_seating;
// vehicle.optional_seating = optional_seating;
// vehicle.highway_mileage = highway_mileage;
// vehicle.city_mileage = city_mileage;
let params = {
TableName: ddbTable,
Item: vehicle
};
let docClient = new AWS.DynamoDB.DocumentClient(dynamoConfig);
docClient.put(params, function(err, data) {
if (err) {
console.log(err);
return cb(err, null);
}
return cb(null, vehicle);
});
};
I expect the response from the api call to be stored in a an object so it can be used to update the dynamodb record
There are various problems in your code. See the comments I left in the code.
Your function is failing because of the asynchronous nature of javascript. Basically, you have a request callback with a result that never gets seen by the rest of your code. Using promises and async/await is one clean way to resolve this. See below:
// use request promise instead of request
// promises will make your life much easier
// https://hackernoon.com/javascript-promises-and-why-async-await-wins-the-battle-4fc9d15d509f
// https://github.com/request/request-promise
const request = require('request-promise')
vehicle.prototype.createVehicle = function(ticket, vehicle, cb) {
// No need to use var anymore with es6, just use let and const
// https://www.sitepoint.com/es6-let-const/
let vehicle_data = [];
vehicle_data.push(vehicle);
let vin_data = _.pluck(vehicle_data, "vin");
let vin_number = vin_data[0];
console.log(vin_number);
const options = {
uri: "https://vindecoder.p.mashape.com/decode_vin?" + "vin=" + vin_number,
headers: {
"X-Mashape-Key": "XXXXXXXXXXXXXXXXXXXXXXXXXX",
Accept: "application/json"
}
};
// Here's the main mistake
// request has a callback function that is asynchronous
// the rest of your code never sees the result because the result doesn't leave the callback
// your code continues to execute without waiting for the result (this is the gist of asynchronity in js)
// function callback(error, response, body) {
// if (!error && response.statusCode == 200) {
// const result = JSON.parse(body);
// const data = result["specification"]; // this variable doesn't leave the scope of this callback
// //console.log(data);
// }
// }
// therefore this is the failure point
// request(options, callback);
// Do this instead
// here I utilize promises and async/ await
// https://medium.com/codebuddies/getting-to-know-asynchronous-javascript-callbacks-promises-and-async-await-17e0673281ee
try {
const result = await request(options);
const data = result["specification"];
} catch (error) {
console.log(error);
}
// Now data is available to be used below in dynamodb
// also, utilize objects, its much cleaner
const carVariables = {
year: data["year"],
make: data["make"],
model: data["model"],
trim_level: data["trim_level"],
engine: data["engine"],
body_style: data["style"],
made_in: data["made_in"],
steering_type: data["steering_type"],
anti_brake_system: data["anti_brake_system"],
fuel_tank: data["tank_size"],
overall_height: data["overall_height"],
overall_length: data["overall_length"],
overall_width: data["overall_width"],
standard_seating: data["standard_seating"],
optional_seating: data["optional_seating"],
highway_mileage: data["highway_mileage"],
city_mileage: data["city_mileage"],
};
vehicle.owner_id = ticket["cognito:username"];
vehicle = { ...vehicle, ...carVariables} // ES6 spread operator does the below code for you:
// one line versus 20. win. // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Spread_syntax
// vehicle.vehicle_year = year;
// vehicle.make = make;
// vehicle.model = model;
// vehicle.trim_level = trim_level;
// vehicle.engine = engine;
// vehicle.body_style = style;
// vehicle.made_in = made_in;
// vehicle.steering_type = steering_type;
// vehicle.anti_brake_system = anti_brake_system;
// vehicle.fuel_tank = fuel_tank;
// vehicle.overall_height = overall_height;
// vehicle.overall_length = overall_length;
// vehicle.overall_width = overall_width;
// vehicle.standard_seating = standard_seating;
// vehicle.optional_seating = optional_seating;
// vehicle.highway_mileage = highway_mileage;
// vehicle.city_mileage = city_mileage;
let params = {
TableName: ddbTable,
Item: vehicle
};
// This will now probably work. yay!
let docClient = new AWS.DynamoDB.DocumentClient(dynamoConfig);
docClient.put(params, function(err, data) {
if (err) {
console.log(err);
return cb(err, null);
}
return cb(null, vehicle);
});
};
This is untested but I believe it should work. And there may some typos or whatever but the main takeaway here is the use of promises and async/await to wait for and expose the result from the request. See the links and references in the comments for further reading.
I just started writing node.js code.
I'm writing a code that extracts data from a pdf file, cleans it up and stores it in a database (using couchdb and accessing that using nano library).
The problem is that the calls are being made asynchronously... so the database get calls (i make some get calls to get a few affiliation files during the clean up) get completed only after the program runs resulting in variables being undefined. is there any way around this?
I've reproduced my code below
const fs = require('fs');
const os = require('os');
var couchDB = require('couch-db').CouchDB;
var pdf_table_extractor = require('pdf-table-extractor');
const filename = "PQ-PRI-0005-1806-01-0000_quoteSlipForLIBVIDGI1.pdf"
var nano = require('nano')('https://couchadmin:difficulttoguessmypassword#dbdev.perilwise.com');
var server = new couchDB('https://db.url.com');
server.auth("admin","admin");
var db = nano.db.use('pwfb');
var temp = [];
//New callView function
async function callView(){
try{
const doc = await view('liabilitymdm','pi');
for (var i =0; i<doc.rows.length;i++){
tmp.push(doc.rows[i]);
};
return doc;
} catch(e){
console.log(e);
};
};
function suc(result){
let ttmp = [];
console.log(result);
var pageTables = result.pageTables;
var firstPageTables = pageTables[0].tables;
ttmp = callView();
//this console log shows Promise { <pending> }
console.log(ttmp)
for (var k = 0; k < firstPageTables.length; k++) {
var temp = firstPageTables[k];
if (temp.length > 0) {
dump.push(temp);
}
}
// console.log(dump);
var insurer = filename.substr(37,8);
read_quote_slip(insurer,dump);
}
var read_quote_slip = (insurer,data) => {
console.log("read_quote_slip correctly entered");
var finOut = {};
if (insurer === "LIBVIDGI"){
finOut.insurer = insurer;
finOut.policyType = data[2][0].replace(/Quotation for/g,"");
finOut.natureOfWork = data[13][3];
let dedpos = indexGetter(data, "Deductible")[0];
finOut.deductible = data[dedpos+1][0];
let cov = indexGetter(data, "Coverage Territory and Jurisdiction")[0];
finOut.coverageTerritory = data[cov+1][0].replace(/Territory/g,"");
finOut.coverageJurisdiction = data[cov+2][0].replace(/Jurisdiction/g,"");
let ext = indexGetter(data,"Extensions")[0];
finOut.coverage = data[ext+1][0].split(/\r?\n/);
let majexc = indexGetter(data,"Major Exclusions")[0];
finOut.exclusions = data[majexc+1][0].split(/\r?\n/);
let prdtl = indexGetter(data,"Description")[0];
let prm = premiumcompute(data,prdtl,dedpos);
finOut.premium = prm;
finCleaned = libvidgi_cleaned(finOut);
// console.log(finCleaned);
}
}
var indexGetter = (words,toFind) => {
var finindex = [];
for (var i = 0; i < words.length; i++){
for (var j = 0; j < words[i].length; j++){
if(words[i][j].indexOf(toFind) >=0 ){
finindex.push(i);
}
}
}
return finindex;
}
var premiumcompute = (data, from, to) => {
let finprem = [];
let numbop = to - from - 2;
let incr = 0;
for (var i = from+2; i < to; i++){
let pr = {};
pr.option = incr+1;
pr.sumInsured = data[i][2].replace(/ /g,"");
pr.premium = data[i][data[i].length - 1].replace(/ /g,"");
finprem.push(pr);
incr +=1;
}
return finprem;
}
var libvidgi_cleaned = (finOut) => {
return finOut;
}
var fal = (result) => {
console.log(result);
console.log("there was an error");
}
var readPDFFile = function(filename){
//Decide which insurer from the filename
// console.log(filename);
console.log(filename.substr(37,8)+"Printed on line 38");
insurer = filename.substr(37,8)
pdf_table_extractor(filename, (result) => {suc(result)} , fal);
}
var libvidgi_data_extract = (data) => {
console.log(data);
let arr = data.pageTables.tables;
for (var i = 0; i <= arr.length; i++ ){
console.log(arr[i]);
}
}
readPDFFile(filename);
This answer assumes you are using Node.js > v7.6
Since db.view accepts a callback, and you wish to wait for it to finish, one solution will be to promisify it - meaning to turn it into a promise which can be awaited. You can use a library like Bluebird or you can even use Node's builtin promisify util. Then you can rewrite callViews:
const {promisify} = require('util');
const view = promisify(db.view);
async function callView() {
try {
const doc = await view('liabilitymdm', 'pi');
// the async operation is now guaranteed to be done
// (if there is an error it will be caught by the catch clause)
for (var i = 0; i < doc.rows.length; i++) {
temp.push(doc.rows[i]);
}
console.log(temp);
} catch (e) {
}
}
If you are not using Node.js > v7.6 (and cannot use async\await you can still utilize promises, by using their then method:
const {promisify} = require('util');
const view = promisify(db.view);
function callView() {
view('liabilitymdm', 'pi')
.then(doc => {
for (var i = 0; i < doc.rows.length; i++) {
temp.push(doc.rows[i]);
}
console.log(temp);
return temp;
})
.then(temp => {
console.log(temp);
})
.catch(e => {});
}
Notice how the first then is returning something which is used in a later then.
To make Node run asynchronously, you can use the keywords async and await.
They work like this:
async function doSomething () {
const formattedData = formatData();
const result = await db.postToDatabase(formattedData);
// the below will not happen until the above line is finished
doSomethingElse(result);
}
It's pretty simple in Node to get functions to execute asynchronously. Just put the async keyword at the beginning of the function definition and then put await in front of anything that you want to block execution until completed.
"use strict";
var Tabletop = require("tabletop");
var base64 = require('base-64');
Tabletop.init( { key: 'xxxxxg46hgfjd',
callback: showInfo,
simpleSheet: true } )
function showInfo(data, tabletop) {
console.log(data);
console.log(base64.encode(data));
}
var vGlobals = {
dataString: base64.encode(data)
};
module.exports = vGlobals;
How can I access the data variable from showInfo, to use in vGlobals? It says that it hasn't been defined.
Your approach is wrong, you can't do it this way because TableTop call your callback asynchronously.
My suggestion (a quick one) :
var dataString = null;
module.exports = function(cb) {
if (dataString == null)
Tabletop.init({
key: 'xxxxxg46hgfjd',
callback: function(data, tabletop) {
dataString = base64.encode(data);
cb(dataString);
},
simpleSheet: true
});
else cb(dataString);
};
And to get the data :
var dataManager = require('./myfile');
dataManager(function(dataString) {
//here is your data do what you want with it
})
You should look/learn more about node/javascript and asynchronous/event-driven programing.
While implementing promises got this code:
var MongoClient = require('mongodb').MongoClient
MongoClient.connect(db_uri, function(err, db) {
if(err) throw err;
var ccoll = db.collection('cdata');
app.locals.dbstore = db;
}
var json= {}
//Auth is a wrapped mongo collection
var Auth = app.locals.Auth;
var coll = app.locals.dbstore.collection('data');
var ucoll = app.locals.dbstore.collection('udata');
var ccoll = app.locals.dbstore.collection('cdata');
var Q = require('q');
//testing with certain _id in database
var _id = require('mongodb').ObjectID('530ede30ae797394160a6856');
//Auth.getUserById = collection.findOne()
var getUser = Q.nbind(Auth.getUserById, Auth);
//getUserInfo gives a detailed information about each user
var getUserInfo = Q.nbind(ucoll.findOne, ucoll);
var getUserData = Q.nbind(ccoll.findOne, ccoll);
//"upr" is a group of users
//getUsers gives me a list of users, belonging to this group
var getUsers = Q.nbind(ucoll.find, ucoll);
//Auth.getUserById = collection.find()
var listUsers = Q.nbind(Auth.listUsers, Auth);
var uupr = {}
var cupr = {}
getUserInfo({_id:_id})
.then(function(entry){
console.log('entry:', entry);
uupr = entry;
var queue = [getUsers({upr:entry.name}), getUserData({_id:entry._id})]
return Q.all(queue);
}
)
.then(function(array2){
console.log('array2:', array2);
cupr = array2[1]
var cursor = array2[0]
var cfill = Q.nbind(cursor.toArray, cursor);
return cfill();
}
)
.then(function(data){
json = {data:data, uupr:uupr, cupr:cupr}
console.log('json:', json)
res.render('test', {json : JSON.stringify(json)})
}
)
Its work can be described by a diagram:
getUserInfo()==>(entry)--+-->getUsers()=====>array2[0]--+-->populate user list===>data--->render
| |
+-->getUserData()==>array2[1]--+
I've used external variables uupr and cupr to store data from first .then calls.
So I have two problems:
1) Avoid using external variables.
2) rearrange code to get alternative flow diagram.
getUserInfo()==>(entry)--+-->getUsers()==>usersList-->populate user list==>usersData-+->render
| |
+-->getUserData()====>uprData-------------------------------+
Any advice is appreciated
Try something along the lines of this pseudo-code:
getUserInfo().then(function(userInfo) {
return Q.all([
userInfo,
getUsers(... userInfo ...).then(convert to array),
getUserData(... userInfo ...)
])
}).spread(function(userInfo, usersArray, userData) {
res.render(...)
}, function(err) {
handle the error
}).done()
You can simply nest them:
getUserInfo({_id:_id})
.then(function(entry){
console.log('entry:', entry);
return Q.all([
getUsers({upr:entry.name}),
getUserData({_id:entry._id})
]);
.spread(function(cursor, cupr) {
console.log('array2:', [cursor, cupr]);
return Q.ninvoke(cursor, "toArray")
.then(function(data){
return {data:data, uupr:entry, cupr:cupr};
});
});
}).then(function(json) {
console.log('json:', json)
res.render('test', {json: JSON.stringify(json)})
});
Now, to let the toArray not wait for the getUserData result, just do those in parallel:
getUserInfo({_id:_id})
.then(function(entry){
console.log('entry:', entry);
return Q.all([
getUsers({upr:entry.name}).invoke("toArray"),
getUserData({_id:entry._id})
]);
.spread(function(data, cupr) {
return {data:data, uupr:entry, cupr:cupr};
});
}).then(function(json) {
console.log('json:', json)
res.render('test', {json: JSON.stringify(json)})
});
(Using invoke instead an explicit then)
Here is a sample of the working async code without mongodb. The problem is, if i replace the vars (data1_nodb,...) with the db.collection.find(); function, all needed db vars received at the end and the for()-loop ends not correct. Hope someone can help. OA
var calc = new Array();
function mach1(callback){
error_buy = 0;
// some vars
for(var x_c99 = 0; x_c99 < array_temp_check0.length;x_c99++){
// some vars
calc[x_c99] = new Array();
calc[x_c99][0]= new Array();
calc[x_c99][0][0] = "dummy1";
calc[x_c99][0][1] = "dummy2";
calc[x_c99][0][2] = "dummy3";
calc[x_c99][0][3] = "dummy4";
calc[x_c99][0][4] = "dummy5";
function start_query(callback) {
data1_nodb = "data1";
data2_nodb = "data2";
data3_nodb = "data3";
data4_nodb = "data4";
calc[x_c99][0][0] = data1_nodb;
calc[x_c99][0][1] = data2_nodb;
calc[x_c99][0][2] = data3_nodb;
callback(data1_nodb,data2_nodb,etc..);
}
start_query(function() {
console.log("start_query OK!");
function start_query2(callback) {
data4_nodb = "data5";
data5_nodb = "data6";
data6_nodb = "data7";
calc[x_c99][0][3] = data4_nodb;
calc[x_c99][0][4] = data5_nodb;
callback(data5_nodb,data6_nodb,etc..);
}
start_query2(function() {
console.log("start_query2 OK!");
function start_query3(callback) {
for(...){
// do something
}
callback(vars...);
}
start_query3(function() {
console.log("start_query3 OK!");
});
});
});
}
callback(calc);
};
function mach2(callback){
mach1(function() {
console.log("mach1 OK!");
for(...){
// do something
}
});
callback(calc,error_buy);
};
mach2(function() {
console.log("mach2 OK 2!");
});
You need to work with the async nature of the collection.find() method and wait for all of them to be done. A very popular approach is to use the async module. This module allows you run several parallel tasks and wait for them to finish with its async.parallel() method:
async.parallel([
function (callback) {
db.foo.find({}, callback);
},
function (callback) {
db.bar.find({}, callback);
},
function (callback) {
db.baz.find({}, callback);
}
], function (err, results) {
// results[0] is the result of the first query, etc
});