Q.js variables passing in parallel flows - node.js

While implementing promises got this code:
var MongoClient = require('mongodb').MongoClient
MongoClient.connect(db_uri, function(err, db) {
if(err) throw err;
var ccoll = db.collection('cdata');
app.locals.dbstore = db;
}
var json= {}
//Auth is a wrapped mongo collection
var Auth = app.locals.Auth;
var coll = app.locals.dbstore.collection('data');
var ucoll = app.locals.dbstore.collection('udata');
var ccoll = app.locals.dbstore.collection('cdata');
var Q = require('q');
//testing with certain _id in database
var _id = require('mongodb').ObjectID('530ede30ae797394160a6856');
//Auth.getUserById = collection.findOne()
var getUser = Q.nbind(Auth.getUserById, Auth);
//getUserInfo gives a detailed information about each user
var getUserInfo = Q.nbind(ucoll.findOne, ucoll);
var getUserData = Q.nbind(ccoll.findOne, ccoll);
//"upr" is a group of users
//getUsers gives me a list of users, belonging to this group
var getUsers = Q.nbind(ucoll.find, ucoll);
//Auth.getUserById = collection.find()
var listUsers = Q.nbind(Auth.listUsers, Auth);
var uupr = {}
var cupr = {}
getUserInfo({_id:_id})
.then(function(entry){
console.log('entry:', entry);
uupr = entry;
var queue = [getUsers({upr:entry.name}), getUserData({_id:entry._id})]
return Q.all(queue);
}
)
.then(function(array2){
console.log('array2:', array2);
cupr = array2[1]
var cursor = array2[0]
var cfill = Q.nbind(cursor.toArray, cursor);
return cfill();
}
)
.then(function(data){
json = {data:data, uupr:uupr, cupr:cupr}
console.log('json:', json)
res.render('test', {json : JSON.stringify(json)})
}
)
Its work can be described by a diagram:
getUserInfo()==>(entry)--+-->getUsers()=====>array2[0]--+-->populate user list===>data--->render
| |
+-->getUserData()==>array2[1]--+
I've used external variables uupr and cupr to store data from first .then calls.
So I have two problems:
1) Avoid using external variables.
2) rearrange code to get alternative flow diagram.
getUserInfo()==>(entry)--+-->getUsers()==>usersList-->populate user list==>usersData-+->render
| |
+-->getUserData()====>uprData-------------------------------+
Any advice is appreciated

Try something along the lines of this pseudo-code:
getUserInfo().then(function(userInfo) {
return Q.all([
userInfo,
getUsers(... userInfo ...).then(convert to array),
getUserData(... userInfo ...)
])
}).spread(function(userInfo, usersArray, userData) {
res.render(...)
}, function(err) {
handle the error
}).done()

You can simply nest them:
getUserInfo({_id:_id})
.then(function(entry){
console.log('entry:', entry);
return Q.all([
getUsers({upr:entry.name}),
getUserData({_id:entry._id})
]);
.spread(function(cursor, cupr) {
console.log('array2:', [cursor, cupr]);
return Q.ninvoke(cursor, "toArray")
.then(function(data){
return {data:data, uupr:entry, cupr:cupr};
});
});
}).then(function(json) {
console.log('json:', json)
res.render('test', {json: JSON.stringify(json)})
});
Now, to let the toArray not wait for the getUserData result, just do those in parallel:
getUserInfo({_id:_id})
.then(function(entry){
console.log('entry:', entry);
return Q.all([
getUsers({upr:entry.name}).invoke("toArray"),
getUserData({_id:entry._id})
]);
.spread(function(data, cupr) {
return {data:data, uupr:entry, cupr:cupr};
});
}).then(function(json) {
console.log('json:', json)
res.render('test', {json: JSON.stringify(json)})
});
(Using invoke instead an explicit then)

Related

Parameter obj to Document() must be an object when trying to convert array to mongoose document with redis

I have using redis to cache my queries. Its working fine with object but not when i get array. It gives me an error **"Parameter "obj" to Document() must be an object, got kids", **. It also happens with count query. Here is my code :
const mongoose = require("mongoose");
const redis = require("redis");
const util = require("util");
const client = redis.createClient(process.env.REDIS_URL);
client.hget = util.promisify(client.hget);
const exec = mongoose.Query.prototype.exec;
mongoose.Query.prototype.cache = async function (options = {}) {
this.useCache = true;
this.hashKey = JSON.stringify(options.key || "");
this.time = JSON.stringify(options.time || 36000);
return this;
};
mongoose.Query.prototype.exec = async function () {
if (!this.useCache) {
return exec.apply(this, arguments);
}
const key = JSON.stringify(
Object.assign({}, this.getQuery(), {
collection: this.mongooseCollection.name,
})
);
// client.flushdb(function (err, succeeded) {
// console.log(succeeded); // will be true if successfull
// });
const cacheValue = await client.hget(this.hashKey, key);
if (cacheValue) {
const doc = JSON.parse(cacheValue);
/*
this.model refers to the Class of the corresponding Mongoose Model of the query being executed, example: User,Blog
this function must return a Promise of Mongoose model objects due to the nature of the mongoose model object having other
functions attached once is created ( validate,set,get etc)
*/
console.log("Response from Redis");
console.log(doc);
console.log(Array.isArray(doc));
return Array.isArray(doc)
? doc.map((d) => new this.model(d))
: new this.model(doc);
}
//await the results of the query once executed, with any arguments that were passed on.
const result = await exec.apply(this, arguments);
client.hset(this.hashKey, key, JSON.stringify(result));
client.expire(this.hashKey, this.time);
console.log("Response from MongoDB");
return result;
};
module.exports = {
clearHash(hashKey) {
client.del(JSON.stringify(hashKey));
},
};
Data in redis - [ 'kids', 'men', 'women' ]
Query - const collectionType = await Product.find() .distinct("collectionType") .cache({ key: "COLLECTION_TYPE" });
can i anyone please tell me what i am doing wrong?
I have solved by directly returning the doc and its working fine. Not sure if it is the right way if i directly do return doc then sending data from redis only

How to get the value from an api call and store it in a variable and updated a dynamodb record

I have a web request which gets some data and returns the response. I am looking to see how to store the response in a variable so it can be used later on in the code.
This is for node js running in a lambda function
/**
* Performs operations for vehicle management actions interfacing primiarly with
* Amazon DynamoDB table.
*
* #class vehicle
*/
/**
* Registers a vehicle to and owner.
* #param {JSON} ticket - authentication ticket
* #param {JSON} vehicle - vehicle object
* #param {createVehicle~callback} cb - The callback that handles the response.
*/
vehicle.prototype.createVehicle = function(ticket, vehicle, cb) {
let vehicle_data = [];
vehicle_data.push(vehicle);
let vin_data = _.pluck(vehicle_data, 'vin');
let vin_number = vin_data[0];
console.log(vin_number);
var options = {
url: 'https://vindecoder.p.mashape.com/decode_vin?' + 'vin=' + vin_number,
headers: {"X-Mashape-Key": "XXXXXXXXXXXXXXXXXXXXXXXXXX","Accept": "application/json"}
};
var data;
function callback(error, response, body) {
if (!error && response.statusCode == 200) {
var result = JSON.parse(body);
var data = result['specification'];
//console.log(data);
}
}
request(options, callback);
var year = data['year'];
var make = data['make'];
var model = data['model'];
var trim_level = data['trim_level'];
var engine = data['engine'];
var body_style = data['style'];
var made_in = data['made_in'];
var steering_type = data['steering_type'];
var anti_brake_system = data['anti_brake_system'];
var fuel_tank = data['tank_size'];
var overall_height = data['overall_height'];
var overall_length = data['overall_length'];
var overall_width = data['overall_width'];
var standard_seating = data['standard_seating'];
var optional_seating = data['optional_seating'];
var highway_mileage = data['highway_mileage'];
var city_mileage = data['city_mileage'];
vehicle.owner_id = ticket['cognito:username'];
// vehicle.vehicle_year = year;
// vehicle.make = make;
// vehicle.model = model;
// vehicle.trim_level = trim_level;
// vehicle.engine = engine;
// vehicle.body_style = style;
// vehicle.made_in = made_in;
// vehicle.steering_type = steering_type;
// vehicle.anti_brake_system = anti_brake_system;
// vehicle.fuel_tank = fuel_tank;
// vehicle.overall_height = overall_height;
// vehicle.overall_length = overall_length;
// vehicle.overall_width = overall_width;
// vehicle.standard_seating = standard_seating;
// vehicle.optional_seating = optional_seating;
// vehicle.highway_mileage = highway_mileage;
// vehicle.city_mileage = city_mileage;
let params = {
TableName: ddbTable,
Item: vehicle
};
let docClient = new AWS.DynamoDB.DocumentClient(dynamoConfig);
docClient.put(params, function(err, data) {
if (err) {
console.log(err);
return cb(err, null);
}
return cb(null, vehicle);
});
};
I expect the response from the api call to be stored in a an object so it can be used to update the dynamodb record
There are various problems in your code. See the comments I left in the code.
Your function is failing because of the asynchronous nature of javascript. Basically, you have a request callback with a result that never gets seen by the rest of your code. Using promises and async/await is one clean way to resolve this. See below:
// use request promise instead of request
// promises will make your life much easier
// https://hackernoon.com/javascript-promises-and-why-async-await-wins-the-battle-4fc9d15d509f
// https://github.com/request/request-promise
const request = require('request-promise')
vehicle.prototype.createVehicle = function(ticket, vehicle, cb) {
// No need to use var anymore with es6, just use let and const
// https://www.sitepoint.com/es6-let-const/
let vehicle_data = [];
vehicle_data.push(vehicle);
let vin_data = _.pluck(vehicle_data, "vin");
let vin_number = vin_data[0];
console.log(vin_number);
const options = {
uri: "https://vindecoder.p.mashape.com/decode_vin?" + "vin=" + vin_number,
headers: {
"X-Mashape-Key": "XXXXXXXXXXXXXXXXXXXXXXXXXX",
Accept: "application/json"
}
};
// Here's the main mistake
// request has a callback function that is asynchronous
// the rest of your code never sees the result because the result doesn't leave the callback
// your code continues to execute without waiting for the result (this is the gist of asynchronity in js)
// function callback(error, response, body) {
// if (!error && response.statusCode == 200) {
// const result = JSON.parse(body);
// const data = result["specification"]; // this variable doesn't leave the scope of this callback
// //console.log(data);
// }
// }
// therefore this is the failure point
// request(options, callback);
// Do this instead
// here I utilize promises and async/ await
// https://medium.com/codebuddies/getting-to-know-asynchronous-javascript-callbacks-promises-and-async-await-17e0673281ee
try {
const result = await request(options);
const data = result["specification"];
} catch (error) {
console.log(error);
}
// Now data is available to be used below in dynamodb
// also, utilize objects, its much cleaner
const carVariables = {
year: data["year"],
make: data["make"],
model: data["model"],
trim_level: data["trim_level"],
engine: data["engine"],
body_style: data["style"],
made_in: data["made_in"],
steering_type: data["steering_type"],
anti_brake_system: data["anti_brake_system"],
fuel_tank: data["tank_size"],
overall_height: data["overall_height"],
overall_length: data["overall_length"],
overall_width: data["overall_width"],
standard_seating: data["standard_seating"],
optional_seating: data["optional_seating"],
highway_mileage: data["highway_mileage"],
city_mileage: data["city_mileage"],
};
vehicle.owner_id = ticket["cognito:username"];
vehicle = { ...vehicle, ...carVariables} // ES6 spread operator does the below code for you:
// one line versus 20. win. // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Spread_syntax
// vehicle.vehicle_year = year;
// vehicle.make = make;
// vehicle.model = model;
// vehicle.trim_level = trim_level;
// vehicle.engine = engine;
// vehicle.body_style = style;
// vehicle.made_in = made_in;
// vehicle.steering_type = steering_type;
// vehicle.anti_brake_system = anti_brake_system;
// vehicle.fuel_tank = fuel_tank;
// vehicle.overall_height = overall_height;
// vehicle.overall_length = overall_length;
// vehicle.overall_width = overall_width;
// vehicle.standard_seating = standard_seating;
// vehicle.optional_seating = optional_seating;
// vehicle.highway_mileage = highway_mileage;
// vehicle.city_mileage = city_mileage;
let params = {
TableName: ddbTable,
Item: vehicle
};
// This will now probably work. yay!
let docClient = new AWS.DynamoDB.DocumentClient(dynamoConfig);
docClient.put(params, function(err, data) {
if (err) {
console.log(err);
return cb(err, null);
}
return cb(null, vehicle);
});
};
This is untested but I believe it should work. And there may some typos or whatever but the main takeaway here is the use of promises and async/await to wait for and expose the result from the request. See the links and references in the comments for further reading.

Get data from nested foreach

I'm building an app using firebase and Node.js. I need to get data from nested foreach. How to do it correctly? Need to return the results of all iterations simultaneously.
exports.userParty = function (userInfo, cb) {
var userID = userInfo.userID;
var clubID = userInfo.clubID;
var refUserParty = ref.child(userID).child('my_party_id');
var party = {};
refUserParty.orderByValue().once("value", function (snapshot) {
var party = {};
snapshot.forEach(function (partyID) {
var refParty = dbb.ref('clubs').child(clubID).child('party').child(partyID.val());
refParty.once('value', function (partyBody) {
party[partyID.val()] = partyBody.val();
//console.log(party);
});
});
cb(party); // {}
});
};
You need to call the callback after all the async functions in the forEach block have completed. You can do this using a simple counter to check all async functions are complete:
...
let completedSnapshots = 0;
snapshot.forEach(function (partyID) {
var refParty = dbb.ref('clubs').child(clubID).child('party').child(partyID.val());
refParty.once('value', function (partyBody) {
party[partyID.val()] = partyBody.val();
completedSnapshots++;
if (completedSnapshots === snapshot.val().length) {
cb(party);
}
});
});
...

How can i write a mocha test for the following function?

I want to write a test for this node.js funcion,This has two arguments request and response. I set the request variable . But dont know how to set response variable.
function addCustomerData(request, response) {
common.getCustomerByMobile(request.param('mobile_phone'), function (customerdata) {
if (!customerdata) {
var areaInterest = request.param('area_interest');
var customerInfo = {userType: request.param('userType'),
firstName : request.param('first_name'),
middleName : request.param('middle_name'),
lastName : request.param('last_name'),
email : request.param('email'),
mobilePhone : request.param('mobile_phone'),
uniqueName : request.param('user_name'),
company : request.param('company')
};
if(customerInfo.email){
customerInfo.email = customerInfo.email.toLowerCase();
}
if(customerInfo.uniqueName){
customerInfo.uniqueName = customerInfo.uniqueName.toLowerCase();
}
if(areaInterest) {
customerInfo.areaInterest = '{' + areaInterest + '}';
}else
areaInterest = null;
addCustomer(request, response, customerInfo, function (data) {
request.session.userId = data;
return response.send({success: true, message: 'Inserted successfully'});
}
);
} else {
return response.send({success: false, message: 'User with this mobile number already exists'});
}
});
}
I wrote the test as follows
describe('signup', function(){
describe('#addCustomer()', function(){
before(function (done) {
request = {};
request.data = {};
request.session = {};
request.data['userType'] = '3';
request.data['first_name'] = 'Shiji';
request.data['middle_name'] = '';
request.data['last_name'] = 'George';
request.data['email'] = 'shiji#lastplot.com';
request.data['mobile_phone'] = '5544332333';
request.data['user_name'] = 'shiji';
request.session['imageArray'] = [];
request.param=function(key){
// Look up key in data
return this.data[key];
};
request1 = {};
request1.data = {};
request1.session = {};
request1.data['area_interest']=["aluva","ernakulam"];
request1.data['userType'] = '1';
request1.data['first_name'] = 'Hari';
request1.data['middle_name'] = 'G';
request1.data['last_name'] = 'Ganesh';
request1.data['email'] = 'hari#lastplot.com';
request1.data['mobile_phone'] = '5544332321';
request1.data['user_name'] = 'hariganesh';
request1.data['company'] = 'HG Realestate';
request1.session['imageArray'] = [];
request1.param=function(key){
// Look up key in data
return this.data[key];
};
done();
});
it('It should list the matching properties', function(done){
async.parallel([
function(callback) {
signup.addCustomerData(request, response, function (result, err) {
should.not.exist(err);
should.exists(result);
callback();
});
},
function(callback) {
signup.addCustomerData(request1, response, function (result, err) {
should.not.exist(err);
should.exists(result);
callback();
});
}],function(){
done();
});
});
But i got the error as response has no method send()
Thanks in Advance.
Your addCustomerData function does not have a callback, it just calls respond.send(). You need to mock the response object, as well as the send method, and put your tests inside of it, but you won't be able to use async.parallel() as, like I already mentioned, your function does not have a callback parameter. If you're testing request/response functions, I suggest you look into Supertest https://github.com/visionmedia/supertest which is widely used for cases like this.

Get model from mongoose db

I'm currently looking into building a small REST based service to which I can POST some data into a mongoose db and GET the data back.
Here's my main.js file:
var http = require("http");
var DAO = require("./DAO");
var express = require("express");
var util = require('util');
var app = express();
app.use(express.bodyParser());
app.post('/postIsles',function(req,res){
DAO[req.method](req.body);
res.send("body" + req.body.name);
});
app.get('/getIsles',function(req,res){
var isleVar = DAO[req.method](req);
res.send(isleVar);
});
app.listen("3000");
console.log("\nApp available at http://127.0.0.1:3000\n");
And DAO.js:
var mongoose = require('mongoose');
//Connect to database
mongoose.connect( 'mongodb://127.0.0.1:27017/library_database' );
//Schemas
var Isle = new mongoose.Schema({
name: String,
description: String,
lastStocked: Date
});
//Models
var IsleModel = mongoose.model( 'Isle', Isle );
function POST(request) {
var name = request.name;
var description = request.description;
var lastStocked = request.lastStocked;
console.log("POST REQ +" + request);
var isle = new IsleModel({
name: name,
description: description,
lastStocked: lastStocked
});
isle.save( function( err ) {
if( !err ) {
return console.log( 'created' );
} else {
return console.log( err );
}
});
}
function GET(request) {
return IsleModel.find( function( err, islesT ) {
if( !err ) {
console.log("isles :"+islesT);
return islesT;
} else {
return console.log( err );
}
});
}
exports.POST = POST;
exports.GET = GET;
When I try to run the GET, I get the following error:
TypeError: Converting circular structure to JSON
at Object.stringify (native)
I'm a bit unsure how to overcome this.
Remember when using Node.js: any operation that involves IO will be asynchronous.
Model#find is an asynchronous method, so isleVar is not set to the result you're expecting. Your result will only be available inside of the anonymous function that you pass into IsleModel.find
To fix your GET method, you'll need to modify your code to take into account the asynchronicity of the DB request and only send the response once your app has had a chance to retrieve data.
Below, is an example of one possible solution to fix /getIsles:
In main.js, modify your get route to pass in res (so it can be handled asynchronously)
app.get('/getIsles',function(req,res){
return DAO[req.method](req, res);
});
In DAO.js, have response send the data inside of your callback to IsleModel.find
function GET(request, response) {
IsleModel.find( function( err, islesT ) {
if( !err ) {
console.log("isles :"+islesT);
response.send(islesT);
} else {
return console.log( err );
}
});
}

Resources