Get a collection and add a value to the response - node.js

I want to create in the Server script a function that can return a collection plus some extra value.
For example:
Meteor.publish("users", function () {
var users;
users = Meteor.users.find();
users.forEach(function (user){
user.profile.image = "some-url";
});
return users;
});
But this don't work proper. My question is: What is the right way to add a value to a collection reponse in a publish function.

There are 2 ways you can implement a publish function:
By returning a cursor (or an array of cursors)
By using this.added(), this.changed() and this.removed().
Only method 2 allows to modify returned documents.
Please refer to Meteor documentation here. However, since the provided sample code might look complex, here is another one:
// server: publish the rooms collection
Meteor.publish("rooms", function () {
return Rooms.find({});
});
is equivalent to:
// server: publish the rooms collection
Meteor.publish("rooms", function () {
var self = this;
var handle = Rooms.find({}).observeChanges({
added: function(id, fields) { self.added("rooms", id, fields); },
changed: function(id, fields) { self.changed("rooms", id, fields); },
removed: function(id) { self.added("rooms", id); },
}
});
self.ready();
self.onStop(function () { handle.stop(); });
});
In the second sample, you can modify the 'field' parameter before sending it for publication, like this:
added: function(id, fields) {
fields.newField = 12;
self.added("rooms", id, fields);
},
Source: this post.

Is this important to do with the server? You could use the transform function on the client:
Client JS
//Somewhere where it can run before anything else (make sure you have access to the other bits of the document i.e services.facebook.id otherwise you'll get a services is undefined
Meteor.users._transform = function(doc) {
doc.profile.image = "http://graph.facebook.com/" + doc.services.facebook.id + "/picture";
return doc;
}
Now when you do:
Meteor.user().profile.image
=> "http://graph.facebook.com/55592/picture"
I have opened an issue before with regards to sharing a transform onto the client: https://github.com/meteor/meteor/issues/821

Related

Correct way to organise this process in Node

I need some advice on how to structure this function as at the moment it is not happening in the correct order due to node being asynchronous.
This is the flow I want to achieve; I don't need help with the code itself but with the order to achieve the end results and any suggestions on how to make it efficient
Node routes a GET request to my controller.
Controller reads a .csv file on local system and opens a read stream using fs module
Then use csv-parse module to convert that to an array line by line (many 100,000's of lines)
Start a try/catch block
With the current row from the csv, take a value and try to find it in a MongoDB
If found, take the ID and store the line from the CSV and this id as a foreign ID in a separate database
If not found, create an entry into the DB and take the new ID and then do 6.
Print out to terminal the row number being worked on (ideally at some point I would like to be able to send this value to the page and have it update like a progress bar as the rows are completed)
Here is a small part of the code structure that I am currently using;
const fs = require('fs');
const parse = require('csv-parse');
function addDataOne(req, id) {
const modelOneInstance = new InstanceOne({ ...code });
const resultOne = modelOneInstance.save();
return resultOne;
}
function addDataTwo(req, id) {
const modelTwoInstance = new InstanceTwo({ ...code });
const resultTwo = modelTwoInstance.save();
return resultTwo;
}
exports.add_data = (req, res) => {
const fileSys = 'public/data/';
const parsedData = [];
let i = 0;
fs.createReadStream(`${fileSys}${req.query.file}`)
.pipe(parse({}))
.on('data', (dataRow) => {
let RowObj = {
one: dataRow[0],
two: dataRow[1],
three: dataRow[2],
etc,
etc
};
try {
ModelOne.find(
{ propertyone: RowObj.one, propertytwo: RowObj.two },
'_id, foreign_id'
).exec((err, searchProp) => {
if (err) {
console.log(err);
} else {
if (searchProp.length > 1) {
console.log('too many returned from find function');
}
if (searchProp.length === 1) {
addDataOne(RowObj, searchProp[0]).then((result) => {
searchProp[0].foreign_id.push(result._id);
searchProp[0].save();
});
}
if (searchProp.length === 0) {
let resultAddProp = null;
addDataTwo(RowObj).then((result) => {
resultAddProp = result;
addDataOne(req, resultAddProp._id).then((result) => {
resultAddProp.foreign_id.push(result._id);
resultAddProp.save();
});
});
}
}
});
} catch (error) {
console.log(error);
}
i++;
let iString = i.toString();
process.stdout.clearLine();
process.stdout.cursorTo(0);
process.stdout.write(iString);
})
.on('end', () => {
res.send('added');
});
};
I have tried to make the functions use async/await but it seems to conflict with the fs.openReadStream or csv parse functionality, probably due to my inexperience and lack of correct use of code...
I appreciate that this is a long question about the fundamentals of the code but just some tips/advice/pointers on how to get this going would be appreciated. I had it working when the data was sent one at a time via a post request from postman but can't implement the next stage which is to read from the csv file which contains many records
First of all you can make the following checks into one query:
if (searchProp.length === 1) {
if (searchProp.length === 0) {
Use upsert option in mongodb findOneAndUpdate query to update or upsert.
Secondly don't do this in main thread. Use a queue mechanism it will be much more efficient.
Queue which I personally use is Bull Queue.
https://github.com/OptimalBits/bull#basic-usage
This also provides the functionality you need of showing progress.
Also regarding using Async Await with ReadStream, a lot of example can be found on net such as : https://humanwhocodes.com/snippets/2019/05/nodejs-read-stream-promise/

Compare API response against itself

I am trying to:
Poll a public API every 5 seconds
Store the resulting JSON in a variable
Store the next query to this same API in a second variable
Compare the first variable to the second
Print the second variable if it is different from the first
Else: Print the phrase: 'The objects are the same' if they haven't changed
Unfortunately, the comparison part appears to fail. I am realizing that this implementation is probably lacking the appropriate variable scoping but I can't put my finger on it. Any advice would be highly appreciated.
data: {
chatters: {
viewers: {
},
},
},
};
//prints out pretty JSON
function prettyJSON(obj) {
console.log(JSON.stringify(obj, null, 2));
}
// Gets Users from Twitch API endpoint via axios request
const getUsers = async () => {
try {
return await axios.get("http://tmi.twitch.tv/group/user/sixteenbitninja/chatters");
} catch (error) {
console.error(error);
}
};
//Intended to display
const displayViewers = async (previousResponse) => {
const usersInChannel = await getUsers();
if (usersInChannel.data.chatters.viewers === previousResponse){
console.log("The objects are the same");
} else {
if (usersInChannel.data.chatters) {
prettyJSON(usersInChannel.data.chatters.viewers);
const previousResponse = usersInChannel.data.chatters.viewers;
console.log(previousResponse);
intervalFunction(previousResponse);
}
}
};
// polls display function every 5 seconds
const interval = setInterval(function () {
// Calls Display Function
displayViewers()
}, 5000);```
The issue is that you are using equality operator === on objects. two objects are equal if they have the same reference. While you want to know if they are identical. Check this:
console.log({} === {})
For your usecase you might want to store stringified version of the previousResponse and compare it with stringified version of the new object (usersInChannel.data.chatters.viewers) like:
console.log(JSON.stringify({}) === JSON.stringify({}))
Note: There can be issues with this approach too, if the order of property changes in the response. In which case, you'd have to check individual properties within the response objects.
May be you can use npm packages like following
https://www.npmjs.com/package/#radarlabs/api-diff

Any way to reduce the amount of concurrent requests to fetch data and cache in nodejs?

I have an Express app which requires very low response rate ~<200ms. Right now we can only get this number but that's a separate topic.
We're planning to fetch a piece of data from the database, if found in Redis return the data if not then fire the request and save that to redis so the next requests can get it from Redis.
I'm running some testing and was wondering if there's a way to reduce the amount of database fetching requests?
For example, currently our application has 300req/s per box. We have six boxes running on AWS. If for the first time that piece of data is not available in Redis, there might be around ~500 requests trying to fetch the data from DB and cache that in Redis. We're trying to reduce that number down. Not sure if there's a way in Node.js or Redis to handle that.
Here's the code that I'm testing.
client.getAsync('key').then(function (data) {
if(data) {
console.log(data); // Return this data if found
res.send(data);
} else {
// I'm trying to reduce the number of calls for concurrent requests in this block.
console.log('not found');
var dataFromDb = // fetch data from DB
client.set('key', dataFromDb); // Fire and forget
res.send('not found'); // Return not found right away
}
});
And I test the call by using ab
ab -n 20 -c 10 http://localhost:8081/redis
This is the results I got
not found
not found
not found
not found
not found
not found
something
not found
something
something
something
something
something
something
something
something
something
something
In this example, there's 7 requests trying to fetch database with the same data and save to Redis.
My question is, is there anyway I can reduce the number of requests down? Because fetching DB is quite slow as of now ~900ms (We're trying to optimize that)
Yes there is. I did same thing. I will describe only logic here. Method to fetchCache should return a promise. Also you keep array of { cacheKey, promise }. Each time you send a request - you add key to this array. When next time you need to fetch cache - you check array first and if key there grabbing this promise. Else calling fetchCache.
Here is my code. It works, but probably hard to read. Should give you a basic understanding.
class DictTranslatableRepo {
constructor(model) {
var self = this;
self.title = model + "s Repo";
self.model = models[model];
self.running = {};
self.curItems = {};
}
*start() {
var self = this;
var curItems = yield self.model.findAll();
_.forEach(curItems, function(row) {
self.curItems[row.key] = row.value;
});
};
*map(from) {
var self = this;
if (from == "") return "";
if (!_.isUndefined(self.curItems[from])) return self.curItems[from];
if (_.isUndefined(self.running[from])) {
self.running[from] = [];
return new Promise(function(resolve, reject) {
self.running[from].push(resolve);
self.job(from, function(err, to) { // Main job
var callbackArr = self.running[from];
delete self.running[from];
_.forEach(callbackArr, function(callback) {
callback(to);
});
});
});
} else {
return new Promise(function(resolve, reject) {
self.running[from].push(resolve);
});
}
};
job(from, callback) {
var self = this;
var to = "as shown";
co(function*() {
try {
to = yield translator.translate(from);
yield self.model.add({key: from, value: to});
self.curItems[from] = to;
callback(null, to);
} catch (err) {
callback(err);
//logger.error("Cant translate entity: " + from);
}
}).catch(function(err) {
// Unhandled Error
callback(new Error(err));
});
};
}
My map method is your fetchCache method.

callback in callback with waterfall in nodejs

I am using MEAN (Mongo Express Angulars NodeJs) for a project. The problem is I have to add one extra attribute to data received from query object. And make new data array with exactly old data array but have one extra attribute. I know how to add attribute and pass them into callback using waterfall model, as I am using multiple callback functions and for loops I am not able to get expected result.
code:
var fetchRevenue = function(restaurantArray, type, startDate, endDate, fn) {
_.forEach(restaurantArray, function(rest) {
fetchDateWiseReport(new Date('07/10/2015'), new Date('07/16/2015'), rest._id, type, function(orders) {
var newOrders = [];
async.waterfall([
function(callback) {
if(orders && orders.length > 0){
async.forEach(orders, function(order) {
getSellingPriceOfItems(order.orders, function(sp) {
order.sp = sp;
newOrders.push(order);
if (newOrders.length === orders.length)
callback(null, newOrders);
});
});
} else {
newOrders.push([]);
}
},
function(newOrders, callback) {
var restArr = []
//get sum of all orders of each restaurant and add into restArr
callback(null, restArr);
},
function(restArr, callback) {
callback(null, newOrders);
}
], function(err, result) {
fn(result);
});
});
});
};
where my functions:
fetchDateWiseReport = fetches restaurant record for given date and send result in callback
getSellingPriceOfItems = query to item model find price for each item and return selling price of given array and send result in callback.
my complete code including all functions is here.
now I want orders should be equal to newOrders with additional attibute 'sp'. But I am unable to get this. Will you suggest me something to proceed?
Use Express way to handle callback problem
in you route
app.get('you/route',fetchDateWiseReport(), second(),finalReturningREsult())
your first function will be doing first async loop function assiggn reult in req.body.firstResult and pass to second function. and so on

Node.js and Redis

I am trying to link up a redis database with a Node.js application I am building to be able to store comments about items. I'm using the node_redis library to handle the connection. When I attempt to retrieve the comments out of the database however only "[true]" is returned. For testing purposes I have stuffed everything into one method and I have hardcoded the values in, but I still receive "[true]".
exports.getComment = function (id){
var comments = new Array();
rc.hmset("hosts", "mjr", "1", "another", "23", "home", "1234");
comments.push(rc.hgetall("hosts", function (err, obj) {
var comment = new Array();
if(err){
comment.push("Error");
} else {
comment.push(obj);
}
return comment;
}));
return comments;
}
Updated the code according to the tutorial and here is the result:
Retrieving the comment:
exports.getComment = function (id, callback){
rc.hgetall(id, callback);
}
Adding the comment:
exports.addComment = function (id, area, content, author){
//add comment into the database
rc.hmset("comment",
"id", id,
"area", area,
"content", content,
"author" , author,
function(error, result) {
if (error) res.send('Error: ' + error);
});
//returns nothing
};
Code to render:
var a = [];
require('../lib/annotations').addComment("comment");
require('../lib/annotations').getComment("comment", function(comment){
a.push(comment)
});
res.json(a);
Node.js is asynchronous. Which means it asynchronously does the redis stuff, and then gets the result back in the callback function.
I suggest you read this tutorial and fully understand it before getting further: http://howtonode.org/node-redis-fun
Basically, this way won't work:
function getComments( id ) {
var comments = redis.some( action );
return comments;
}
But it has to be this way:
function getComments( id, callback ) {
redis.some( action, callback );
}
This way, you use the API like this:
getComments( '1', function( results ) {
// results are available!
} );
The problem lays within the actual Redis-Node library when the call to addComment is made as it is below.
require('../lib/annotations').getComment("comment", function(comment){
a.push(comment)
});
This call is missing an argument in the callback function. The first argument is the error report which should return null if everything is ok, the second is the actual data. So it should be structured like the call below.
require('../lib/annotations').getComment("comment", function(comment){
a.push(err, comment)
});

Resources