Building a simple news feed in node + Mongodb + Redis - node.js

Question

My goal is to build a simple news feed in node.js with the help of mongodb and redis. It similar like twitter
So the scenario is pretty straight forward, once User A follow User B. Later on User's A News feed (Home page) will be shown User B's Activity like what he posted.
Schema for User
const UserSchema = new Schema({
email: { type: String, unique: true, lowercase: true},
});
const followSchema = new Schema(
{
user: { type: Schema.Types.ObjectId, required: true, ref: 'User' },
target: { type: Schema.Types.ObjectId, required: true, ref: 'User' },
});
Currently the design of my user's schema is pretty simple, when I follow another user, I will just create the Follow Schema Object
and there is another schema, which is post schema
/* This is similar like the Tweet */
var PostSchema = new Schema({
// Own by the user
creator: { type: Schema.Types.ObjectId, ref: 'User' }
body: String,
});
This schema is for user to post anything, similar like twitter posting.
Let say I have followed bunch of users
{
user: 'me',
target: 'draco'
},
{
user: 'me',
target: 'donald'
},
{
user: 'me',
target: 'joker'
}
and let say one of my followers, post something. How do i present it to my current news feed?
/* I'm following Joker */
app.post('/follow', (req, res, next) => {
let follow = new Follow();
follow.user = "me";
follow.target = "joker";
// Do i need to use redis to subscribe to him?
follow.save();
})
/* Joker posted something */
app.post('/tweet',(req, res, next) => {
let post = new Post();
post.creator = "joker";
post.body = "Hello my name is joker"
post.save();
// Do i need to publish it using redis so that I will get his activity?
});
Here's my attempt
app.get('/feed', function(req, res, next) {
// Where is the redis part?
User.findOne({ _id: req.user._id }, function(err, foundUser) {
// this is pretty much my attempt :(
})
})
When should I use redis to actually do the pub and sub? so that I could take the content of one of my followers and show it on my timeline?

Answer 1

I have built a social network which has a news feed, too. Here is how I did it.
Basically, you have 2 methods to built a newsfeed:
Fanout on write (push) method
Fanout on read (pull) method
Fanout on write
First, you will need another collection:
const Newsfeed = new mongoose.model('newsfeed', {
owner: {type: mongoose.Types.ObjectId, required: true},
post: {type: mongoose.Types.ObjectId, required: true}
});
When a user post something:
Get n follower
Push (fanout) this post to n follower
When a user get a feed:
Get from Newsfeed collection
Example:
router.post('/tweet', async (req, res, next) => {
let post = await Post.create({});
let follows = await Follow.find({target: req.user.id}).exec();
let newFeeds = follows.map(follow => {
return {
user: follow.user,
post: post.id
}
});
await Newsfeed.insertMany(newFeeds);
});
router.get('/feed', async (req, res, next) => {
let feeds = await Newsfeed.find({user: req.user.id}).exec();
});
Fanout on read
When a user post something:
Save
When a user get feed
Get n following
Get posts from n following
Example
router.post('/tweet', async (req, res, next) {
await Post.save({});
});
router.get('/feeds', async (req, res, next) {
let follows = await Follow.find({user: req.user.id}.exec();
let followings = follows.map(follow => follow.target);
let feeds = await Post.find({user: followings}).exec();
});
You don't need Redis or pub/sub to implement a newsfeed. However, in order to improve the performance, you may need Redis to implement some kind of cache for this.
For more information or advance technique, you may want to take a look at this.

Answer 2

User Schema :
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
const userSchema = new Schema({
name:{type:String},
email: { type: String, unique: true, lowercase: true},
},{
collection: 'User'
});
var User = module.exports = mongoose.model('User', userSchema);
Follow Schema :
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
var followSchema = new Schema(
{
follow_id: { type: Schema.Types.ObjectId, required: true, ref: 'User' },
leader_id: { type: Schema.Types.ObjectId, required: true, ref: 'User' }
},{
collection:'Follow'
});
var Follow = module.exports = mongoose.model('Follow', followSchema);
Post Schema :
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
var postSchema = new Schema({
creator: { type: Schema.Types.ObjectId, ref: 'User' }
body: {type: String , required:true},
created_at :{type:Date , default:Date.now}
},{
collection:'Post'
});
var Post = module.exports = mongoose.model('Post', postSchema);
Now Suppose you have 3 users in User collection :
{ _id: ObjectID('5a2ac68d1413751391111111') ,name:'John' , email:'john#gmail.com'}
{ _id: ObjectID('5a2ac68d1413751392222222') ,name:'Morgan' , email:'morgan#yahoo.com'}
{ _id: ObjectID('5a2ac68d1413751393333333') ,name:'Emily' , email:'emily#outlook.com'}
Now John Follows Morgan and Emily :
so in Follow collection there are two records
1) follow_id = John's ID and leader_id = Morgan's ID
2) follow_id = John's ID and leader_id = Emily's ID
{
_id: ObjectID('5a2ac68d141375139999999'),
follow_id : ObjectID('5a2ac68d1413751391111111'),
leader_id : ObjectID('5a2ac68d1413751392222222')
},
{
_id: ObjectID('5a2ac68d1413751393333333'),
follow_id : ObjectID('5a2ac68d1413751391111111'),
leader_id : ObjectID('5a2ac68d1413751393333333')
}
Now if you want to get User's Following :
app.get('/following/:user_id',function(req,res){
var userid=req.params.user_id;
Follow.find({follow_id:mongoose.mongo.ObjectID(userid)})
.populate('leader_id')
.exec(function(err,followings){
if(!err && followings){
return res.json({followings:followings});
}
});
});
for getting User's Followers :
app.get('/followers/:user_id',function(req,res){
var userid=req.params.user_id;
Follow.find({leader_id:mongoose.mongo.ObjectID(userid)})
.populate('follow_id')
.exec(function(err,followers){
if(!err && followers){
return res.json({followers:followers});
}
});
});
npm install redis
in your app.js :
var redis = require('redis');
var client = redis.createClient();
When one user create post :
app.post('/create_post',function(req,res){
var creator=new mongoose.mongo.ObjectID(req.body.creator);
var postbody=req.body.body;
async.waterfall([
function(callback){
// find followers of post creator
Follow.find({leader_id:creator})
.select({ "follow_id": 1,"leader_id":0,"_id": 0})
.exec(function(err,followers){
if(!err && followers){
callback(null,followers);
}
});
},
function(followers, callback){
// saving the post
var post=new Post({
creator: creator,
body: postbody
});
post.save(function(err,post){
if(!err && post){
// adding newly created post id to redis by key userid , value is postid
for(var i=0;i<followers.length;i++){
client.sadd([followers[i].follow_id,post.id]);
}
callback(null,post);
}
});
}
], function (err, result) {
if(!err && result){
return res.json({status:"success",message:"POST created"});
}
});
});
Now For Getting User NewsFeed :
1) first get array of postid from redis key of userid
2) loop through postid and get post from mongo
Function for get newsfeed by userid :
app.get('/newsfeed/:user_id',function(req,res){
var userid=req.params.user_id;
client.smembers(userid,function(err, reply) {
if(!err && reply){
console.log(reply);
if(reply.length>0){
var posts=[];
for(var i=0;i<reply.length;i++){
Post.findOne({_id:new mongoose.mongo.ObjectID(reply[i])}).populate('creator').exec(function(err,post){
posts.push(post);
});
}
return res.json({newsfeed:posts});
}else{
// No News Available in NewsFeed
}
}
});
});
Here we have use redis to store [userid,array of postids] for newsfeed ,
but if you dont want to use redis than just use below Newsfeed Model and store user_id and post_id for newly created post and then display it.
NewsFeed Schema :
var mongoose = require('mongoose');
var Schema = mongoose.Schema;
var newsFeedSchema = new Schema({
user_id: {type: Schema.Types.ObjectId, refer:'User' , required:true}
post_id: {type: Schema.Types.ObjectId, refer:'Post' , required:true},
},{
collection:'NewsFeed'
});
var NewsFeed = module.exports = mongoose.model('NewsFeed', newsFeedSchema);
Helpful link for Redis : https://www.sitepoint.com/using-redis-node-js/
for Async : https://caolan.github.io/async/docs.html#