I have this route for this task:
var mongoose = require('mongoose');
var gfs;
var Grid = require('gridfs-stream');
var connection = mongoose.createConnection(mongodb://localhost/images);
connection.once('open', function () {
gfs = Grid(connection.db, mongoose.mongo);
});
app.get('/api/image/:file', function(req, res) {
res.set('Content-Type', getType(req.params.file));
gfs.createReadStream({
filename: '/api/image/'+req.params.file
}).pipe(res);
});
For some reason it's really slow, can't figure why is that. Even when i run it on local. If i use something like mongoose-fs or something familiar it will be faster?
Also every time image is loading, it don't keep that image in cache(every time is 200). How can i tell to node that this have to be cached? At the moment i have express way app.set('view cache', true);.
Related
I am trying to upload an image to my MongoDB. I'm using Mongoose, gridfs-stream and some other packages I was told to install (I'm relatively new to the whole MEAN ecosystem)
The request seems to go through fine, and the collections used for fs-stream do get auto added to my db, but the request always seems to time out with no error being returned and nothing being logged by nodemon.
Here is my code:
const router = require('express').Router();
const database = require('../config/database');
var mongoose = require('mongoose');
var Grid = require('gridfs-stream');
Grid.mongo = mongoose.connection;
var conn = mongoose.connection;
var gfs = new Grid(conn.db, mongoose.mongo);
router.get('/', function(req, res) {
res.send("Greetings!");
})
router.post('/img', function(req, res) {
var part = req.files.fileField;
var writeStream = gfs.createWriteStream({
filename: part.name,
mode: 'w',
content_type:part.mimetype
});
writeStream.on('close', function() {
return res.status(200).send({
message: 'Success'
});
writeStream.end();
})
});
I have tried running the mongodb locally, and on mlab, I have also tried images of varying sizes, most of which were less than 10MB and some less than 1MB. I'm at a loss of where to go from here as my research into the matter has also offered no answers.
I am trying to connect mongodb to my express nodejs web application. I am fresh new to nodejs. I am following this tutorial video but I couldn't complete it due to the connection of mongodb.
the app.js code I have:
var express = require('express');
var app = express();
var bodyParser = require('body-parser');
var mongoose = require('mongoose');
Genre = require('./models/genre');
let conn = mongoose.connection;
conn.openUri('mongodb://localhost/bookstore');
conn.on('error', err => console.error('mongodb connection error',
err));
conn.on('connected', () => console.info(`Connected to mongodb`));
conn.on('disconnected', () => console.info('Disconnected from
mongodb'));
app.get('/', function(req, res){
res.send('Hello World');
});
app.get('api/genres', function(req , res){
Genre.getGenres(function(err, genres){
if(err){
throw err;
}
res.json(genres);
})
});
app.listen(3666);
console.log('Server Running On http://localhost:3666');
and this is the genre.js
var mongoose = require('mongoose');
var genreSchema = mongoose.Schema({
name:{
type: String,
requires: true
},
create_date:{
type: Date,
default: Date.now
}
});
var Genre = module.exports = mongoose.model('Genre', genreSchema);
module.exports.getGenres = function(callback, limit){
Genre.find(callback).limit(limit);
}
and this is a picture of the database in the terminal
https://i.stack.imgur.com/S3gFb.png
And the information in genres collection in the database
https://i.stack.imgur.com/sJFE6.png
Once I open the main page I get the Hello World but once I add api/genres which I should get the data from mongodb I get this error
https://i.stack.imgur.com/B4c8o.png
and this is the files structures
https://i.stack.imgur.com/okHIN.png
I know this is a basic question but I couldnt figured out I check on google there are others way to connect to the database but I need to know why this particular way which I just followed from the tutorial video havent worked.
As you noticed I am a new to nodejs web development so if you could suggest websites or youtube channels to get me start it I would appreciate it.
It seems that it is not a db connection problem. Route matching http://localhost:3666/api/genres is not found in your application. Replace api/genres with /api/genres and I guess everything will work properly
I'm trying to parse and insert a big csv file into MongoDB but when the file extends 100'000 rows I get a bad response from the server. And the files I need to insert are usually above 200'000 rows.
I've tried both bulk insert (insertMany) and Babyparse(Papaparse) streaming approach to insert the file row by row. But with poor results.
Node api:
router.post('/csv-upload/:id', multipartMiddleware, function(req, res) {
// Post vartiables
var fileId = req.params.id;
var csv = req.files.files.path;
// create a queue object with concurrency 5
var q = async.queue(function(row, callback) {
var entry = new Entry(row);
entry.save();
callback();
}, 5);
baby.parseFiles(csv, {
header: true, // Includes header in JSON
skipEmptyLines: true,
fastMode: true,
step: function(results, parser) {
results.data[0].id = fileId;
q.push(results.data[0], function (err) {
if (err) {throw err};
});
},
complete: function(results, file) {
console.log("Parsing complete:", results, file);
q.drain = function() {
console.log('All items have been processed');
res.send("Completed!");
};
}
});
});
This streaming approach results in: POST SERVER net::ERR_EMPTY_RESPONSE
Not sure if I'm using the async.queue correctly though.
Is there a better and more efficient way to do this OR am I doing something wrong?
Express Server:
// Dependencies
var express = require('express');
var path = require('path');
var bodyParser = require('body-parser');
var routes = require('./server/routes');
var mongoose = require("mongoose");
var babel = require("babel-core/register");
var compression = require('compression');
var PORT = process.env.PORT || 3000;
// Include the cluster module
var cluster = require('cluster');
mongoose.connect(process.env.MONGOLAB_URI || 'mongodb://localhost/routes');
// Code to run if we're in the master process
if (cluster.isMaster) {
// Count the machine's CPUs
var cpuCount = require('os').cpus().length;
// Create a worker for each CPU
for (var i = 0; i < cpuCount; i += 1) {
cluster.fork();
}
// Code to run if we're in a worker process
} else {
// Express
var app = express();
app.use(bodyParser.json({limit: '50mb'}));
app.use(bodyParser.urlencoded({limit: '50mb', extended: true}));
// Compress responses
app.use(compression());
// Used for production build
app.use(express.static(path.join(__dirname, 'public')));
routes(app);
// Routes
app.use('/api', require('./server/routes/api'));
app.all('/*', function(req, res) {
res.sendFile(path.join(__dirname, 'public/index.html'));
});
// Start server
app.listen(PORT, function() {
console.log('Server ' + cluster.worker.id + ' running on ' + PORT);
});
}
Handling the import:
Great question, from my experience by far the fastest way to insert a csv into mongo is via the command line:
mongoimport -d db_name -c collection_name --type csv --file file.csv --headerline
I don't believe mongoose has a way of calling mongoimport (someone correct me if I'm wrong)
But it's simple enough to call via node directly:
var exec = require('child_process').exec;
var cmd = 'mongoimport -d db_name -c collection_name --type csv --file file.csv --headerline';
exec(cmd, function(error, stdout, stderr) {
// do whatever you need during the callback
});
The above will have to be modified to be dynamic, but it should be self-explanatory.
Handling the upload:
Uploading the file from a front-end client is another challenge.
Most browsers will timeout if you make a request to a server and don't get a response within 60 seconds (probably what you are referring to above)
One solution would be to open a socket connection (search for socket.io in npm) for details. This will create a constant connection to the server and won't be subject to the timeout restrictions.
If uploading is not an issue, and the timeout is due to the slow parsing/inserting then you may not have to worry about this once you implement the above.
Other considerations:
I'm not sure exactly what you need to send back to the user, or what parsing needs to take place. But that can either be done outside of the normal request/response cycle, or can be handled during a socket connection if it's needed during one request/response cycle.
I'd like to know how to work with connectivity to a database in MEAN stack application. In particular, when should I create a connection to a database and when should I destroy a connection to a database. Should I create and destroy a connection on every new HTTP request or should I store a once created connection and use it for any subsequent requests as long as possible. I use Mongoose as a modeling tool.
Here is an example.
This is my routes.js file with a route /index. A request to this route should fetch some date from MongoDb database. It bothers me how I connect and disconnect to a database now. Yes, I connect and disconnect to a database exactly as written in Mongoose docs, but it it the right way to do it in a serious production environment?
var express = require('express');
var router = express.Router();
var config = require('./db-config');
// I create a Mongoose instance as a module object,
// as opposite to create it in every request handler function below.
var mongoose = require('mongoose');
var productSchema = require('../db/productSchema'); // model schema is also a module-wide object
// And here is a request handler function.
// It is called on every request as a brand new.
// I create and destroy a database connection inside this request handler
router.get('/index', function(req, res, next) {
// I connect to a database on every request.
// Do I need to do it here in a request handler?
// May I do it outside of this request handler on a module-wide level?
mongoose.connect('mongodb://my_database');
// I create a new connection here in a request handler.
// So it lives only during this request handler run.
// Is this the right way? May I do it outside of this request handler
// on a module-wide level and somehow keep this connection and use it
// in every subsequent requests to this or any other route in the app?
var db = mongoose.connection;
db.on('connecting', function() {
console.log('connecting');
});
db.on('connected', function() {
console.log('connected');
});
db.on('open', function() {
console.log('open');
});
db.on('error', console.error.bind(console, 'connection error'));
db.once('open', function(cb) {
var Product = mongoose.model('Product', productSchema);
Product.find({category: "books"}, function(err, prods) {
if (err) return console.error(err);
// I close a connection here in a callback.
// As soon as successfully fetched the data.
// Do I need to close it after every request?
// What is the right place and time to do it?
db.close(disconnect);
res.json(prods);
});
});
})
Found some good answers:
https://softwareengineering.stackexchange.com/questions/142065/creating-database-connections-do-it-once-or-for-each-query
What are best practices on managing database connections in .NET?
Its best practice to have your db connection in a separate module (db.js)
var mongoose = require('mongoose')
mongoose.connect('mongodb://localhost/dbname', function(){
console.log('mongodb connected')
})
module.exports = mongoose
Each model should have a separate module that takes in the db connection (post.js)
var db = require('../db.js')
var Post = db.model('Post', {
username: {type: String, required: true},
body: {type: String, required: true},
date: { type: Date, required: true, default: Date.now }
})
module.exports = Post
Then whenever you need to use that data set just require it and make calls
var Post = require('/models/post')
Post.save()
Post.find()
This is an opinion based question I'd say. What I use for my app is
app.get('/', function (req, res) {
res.sendfile('index.html');
});
mongoose.connect('mongodb://localhost:27017/my_db');
This way I create a connection once rather than on every HTTP request. Your way should work fine but it seems you will have to connect and disconnect the db to your app way too many times specially when the app is in development.
You want your connection to act like a singleton so as mentioned in the answer above it makes sense to do it outside of, and preferable before your routes:
var compression = require('compression');
var express = require('express');
var app = express();
var port = process.env.PORT || 8080;
var cookieParser = require('cookie-parser');
var bodyParser = require('body-parser');
var session = require('express-session');
...
app.use(compression());
// db
var mongoose = require('mongoose');
var configDB = require('./config/database.js');
mongoose.connect(configDB.url); // connect to our database
config/database.js:
module.exports = {
'url' : '#localhost:27017/dbname'
};
This is my solution :
import express from 'express';
import mongoose from 'mongoose';
import { name } from '../package.json';
import * as localconfig from './local-config';
import debug from 'debug';
debug(name);
const app = express();
const port = process.env.PORT || 3000;
const mongoUrl = localconfig.credentials.MONGO_URL;
import usersRoutes from './routes/users/user-routes';
app.use('/v1/users', usersRoutes);
mongoose.connect(mongoUrl)
.then(() => {
debug('DB connection successful');
app.listen(port, '0.0.0.0', () => {
debug(`Running on port ${port}`);
});
})
.catch((err) => {
debug(err);
});
You should first check weather the connection is successful or not and only then listen to a certain port. This is my app.js file where all the routes are loaded, so you do not have to call the db connection in all your files. You have a single config file where all the config is done. Your router file user-routes.js will look something similar to this:
import express from 'express';
import User from '../models/user'
const router = express.Router();
router.get('/', (req, res, next) => {
User.find()
.then((response) => res.json(response))
.catch((err) => next(err));
});
module.exports = router;
I am using Cloude 9 environment for developing my nodejs app. In that I have written code to connect to mongodb database. I am successfully connecting to database and adding record to collection.
Now I want to send the collection info in return. But using res.send(collectionInfo); is not working.
Let me know how should I achieve this
Below is the code of my server.js file
var Db = require('mongodb').Db;
var http = require('http');
var path = require('path');
var async = require('async');
var socketio = require('socket.io');
var express = require('express');
var ejs = require('ejs');
var app = express();
var helpers = require('express-helpers')
var MongoClient = require('mongodb').MongoClient;
var Server = require('mongodb').Server;
var db;
helpers(app);
var bodyParser = require('body-parser');
app.use(bodyParser.json()); // for parsing application/json
app.use(bodyParser.urlencoded({extended: true})); // for parsing application/x-www-form-urlencoded
var server = http.Server(app);
server.listen(process.env.PORT || 3000, process.env.IP || "0.0.0.0", function () {
var addr = server.address();
console.log("Chat server listening at", addr.address + ":" + addr.port);
});
app.use(express.static(__dirname + '/public'));
app.set('views', __dirname + '/public/views');
app.engine('html', require('ejs').renderFile);
app.set('view engine', 'html');
//app.use(express.static(__dirname + '/client'));
app.use(express.static(path.join(__dirname, '/client')));
// MongoDB Connection
app.use(function(req, res, next) {
next();
})
app.get('/monogdb', function (req, res) {
res.render('monogdb.ejs');
});
app.post('/ajax-mongo-connect', function (req, res) {
var mongoClient = new MongoClient(new Server('localhost', 27017));
mongoClient.open(function(err, mongoClient) {
if(err){
console.log(err);
}else{
var db = mongoClient.db("mydb");
db.createCollection("students", { name : req.body.nm, description : req.body.desc, location : req.body.loc } );
console.log('database connected',db);
var collectionInfo = db.collection("students");
mongoClient.close();
//res.setHeader('Content-Type', 'application/json');
res.send(collectionInfo);
}
})
})
As per #Roman Sachenko answer, I have tried to use
res.send(collectionInfo.toJSON()); but it is giving below error
/home/ubuntu/workspace/node_modules/mongodb/lib/mongodb/db.js:299
throw err;
^
TypeError: Object #<Collection> has no method 'toJSON'
at /home/ubuntu/workspace/server.js:66:41
at MongoClient.open
(/home/ubuntu/workspace/node_modules/mongodb/lib/mongodb/mongo_client.js:103:5)
at Db.open (/home/ubuntu/workspace/node_modules/mongodb/lib/mongodb/db.js:296:11)
at process._tickCallback (node.js:442:13)
and using res.send({data: collectionInfo}); gives error
home/ubuntu/workspace/node_modules/mongodb/lib/mongodb/db.js:299
throw err;
^
TypeError: Converting circular structure to JSON
at Object.stringify (native)
at ServerResponse.res.json (/home/ubuntu/workspace/node_modules/express/lib/response.js:185:19)
at ServerResponse.res.send (/home/ubuntu/workspace/node_modules/express/lib/response.js:117:21)
at /home/ubuntu/workspace/server.js:67:21
at MongoClient.open (/home/ubuntu/workspace/node_modules/mongodb/lib/mongodb/mongo_client.js:103:5)
at Db.open (/home/ubuntu/workspace/node_modules/mongodb/lib/mongodb/db.js:296:11)
at process._tickCallback (node.js:442:13)
Try to return this: res.status(200).json({'myCollection' : collectionInfo});.
You can find more details about express response here
Update:
After you explain the details, take a look at the code below:
app.post('/ajax-mongo-connect', function (req, res) {
var mongoClient = new MongoClient(new Server('localhost', 27017));
mongoClient.open(function(err, mongoClient) {
if(err){
console.log(err);
res.status(500).json({message : 'OMG, an error occurred'});
}else{
var db = mongoClient.db("mydb");
db.createCollection("students", { name : req.body.nm, description : req.body.desc, location : req.body.loc } );
console.log('database connected',db);
var collectionInfo = db.collection("students");
// Here we will find all students
collectionInfo.find({}).toArray(function(err, students) {
// so now, we can return all students to the screen.
res.status(200).json({'myCollection' : students});
}
}
})
})
Cheers!
Mongoose ODM
First of all I would like to recommend you using Mongoose ODM:
https://github.com/Automattic/mongoose
So you will make you work with database much easier.
Basically it returns (Mongoose) normal object as results, but in case of issues you may try to use toObject() or toJSON() or as it mentioned create own object like {data: mongoCollection}
Examples:
res.send(collectionInfo.toObject());
res.send(collectionInfo.toJSON());
res.send({data: collectionInfo});
Please refer to the link in case of questions:
http://mongoosejs.com/docs/guide.html#toJSON
Native Driver
As for native driver, it also should return normally-constructed object, but according to issues I faced with in the past, JSON.stringify always helps if you set headers manually.
You may also check the contents of your entity. So you can just output it by console.log(collectionInfo);
Then just make sure that there is correct object inside.
And according to results you can take actions like:
res.send(JSON.stringify(collectionInfo)) //set headers manually
res.json(JSON.stringify(collectionInfo)) //you don't need to set headers
At least you will know what exactly is inside of collectionInfo. I think it will be enough to investigate the issue.
You can view circular JSON objects by doing this in node.js:
const util = require('util') // Native node module
util.inspect(circularObj)
You can call it from anywhere in the code, so it's very versatile.