I have an http server and every time it gets a post request, it is supposed to insert the data into MongoDB. This server is supposed to be constantly running and accepting thousands of request in any given second.
How can I maximize the efficiency and speed of my code? I feel like my current code is not fast enough and furthermore wastes CPU power when it makes a new db every time it receives a request.
My current layout
var server = http.createServer(function (req, res) {
req.on('data', function(chunk) {
//Receive my data
});
req.on('end', function() {
//JSON parse my data
var db = new Db('test', new Server("111.111.11.111", 27017,{auto_reconnect: false}), {safe: true});
db.open(function(err, db) {
//Insert data into DB
db.close();
});
});
}); //End Http server
server.listen(8999);
I have tried replacing db.open with MongoClient.connect, but that considerably slows down processing and I don't know why. In this case, the older version of MongoDB Native for node js seems to work faster.
You'll want to shift to an approach where you open a large pool of connections during startup that are shared by your HTTP request handlers. To tweak the MongoDB connection pool size to suit whatever scalability needs you have, pass an options parameter to your MongoClient.connect call.
var options = {
server: {
// The number of pooled connection available to your app.
poolSize: 100
}
};
mongodb.MongoClient.connect('mongodb://111.111.11.111/test', options,
function(err, db) {
var server = http.createServer(function (req, res) {
// Your req.on calls go here, directly using db rather than creating
// new instances. Don't close db either.
});
server.listen(8999);
}
);
Not sure if this would be better, but you can encapsulate that server inside the db, therefore persisting the connection:
var db = new Db('test', new Server("111.111.11.111", 27017,{auto_reconnect: false}), {safe: true});
db.open(function(err, db) {
var server = http.createServer(function (req, res) {
//now do stuff thru constant open connection
});
db.close();
});
Related
Currently, I have a lot of linux based clients downloading firmware updates from my webserver.
After the client has successfully downloaded the firmware file, my server needs to execute a few scripts, which logs in to the client and performs some tasks.
Is there a way for a node server to keep track of the clients download progress, so I can execute the needed scripts once the file has been downloaded?
Ok so I will try.
If you serve your Firmware as static files through Apache/Nginx and direct url call. You don't get the progress inside your NodeJS script.
If you serve your files via stream inside a express controller you can listen to the progress. Look at this answer here https://stackoverflow.com/a/42273080/3168392
You will have to use a socket connection to make sure the node server gets update from the client of the progress of the file being downloaded.
Something like this
CLIENT_CODE
var socket = io('http://localhost');
socket.on('connect', function(){});
socket.on('data_reciving', parts.pop(),function(percentage){
if(parse_data(percentage) === 100){
client.socket.emit('downloadCompleted', {fileName:'test'});
}else{
//do nothing
}
});
SERVER_CODE:
sockets.on('connection', function (socket) {
//listen to the event from client
socket.on('downloadCompleted', function (data) {
connect_to_client();
do_some_operation();
socket.emit('ALLDONE',{some_data});
});
});
I guess this helps ,you can use this post for reference.
If you just want to run some code when a download has finished, you can use on-finished:
const onFinished = require('on-finished');
app.use((req, res, next) => {
onFinished(res, (err, res) => {
...log some data, perform some housekeeping, etc...
});
next();
});
As is, it will attach a "finished" listener to all responses, which is probably not what you want. Since this is plain Express middleware, you can attach it to specific routes instead (but how depends on how exactly the files are being served).
I found some code that seems to fit my needs.
With the code below, I can detect both the progress of a user's download from the server-side, and fire an event once the file transfer completes.
var http = require("http");
var fs = require("fs");
var filename = "./large-file";
var serv = http.createServer(function (req, res) {
var sent = 0;
var lastChunkSize = 0;
var stat = fs.statSync(filename);
res.setHeader('Content-disposition', 'attachment; filename=large-file.iso');
res.setHeader('Accept-Ranges', 'bytes');
res.setHeader('Keep-Alive', 'timeout=5, max=100');
res.writeHeader(200, {"Content-Length": stat.size});
var fReadStream = fs.createReadStream(filename, { highWaterMark: 128 * 1024 });
fReadStream.on('data', function (chunk) {
if(!res.write(chunk)){
fReadStream.pause();
lastChunkSize = chunk.length;
console.log('Sent', sent, 'of', stat.size);
}
});
fReadStream.on('end', function () {
console.log('Transfer complete.');
res.end();
});
res.on("drain", function () {
sent += lastChunkSize;
fReadStream.resume();
});
});
serv.listen(3001);
I am using the express framework and would like to connect to a mongodb without using mongoose, but with the native nodejs Mongodb driver. How can I do this without creating a new connection every time?
To handle get or post requests I currently open a new connection to the db for every request and close it on completion of the request. Is there a better way to do this? Thanks in advance.
Following the example from my comment, modifying it so that the app handles errors rather than failing to start the server.
var express = require('express');
var mongodb = require('mongodb');
var app = express();
var MongoClient = require('mongodb').MongoClient;
var dbURL = "mongodb://localhost:27017/integration_test";
var db;
// Initialize connection once
MongoClient.connect(dbURL, function(err, database) {
if(err) return console.error(err);
db = database;
// the Mongo driver recommends starting the server here
// because most apps *should* fail to start if they have no DB.
// If yours is the exception, move the server startup elsewhere.
});
// Reuse database object in request handlers
app.get("/", function(req, res, next) {
var collection = "replicaset_mongo_client_collection";
db.collection(collection).find({}, function(err, docs) {
if(err) return next(err);
docs.each(function(err, doc) {
if(doc) {
console.log(doc);
}
else {
res.end();
}
});
});
});
app.use(function(err, req, res){
// handle error here. For example, logging and
// returning a friendly error page
});
// Starting the app here will work, but some users
// will get errors if the db connection process is slow.
app.listen(3000);
console.log("Listening on port 3000");
var mongodb = require('mongodb');
var uri = 'mongodb://localhost:27017/dbname';
module.exports = function(callback) {
mongodb.MongoClient.connect(uri, callback);
};
Ad this snippet in a file say connect.js and then require this file(connect.js) in your file where you are declaring your functions for http requests.
My server need to handle too many requests (ex:100 req/sec) at a time and it involves db(mongodb) operations. To accomplish that, I've chosen Express Server to handle those request. I set up server with the express. Here I want to create only one db-connection for entire server.
To achieve it:
in app.js
var mongo = require('mongodb');
var monk = require('monk');
var db = monk('localhost:27017/testdb1');
app.use(function (req, res, next) {
req.db1 = db;
//console.log(req.db1);
next();
});
in routes.js
router.get('/user', function (req, res) {
if (req) {
find = {};
var db1 = req.db1;
var userCollection = db1.get(collections.user);
var testCollection1 = db1.get(collections.testTable1);
var testCollection2 = db1.get(collections.testTable2);
userCollection.find(find, function (err, doc) {
console.log('1');
});
testCollection1.find(find, function (err, doc) {
console.log('2');
});
testCollection2.find(find, function (err, doc) {
console.log('3');
});
}
});
And the Problem is:
Let current mongodb connections(cmc) = 1;
server creates one connection as it lifts(cmc++). When I make a request to route(/user) another connection is created(cmc++) for the first time. When I make a request again, one more connection is created(cmc++). After that although I made a number of requests to that route the count remains the same(cmc=4). What exactly happening there ?
I need to update more collections when request is made. As no.of operations on collections increases connection count increasing.
why the connection count is increasing. I want to create only one connection.
I'm writing a Node.js web server that uses a Postgres database. I used to connect on each new request like this:
app.get('/', function (req, res) {
pg.connect(pgconnstring, function (err, client) {
// ...
});
});
But after a few requests, I noticed 'out of memory' errors on Heroku when trying to connect. My database has only 10 rows, so I don't see how this could be happening. All of my database access is of this form:
client.query('SELECT * FROM table', function (err, result) {
if (err) {
res.send(500, 'database error');
return;
}
res.set('Content-Type', 'application/json');
res.send(JSON.stringify({ data: result.rows.map(makeJSON) }));
});
Assuming that the memory error was due to having several persistent connections to the database, I switched to a style I saw in several node-postgres examples of connecting only once at the top of the file:
var client = new pg.Client(pgconnstring);
client.connect();
app.get('/', function (req, res) {
// ...
});
But now my requests hang (indefinitely?) when I try to execute a query after the connection is disrupted. (I simulated it by killing a Postgres server and bringing it back up.)
So how do I do one of these?
Properly pool Postgres connections so that I can 'reconnect' every time without running out of memory.
Have the global client automatically reconnect after a network failure.
I'm assuming you're using the latest version of node-postgres, in which the connection pooling has been greatly improved. You must now check the connection back into the pool, or you'll bleed the connections:
app.get('/', function (req, res) {
pg.connect(pgconnstring, function (err, client, done) {
// do some stuff
done();
});
});
As for error handling on a global connection (#2, but I'd use the pool):
client.on('error', function(e){
client.connect(); // would check the error, etc in a production app
});
The "missing" docs for all this is on the GitHub wiki.
I'm writing some simple web app with Node.JS and want to use mongoDB as main data storage.
Node-mongodb-native driver requires to make chained calls before you actually can query or store data (open DB connection, authenticate, get collection).
Where is the best place to do this initialization - within each request handler or globally, when initializing application?
You are MUCH better putting the Mongo initialization outside of your request handler - otherwise it will re-connect for every page that is served:
var mongo = require('mongodb');
// our express (or any HTTP server)
var app = express.createServer();
// this variable will be used to hold the collection for use below
var mongoCollection = null;
// get the connection
var server = new mongo.Server('127.0.0.1', 27017, {auto_reconnect: true});
// get a handle on the database
var db = new Db('testdb', server);
db.open(function(error, databaseConnection){
databaseConnection.createCollection('testCollection', function(error, collection) {
if(!error){
mongoCollection = collection;
}
// now we have a connection - tell the express to start
app.listen(80);
});
});
app.use('/', function(req, res, next){
// here we can use the mongoCollection - it is already connected
// and will not-reconnect for each request (bad!)
})