I'm beginning with NodeJs and I'm facing a problem I'm sure a lot of people may already have solved.
I have a basic NodeJs web server that serves files if a file is found, or 404:
var http = require('http'),
url = require('url'),
fs = require('fs'),
mongoose = require('mongoose'),
fileSystem = require('fs'),
path = require('path'),
util = require('util'),
EventEmitter = require('events').EventEmitter;
var mimeTypes = {
"html": "text/html",
"jpeg": "image/jpeg",
"jpg": "image/jpeg",
"png": "image/png",
"js": "text/javascript",
"css": "text/css"};
var server;
var Schema = mongoose.Schema;
mongoose.connect('mongodb://localhost/test');
var db = mongoose.connection;
var personneSchema = new Schema({
nom: String,
prenom: String
});
var Personne = db.model('Personne', personneSchema);
var personneAdresseSchema = new Schema({
idPersonne: String,
idAdresse: String
});
var PersonneAdresse = db.model('PersonneAdresse', personneAdresseSchema);
var adresseSchema = new Schema({
ligne1: String,
ligne2: String,
codePostal: String,
ville: String
});
var Adresse = db.model('Adresse', adresseSchema);
db.on('error', console.error.bind(console, 'connection error:'));
db.once('open', function () {
server = http.createServer(function (request, response) {
if (request.url=='/persons') {
console.log('> Persons request');
/* IMPLEMENTATION PROBLEM HERE */
var retour='[]';
retour=retour.substr(0, retour.length-1)+']';
response.writeHead(200, {
'Cache-Control': 'no-cache, must-revalidate',
'Expires': 'Mon, 26 Jul 1997 05:00:00 GMT',
'Content-type': 'application/json'
});
response.end(retour);
console.log('> end of main function');
return;
}
var uri = url.parse(request.url).pathname;
var filename = path.join(process.cwd(), uri);
console.log("> " + filename);
fs.exists(filename, function(exists) {
if ((!exists) || (fs.lstatSync(filename).isDirectory())) {
console.log(">> fichier inexistant : " + filename);
response.writeHead(200, {'Content-Type': 'text/plain'});
response.write('404 Not Found\n');
response.end();
// Stopper tout traitement :
return;
}
var mimeType = mimeTypes[path.extname(filename).split(".")[1]];
response.writeHead(200, {'Content-Type':mimeType});
var fileStream = fs.createReadStream(filename);
fileStream.pipe(response);
});
});
// Listen on port 8000, IP defaults to 127.0.0.1
server.listen(8000);
// Put a friendly message on the terminal
console.log("Server running at http://127.0.0.1:8000/");
});
The problem is: the Webserver needs to get a response ready to be sent before the end of the "main" function. But if you try to do a mongoose call, it's not synchronous. If you put the following code in the main function:
Personne.find({}).select('nom prenom').exec(function (err, p) {
console.log('> Persons request finished');
});
The log shows something like:
> Persons request
> end of main function
> Persons request finished
So the main function that needs "response" to be filled can't have it filled with Personne.find({}) because Personne.find({}) finished after. What is the way to handle this? I cant find a very simple, self explaining example on the Web (nodejs + mongoose always gives me solutions with either nodejs alone, mongoose alone, or using full Web frameworks, whereas I just need a simple working example).
Any code that you want to be executed after the database call has finished has to be in the callback function for the database call. So you can't just put the database call in your main function -- you have to move all of the code from your main function that needs to be executed after the database call into the callback function instead.
If you want the server to start after your mongo query finishes, you can either use a callback:
db.once('open', function () {
server = http.createServer(function (request, response) {
Personne.find({}).select('nom prenom').exec(function (err, persons) {
// Log the result of the query
console.log(persons)
// After the query is executed, you can use the results anywhere else inside your logic
if (request.url=='/persons') {
console.log('> Persons request');
/* IMPLEMENTATION PROBLEM HERE */
var retour='[]';
retour=retour.substr(0, retour.length-1)+']';
response.writeHead(200, {
'Cache-Control': 'no-cache, must-revalidate',
'Expires': 'Mon, 26 Jul 1997 05:00:00 GMT',
'Content-type': 'application/json'
});
response.end(retour);
console.log('> end of main function');
return;
}
var uri = url.parse(request.url).pathname;
var filename = path.join(process.cwd(), uri);
console.log("> " + filename);
fs.exists(filename, function(exists) {
if ((!exists) || (fs.lstatSync(filename).isDirectory())) {
console.log(">> fichier inexistant : " + filename);
response.writeHead(200, {'Content-Type': 'text/plain'});
response.write('404 Not Found\n');
response.end();
// Stopper tout traitement :
return;
}
var mimeType = mimeTypes[path.extname(filename).split(".")[1]];
response.writeHead(200, {'Content-Type':mimeType});
var fileStream = fs.createReadStream(filename);
fileStream.pipe(response);
});
});
});
// Listen on port 8000, IP defaults to 127.0.0.1
server.listen(8000);
// Put a friendly message on the terminal
console.log("Server running at http://127.0.0.1:8000/");
});
Related
I am creating a Rest API in Node.js and Express. It connects with remote HANA database & execute one query. Now I want to stream HTTP response so that I can send it to browser into chunks, rather than sending it completely since it's a very large data.
I tried something which is giving me no output. I don't know the reason. If I send the complete response to browser using response.send (data), it's working fine. But streaming is now working.
I have added code snippet below.
const express = require("express");
const APP = express();
const HANA_DB = require('hdb');
const BODY_PARSER = require("body-parser");
start();
function start() {
startServer();
initializeExpress();
APP.get("/data", function(request, response) {
var connection = HANA_DB.createClient({
host : "hostname",
port : "port",
user : "username",
password : "password"
});
connection.on('error', function (error) {
console.log("Error in database connection...");
});
connection.connect(function (error) {
if (error) {
console.log("Error in database connection...");
return;
}
var query = "SELECT * FROM TableName";
connection.exec(query, function(error, result) {
if(error) {
response.send("Getting error while fetching result...");
return;
}
//response.send(data);
var datalength = 0;
request.on('data', function(chunk) {
datalength += chunk.length;
console.log("DATA EVENT: " + datalength);
response.send(datalength);
})
.on('end', function() {
console.log("END EVENT: " + datalength);
response.send(datalength);
});
});
});
});
};
function initializeExpress() {
APP.all('/*', function(request, response, next) {
response.header("Access-Control-Allow-Origin", "*");
response.header("Access-Control-Allow-Headers", "X-Requested-With");
response.header('Access-Control-Allow-Methods', 'PUT, GET, POST, DELETE, OPTIONS');
response.header('Access-Control-Allow-Headers', 'Content-Type');
next();
});
APP.use(BODY_PARSER.json());
APP.use(BODY_PARSER.urlencoded({ extended: true }));
};
function startServer(config) {
var server = APP.listen("8081", function(error) {
if(error) {
console.log("Unable to connect to 127.0.0.1:8081");
return;
}
console.log("Server is listening at - 127.0.0.1:8081");
});
};
I think the logic you are using is wrong for the streaming the data.
Use res.write instead of res.send and you also have to read streaming data from your database instead of one time connection.exec
I am giving you an example code where you will get some idea about streaming data in Expressjs.
var http = require( 'http' );
http.createServer( function ( req, res ) {
res.writeHead( 200, {
'Content-Type': 'text/plain; charset=utf-8',
'Transfer-Encoding': 'chunked',
'X-Content-Type-Options': 'nosniff'
} );
res.write( 'Beginning\n' );
var count = 10;
var io = setInterval( function () {
res.write( 'Doing ' + count.toString() + '\n' );
count--;
if ( count === 0 ) {
res.end( 'Finished\n' );
clearInterval( io );
}
}, 1000 );
} ).listen( 8000 );
The problem is here request.on('data',. request refers to the browser request.
You cannot use streaming with .exec(), because the callback function of exec is called with the rows as parameters.
To use streaming, use the .execute() method, which passes a resultset to the callback function.
I never used hdb, so I cannot give the code to use.
I have a problem with streaming video files from a server to another.
I wrote this script
var request = require("request"),
express = require("express");
var app = express();
app.get("/cast", function(req, res) {
var url = req.query.video;
res.writeHead(200, {
'Content-Type': 'video/mp4'
});
request({
url: url,
headers: {
Referer: "http://example.com/1706398/" + url
}
})
.on('response', function(response) {
response.on('data', function(data) {
console.log("data chunk received: " + data.length)
});
response.on('end', function(data) {
console.log('Video completed');
});
})
.pipe(res);
});
app.listen(8080);
But video response sometimes works sometimes is corrupted, instead if request's data is written in a writeable buffer and saved as video file it works with any url.
I cannot found any error or problem in my code, here some urls :
Here some url that I tryed:
https://gist.github.com/FrancisCan/f2bb86f8ff73b45fa192
Thanks :)
Remove the writeHead 200, when you are streaming, you should return http 206 results back (partial content), and not http200. I have the same scenario as you (streaming a video file from a blob container in the cloud to an angular application), there is no need for the http200 response.
Update: adding some code on how I do it:
AzureStorageHelper.prototype.streamBlob = function streamBlob(req, res, blob, params) {
if(!params){
params.container = container;
}
blob_service.getBlobProperties(params.container, blob, function (error, result, response) {
if(!result) return res.status(404).end();
if(error) return res.status(500).end();
var blobLength = result.contentLength;
var range = req.headers.range ? req.headers.range : "bytes=0-";
var positions = range.replace(/bytes=/, "").split("-");
var start = parseInt(positions[0], 10);
var end = positions[1] ? parseInt(positions[1], 10) : blobLength - 1;
var chunksize = (end - start) + 1;
var options = {
rangeStart: start,
rangeEnd: end,
}
//this is what's interesting for your scenario. I used to set this up myself but it's is now handled by the Azure Storage NodejsSDK
/*res.writeHead(206, {
'Accept-Ranges': 'bytes',
'Content-Range': "bytes " + start + "-" + end + "/" + blobLength,
'Content-Type': result.contentType,
'Content-Length': chunksize,
'Content-MD5': result.contentMD5,
});*/
var options = {
rangeStart: start,
rangeEnd: end,
}
//this is an API from the Azure Storage nodejsSDK I use. You might want to check the source of this method in github to see how this lib deals with the http206 responses and the piping
blob_service.getBlobToStream(params.container, blob, res, options, function (error, result, response) {
if (error) {
return res.status(500).end();
}
});
});
I am writing a program that will stream a video file that is currently being downloaded onto the drive. The problem I am having seems to be getting the browser to actually play the video. the script listens for file changes, and then streams the rest, but the browser doesnt do anything besides display a blank Video page.
var fs = require('fs'),
http = require('http'),
filename = '/home/qrpike/Videos/topgears.mp4';
http.createServer(function (req, res) {
console.log(req.url);
if( req.url == '/video.mp4'){
res.writeHead(200,{
'Content-Type' : 'video/mp4',
'Cache-Control' : 'public',
'Connection' : 'keep-alive',
'Content-Disposition' : 'inline; filename=topgears.mp4;',
'Content-Transfer-Encoding' : 'binary',
'Transfer-Encoding' : 'chunked'
});
fs.open(filename, 'r', function(err, fd) {
if (err) throw new Error('Could not open file');
var position = 0;
fs.stat(filename, read);
fs.watchFile(filename, read.bind(null, null));
function read(err, stat) {
var delta = stat.size - position;
if (delta <= 0) return;
fs.read(fd, new Buffer(delta), 0, delta, position, function(err, bytes, buffer) {
console.log("err", err, "bytes", bytes, "position",position,"delta",delta);
res.write(buffer.toString('binary'));
});
position = stat.size;
}
});
}
}).listen(1337);
console.log('Server running at http://127.0.0.1:1337/');
So this answer depends on growing-file, which in theory does what you want. My concern is that the project hasn't had a commit in two years so I have no idea if it still works. That being said, this worked for me locally (though I didn't test piping to the video file):
var fs = require('fs');
var http = require('http');
var GrowingFile = require('growing-file');
var FILENAME = '/home/dave/Desktop/video/video.ogg';
var server = http.createServer(function(req, res) {
var file;
if (req.url === '/video.ogg') {
res.writeHead(200, {
'Content-Type': 'video/ogg'
});
file = GrowingFile.open(FILENAME);
file.pipe(res);
} else {
res.statusCode = 404;
res.end('Not Found');
}
});
server.listen(1337);
So i have next test code, which i found here
// Include http module,
var http = require('http'),
// And mysql module you've just installed.
mysql = require("mysql");
// Create the connection.
// Data is default to new mysql installation and should be changed according to your configuration.
var connection = mysql.createConnection({
user: "root",
password: "pass",
database: "db_name"
});
// Create the http server.
http.createServer(function (request, response) {
// Attach listener on end event.
request.on('end', function () {
// Query the database.
connection.query('SELECT * FROM your_table;', function (error, rows, fields) {
response.writeHead(200, {
'Content-Type': 'x-application/json'
});
// Send data as JSON string.
// Rows variable holds the result of the query.
response.end(JSON.stringify(rows));
});
});
// Listen on the 8080 port.
}).listen(8080);
i download mysql module, install mysql of course, and run next script, and get no results. Can you give me advice what i do wrong?
When i trying to load
http://localhost:8080/
browser trying to load page several minutes, and also no result.
Thanks.
update
I think you forgot
connection.connect()
I'm using these for while now and it works fine for me.
// Include http module,
var http = require('http'),
// And mysql module you've just installed.
mysql = require("mysql");
// Create the connection.
// Data is default to new mysql installation and should be changed according to your configuration.
var connection = mysql.createConnection({
host: "localhost",
user: "root",
password: "pass",
database: "db_name"
});
connection.connect();
// Create the http server.
http.createServer(function (request, response) {
// Attach listener on end event.
request.on('end', function () {
// Query the database.
connection.query('SELECT * FROM your_table;', function (error, rows, fields) {
response.writeHead(200, {
'Content-Type': 'x-application/json'
});
// Send data as JSON string.
// Rows variable holds the result of the query.
response.end(JSON.stringify(rows));
});
});
// Listen on the 8080 port.
}).listen(8080);
Few previous answers that should help you
Express.JS + Node-Mysql and 1 connection per http request
How to query mysql before starting http server in nodejs
get simple info from db into a node.js http server response
to use pools in mysql
Node.js MySQL Needing Persistent Connection
A more little complex solution separate your code into files and put the db code either in db.js or in reponse handler or router.
My code is using pools.
Save to the five files
then run npm install
e.g
(please excuse the terrible formatting)
// requestHandlers.js
// request handler with mysql code
var mysql = require("mysql");
var Query1 = "SELECT * from Tablexyz WHERE xx = 'doh'";
var body = '<html>'+
'<head>'+
'<meta http-equiv="Content-Type" content="text/html; '+
'charset=UTF-8" />'+
'</head>'+
'<body>'+
'<h1> Query </h1>';
var body1 = '</body>'+
'</html>';
var pool = mysql.createPool({
host : 'host',
user : 'dbuser',
password : "pword",
database : 'database',
connectionLimit: 10,
queueLimit: 10
});
function qquery(callback){
console.log('in qquery');
pool.getConnection(function(err, connection) {
pool.query( Query1 , function(err, rows ,fields) {
if (err) {
return callback(new Error("An error has occured" + err));
}
if (rows.length > 0){
callback( rows);
connection.release();
}
});
});
}
function start(response) {
console.log("Request handler 'start' was called.");
qquery( function (rows , err) {
if (err) {
console.log(err);
}
else{
console.log(rows);
response.writeHead(200, {"Content-Type": "text/html"});
response.write(body);
response.write( 'query: ' + JSON.stringify( rows ) );
response.write(body1);
response.end();
}
});
}
exports.start = start;
///////////////////////////////////- new file
//server.js
var http = require("http");
var url = require("url");
function start(route, handle) {
function onRequest(request, response) {
var pathname = url.parse(request.url).pathname;
console.log("Request for " + pathname + " received.");
route(handle, pathname, response);
}
http.createServer(onRequest).listen(8888);
console.log("Server has started.");
}
exports.start = start;
///////////////////////////////////-new file
// router.js
function route(handle, pathname, response) {
console.log("About to route a request for " + pathname);
if (typeof handle[pathname] === 'function') {
handle[pathname](response);
} else {
console.log("No request handler found for " + pathname);
response.writeHead(404, {"Content-Type": "text/plain"});
response.write("404 Not found");
response.end();
}
}
exports.route = route;
///////////////////////////////////- new file
//index.js
var server = require("./server");
var router = require("./router");
var requestHandlers = require("./requestHandlers");
var handle = {}
handle["/"] = requestHandlers.start;
handle["/start"] = requestHandlers.start;
handle["/upload"] = requestHandlers.upload;
server.start(router.route, handle);
///////////////////////////////////- new file
//package.json
{
"name": "application-name",
"version": "0.0.1",
"private": true,
"scripts": {
"start": "node app.js"
},
"dependencies":
{
"express": "3.4.0",
"jade": "*",
"stylus": "*",
"mysql": "*"
}
}
#Ishikawa Yoshi
use a callback
// Include http module
var http = require('http')
// And mysql module you've just installed.
mysql = require("mysql");
var Query1 = "SELECT * from Tablexyz WHERE xx = 'doh'";
var connection = mysql.createConnection({
host : 'host',
user : 'dbuser',
password : "pword",
database : 'database',
});
connection.connect();
function getData(callback){
console.log('in getData');
connection.query( Query1 , function(err, rows ,fields) {
if (err) {
return callback(new Error("An error has occured" + err));
}
if (rows.length > 0){
callback( rows);
}
});
}
// Create the http server.
http.createServer(function (request, response) {
// Attach listener on end event.
// Query the database.
getData(function (rows , err) {
if (err) {
console.log(err);
}
else{
response.writeHead(200, {"Content-Type": "text/html"});
response.write( 'query: ' + JSON.stringify( rows ) );
response.end();
}
});
// Listen on the 4001 port.
}).listen(4001);
For some reason, this code keeps failing :
var http = require('http');
var url = require ('url');
var jade = require('jade');
var fs = require('fs');
http.createServer(function(req, res) {
// Request handlers
var path = url.parse(req.url, true).pathname;
var dot = path.lastIndexOf('.');
var extension = path.substr(dot + 1);
// Response handlers
switch(extension) {
case 'css' :
res.writeHeader(200, {"Content-Type": "text/css"});
fs.readFile(path.substr(1, dot - 1) + '.css', function (err, css) {
if(err) {
res.write(err, 'utf8')
};
res.write(css, 'utf8')
});
break;
case 'js' :
res.writeHeader(200, {"Content-Type": "text/javascript"});
fs.readFile(path.substr(1, dot - 1) + '.js', function (err, js) {
if(err) {
res.write(err, 'utf8')
};
res.write(js, 'utf8')
});
break;
default :
res.writeHeader(200, {"Content-Type": "text/html"});
jade.renderFile(path.substr(1, path.length) + '.jade', { title: 'test' }, function(err, html) {
if(err) {
res.write(err, 'utf8')
};
res.write(html, 'utf8');
});
break;
}
res.end();
}).listen(8080);
I don't get any errors, it's just that the responses seem to be empty.
There is a header but no css or javascript file.
The way I link them in Jade is the following :
link(rel='stylesheet', href='/css/bootstrap.min.css')
link(rel='stylesheet', href='/css/style.css')
Thank you in advance !
You've called res.end() too soon. Instead, put it after the last res.write() in each of your fs.readFile handlers.