Node emitters with express - node.js

I want to make serial work on backend. I want to select geo_unit and do another work and write response to client.
I read http://nodejs.org/docs/v0.4.7/api/events.html#events and Error: Can't set headers after they are sent to the client
Everything works fine when I send the requests slowly e.g. 1 in each 3 seconds.
But it doesnot work when i send the requests fast. and i get the following error.
Do i define something globally?
Error: Can't set headers after they are sent.
at ServerResponse.OutgoingMessage.setHeader (http.js:691:11)
at ServerResponse.res.setHeader (E:\git\xxxxxx-nodejs\node_modules\express\node_modules\connect\lib\patch.js:63:22)
at ServerResponse.res.set.res.header (E:\git\xxxxxx-nodejs\node_modules\express\lib\response.js:527:10)
at ServerResponse.res.json (E:\git\xxxxxx-nodejs\node_modules\express\lib\response.js:194:36)
at EventEmitter. (E:\git\xxxxxx-nodejs\routes\api_scrapper.js:17:17)
at EventEmitter.emit (events.js:117:20)
at ScrapperAPI.externalLocationCallback (E:\git\xxxxxx-nodejs\routes\api_scrapper.js:27:20)
at Query._callback (E:\git\xxxxxx-nodejs\routes\api_scrapper.js:51:21)
at Query.Sequence.end (E:\git\xxxxxx-nodejs\node_modules\mysql\lib\protocol\sequences\Sequence.js:78:24)
at Query._handleFinalResultPacket (E:\git\xxxxxx-nodejs\node_modules\mysql\lib\protocol\sequences\Query.js:143:8)
I am using the following code :
var express = require('express'),
http = require('http'),
path = require('path');
var app = module.exports = express();
var server = require('http').createServer(app);
var io = require('socket.io').listen(server);
var request = require('request');
var cheerio = require('cheerio');
var mysql = require('mysql');
var pool = mysql.createPool({
host : 'localhost',
user : 'root',
database :'evevgez_development',
password : 'root',
// socketPath : '/var/run/mysqld/mysqld.sock'
});
var sahibinden = require('./routes/my_scrapper').newScrapperAPI(pool);
app.set('port', process.env.PORT || 3000);
app.set('views', __dirname + '/views');
app.set('view engine', 'jade');
app.use(express.logger('dev'));
app.use(express.bodyParser());
app.use(express.methodOverride());
app.use(express.static(path.join(__dirname, 'public')));
app.use(app.router);
app.use(express.errorHandler());
app.get('/node1', sahibinden.scrap);
server.listen(app.get('port'), function () {
console.log('Express server listening on port ' + app.get('port'));
});
and myModule is like that
var request = require('request');
var cheerio = require('cheerio');
var iconv = require('iconv-lite');
var EventEmitter = require('events').EventEmitter;
var emitter= new EventEmitter();
function ScrapperAPI(pool) {
this.scrap = function (req, res,next) {
var query = req.query;
console.log('req.params:'+req.query);
console.log('IL_ID:'+query.il_id);
emitter.emit('location',query);
emitter.on('success',function(rows){
res.json(rows);
});
};
emitter.on('location',function(query){
console.log('quey1:'+query);
getExternalLocation(query )
});
var getExternalLocation = function (query ) {
try {
pool.getConnection(function (err, connection) {
console.log('query:'+query);
connection.query("select * from external_geo_units where geo_unit_id = ? ",query.il_id, function (err, rows) {
console.log(err);
if (err) callback(err);
emitter.emit('success',rows);
connection.release();
});
});
} catch (e) {
console.log(e);
}
};
}
module.exports.newScrapperAPI = function (pool) {
return new ScrapperAPI(pool);
}

Using an EventEmitter in this way isn't the best solution for what you're doing, since you're just using it within a request to make a database call and return some rows -- something that could be more easily and simply done with just callbacks.
However if you want to keep the global EventEmitter, you have to switch the on in emitter.on('success' and emitter.on('location' to once instead, to ensure they only get called once per request.

Related

How to properly query Postgresql database using pg with Node.js?

I am attempting to query a Postgresql database using Node, Express, and the pg module. I am getting the error "TypeError: Cannot read property 'query' of undefined"
I have looked at the node-postgres module website for the correct syntax as well as various other websites. I know that the syntax used to enable pg changed after pg version 6.3.0.
const express = require("express");
const path = require("path");
const bodyparser = require("body-parser");
const consolidate = require("consolidate");
const dust = require("dustjs-helpers");
const pg = require("pg");
var pool = new pg.Pool();
let app = express();
let connect = "postgres://postgres:secret#localhost:5432/my_database";
app.engine("dust", consolidate.dust);
app.set("view engine", "dust");
app.set("views", __dirname + "/views");
app.use(express.static(path.join(__dirname, "public")));
app.use(bodyparser.json());
app.use(bodyparser.urlencoded({extended: false}));
app.get("/", function(request, response, next) {
pool.connect(function(err, client, done) {
if (err) {
console.log(`Not able to get a connection: ${err}.`);
response.status(400).send(err);
}
client.query("SELECT * FROM recipes", function(err, result) {
done();
});
pool.end();
});
});
app.listen(3000, () => {
console.log("Server running on port: 3000");
});
Going to http://server_ip:3000 should show a webpage instead node crashes with this error:
client.query("SELECT * FROM recipes", function(err, result) {
^
TypeError: Cannot read property 'query' of undefined
You need to pass your connection string to pool when you instantiate it.
var pool = new pg.Pool({
connectionString: 'postgres://postgres:secret#localhost:5432/my_database',
});

NodeJS procedure execution flow

I am new to Nodejs and aslo coming from a procedural language background so I have this need to know the execution flow of my code. I have a general question about the flow of Nodejs procedures. This is the scenario:
The code structure:
Appnamefolder
...standard node folders(.idea,css,fonts,etc)
...model
....database.js (connect to db and execute db queries)
...public
...routes
....users.js (GET and POST procedures....calls db queries via module.export)
...views
...app.js
...other js files
The question concerns the database. Since the DB connect is not in the app.js file but in a .js file in the model folder at what point is the DB connection made? and is a connection made every time a DB query is made?
What I hope to happen is that the DB connect is made one time and remain connected until the app is terminated. I tried putting the DB connect in app.js but I get an error when I attempt a DB query so I have to place the DB connect in the same file as the DB queries......Somehow this seems wrong to me. Can anyone explain how node handle this flow?....will be appreciative of any assistance.
EDIT: HERE IS A SAMPLE OF THE CODE
app.js
var express = require('express');
var path=require('path');
var bodyParser = require('body-parser');
var cookieParser=require('cookie-parser');
var expressSession=require('express-session');
var expejs = require('ejs');
var expressValidator = require('express-validator');
var flash = require('connect-flash');
var expressSession = require('express-session');
var expressLayouts=require("express-ejs-layouts") // add this requirement
var passport = require('passport');
var LocalStrategy = require('passport-local').Strategy;
var crypto = require("crypto");
var loaddealerTable=require('./loaddealerTable');
//var neo4j = require('neo4j-driver').v1;
// var neo4jdb = neo4j.driver("bolt://localhost:7474", neo4j.auth.basic("neo4j", "password"),
// {
// trust: "TRUST_ON_FIRST_USE",
// encrypted:true
// });
//***************Notifications Permission*******
var routes = require('./routes/index');
var users = require('./routes/users');
var csocket=require('./socketconnections');
var app=express();
var server=require('http').createServer(app);
sockets = require('./socketserver');
//rpaMessageWaiting = require('./getRPAmessages');
//var io=require('socket.io').listen(server);
// View Engine
app.set('views', path.join(__dirname,'views'));
app.set('view options', { layout:'layout.ejs' });
app.set('view engine','ejs');
//bodyParsers middleware
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: true }));
app.use(cookieParser());
app.use(expressLayouts);
//set up public folder
app.use(express.static(path.join(__dirname, '/public')));
// set express session with secret
app.use(expressSession({ secret: process.env.SESSION_SECRET || 'secret',
resave: true,
saveUninitialized: true
}));
// Passport initialization
app.use(passport.initialize());
app.use(passport.session());
//Express Validitor...validate inputs..taken from github middleware options
app.use(expressValidator({
errorFormatter: function(param, msg, value) {
var namespace = param.split('.')
, root = namespace.shift()
, formParam = root;
while(namespace.length) {
formParam += '[' + namespace.shift() + ']';
}
return {
param : formParam,
msg : msg,
value : value
};
}
}));
// connect flash middleware
app.use(flash());
// set global variables for flash messages
app.use(function (req, res, next)
{
res.locals.success_msg = req.flash('success_msg');
res.locals.error_msg = req.flash('error_msg');
res.locals.error = req.flash('error');
res.locals.user = req.user || null;
next();
});
// Middelware for route files
app.use('/', routes);
app.use('/users', users); //need to check routing
sockets.startSocketServer(server);
//load dealer table
console.log("load dealer table");
**loaddealerTable(); //First call on the DB**
//============socket io listening=======================
app.set('port',(process.env.PORT|| 3000));
server.listen(app.get('port'), function()
{
console.log('Server started on port '+app.get('port'));
// console.log('Server started on port .....');
// app.get('/index',function (req,res) {
// // body...
// res.render(__dirname+'/index');
});
loaddealertable.js
var loaddealerTable=function()
{
var memorytbl=require('./memorytables');
var User = require('./model/user');
var getHashKey=require('./gethashkey');
const hashMax=1000;
console.log("call get dealers from DB");
User.getallDealers(function(err,dealerFound,result)
{
if (dealerFound)
{
//
for (i=0; i< result.records.length; i++)
{
memorytbl.Dealer.email =result.records[i].get(0).properties.email;
memorytbl.Dealer.name =result.records[i].get(0).properties.name;
memorytbl.Dealer.telephone =result.records[i].get(0).properties.storenumber;
memorytbl.Dealer.creditcard =result.records[i].get(0).properties.creditcard;
memorytbl.Dealer.delivery =result.records[i].get(0).properties.delivery;
memorytbl.Dealer.location =result.records[i].get(0).properties.location;
memorytbl.Dealer.rating =result.records[i].get(0).properties.rating;
var hashIndex = getHashKey(memorytbl.Dealer.email ,hashMax);
memorytbl.DealersQ[hashIndex]=memorytbl.Dealer;
} //end of for i
} //end of if....
else
{
console.log("No dealers found....table is empty");
}
}) //end of loaddealers table db call
} //end of load dealers table function
module.exports=loaddealerTable;
user.js
var express = require('express');
var bcrypt = require('bcryptjs');
var router = express.Router();
var neo4j = require('neo4j-driver').v1;
var driver = neo4j.driver("bolt://localhost", neo4j.auth.basic("neo4j", "password"));
var session = driver.session();
//============Load Memory Tables=========================
router.getallDealers=function(callback) {
session
.run ("MATCH (user:Dealer) RETURN user")
.then (function(result)
{
if ( !result.records[0])
{
console.log("No Dealers Found");
session.close();
if (typeof callback==="function") {
return callback(null,false,result);
}
} // end of if not found
else
{
console.log("Dealer Found");
session.close();
if (typeof callback === "function")
{
return callback(null, true, result);
}
}
// or close session here??
}) //end of .then block
.catch(function(err)
{
console.log("DB call error: "+err);
}); //.then block
} //end of get dealers
For a start you have to understand that each file is a module.
Your app starts with running a single js file (module) like node app.js.
One module can load another module and another and so on.
Your folder structure does not have any effect on the order by itself. It all depends on your code and in what order do you load modules. While your sync code will run in the order you write it your async code will run in the future and you need to understand the event loop to understand what happens in your code.
From the small context you gave I guess you might tried to query your database before you connected to it. I don't see your code but it can happen regardless of where you connect (app.js file or another).

Start a proxy server with nodejs so that it serves requests preprended with "/wps_proxy/wps_proxy?url="

I want to start a proxy server with node.js so that it serves requests preprended with "/wps_proxy/wps_proxy?url=". I want it so I can use the wps-js library of north52 (check the installation tips) . I have already a server where I run my application.
What I did try until now is :
the server.js file
var express = require('express');
var bodyParser = require('body-parser');
var fs = require('fs');
var path = require("path");
var app = express();
app.use(express.static(__dirname + '/' + 'public'));
var urlencodedParser = bodyParser.urlencoded({ extended: false });
//****** this is my try ******************************
app.get('/wps_proxy/wps_proxy',function (req,res){
res.sendfile(__dirname + '/' + 'public/wps_proxy/wps-js/target/wps-js-0.1.2-SNAPSHOT/example.html');
if(req.query !== undefined){//because it enters sometimes without url
var http = require('http');
//Options to be used by request
var options = {
host:"geostatistics.demo.52north.org",//fixed given data
port:"80",
path:"/wps/WebProcessingService"
};
var callback = function(response){
var dat = "";
response.on("data",function(data){
dat+=data;
});
response.on("end", function(){
res.end(dat)
})
};
//Make the request
var req = http.request(options,callback);
req.end()
}
})
var ipaddress = process.env.OPENSHIFT_NODEJS_IP||'127.0.0.1';
var port = process.env.OPENSHIFT_NODEJS_PORT || 8080;
app.set('port', port);
app.listen(app.get('port'),ipaddress, function() {
console.log( 'Server started on port ' + app.get('port'))
})
//***************************************
but its not working.. I think that the data are not sent back correctly..
This is a live example of what I want to do.. http://geoprocessing.demo.52north.org/wps-js-0.1.1/
and this is a live example of my application (check the console for errors) http://gws-hydris.rhcloud.com/wps_proxy/wps_proxy
I did find my answer from this post How to create a simple http proxy in node.js? so the way i solve it was:
app.get('/wps_proxy/wps_proxy',function (req,res){
var queryData = url.parse(req.url, true).query;
if (queryData.url) {
request({
url: queryData.url
}).on('error', function(e) {
res.end(e);
}).pipe(res);
}
else {
res.end("no url found");
}
})

Can't receive response with 503 error (Service Unavailable) in aws-sdk(node.js)

app.js
var express = require('express');
var express_namespace = require('express-namespace');
var path = require('path');
var favicon = require('serve-favicon');
var http = require('http');
var https = require('https');
var e_logger = require('morgan');
var logger = require('./logger.js').getLogger('framework');
var cookieParser = require('cookie-parser');
var bodyParser = require('body-parser');
var routes = require('./routes/index');
var awsControl = require('./routes/awsControl')
var aws = require('aws-sdk');
var ec2 = new aws.EC2();
var app = express();
var server = http.createServer(app);
var env = process.env.NODE_ENV || 'development';
if ('development' == env) {
app.set('port', 80);
app.use(express.static(path.join(__dirname, 'public')));
app.set('views', path.join(__dirname, 'views'));
app.engine('html', require('ejs').renderFile);
app.set('view engine', 'html');
app.use(cookieParser());
app.use(e_logger('dev'));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.use(require('stylus').middleware(path.join(__dirname, 'public')));
app.use('/', routes);
}
// routes
app.namespace('/', function () {
app.get('describeInstances', awsControl.describeInstances);
});
server.listen(app.get('port'), function () {});
// catch 404 and forward to error handler
app.use(function (req, res, next) {
var err = new Error('Not Found');
err.status = 404
logger.error(err);
next(err);
});
awsControl.js
var aws = require('aws-sdk');
var util = require('util');
aws.config.update({
accessKeyId: "myKey",
secretAccessKey: "mySecretKey",
region: "ap-northeast-1"
});
console.log("config = " + util.inspect(aws.config));
var ec2 = new aws.EC2({ region: "ap-northeast-1" });
var app01 = 'aaaaaaaa';
var DB01 = 'bbbbbbbb';
exports.describeInstances = function (req, res) {
var params = {
Filters: [{
Name: 'vpc-id',
Values: ['vpc-cccccccc']
}],
DryRun: false
};
ec2.describeInstances(params, function (err, data) {
if (err) { // an error occurred
console.log(err, err.stack);
} else { // successful response
console.log(data);
}
});
}
control.js
var Control = function () {
this.initializePage();
};
Control.prototype = new Object();
Control.prototype = {
initializePage : function () {
$.ajax({
type: "GET",
url: "describeInstances",
success : function (data, status, jQxhr) {
console.log("### describeInstances success");
console.log(data);
},
error : function (jQxhr, status, error) {
console.log("### describeInstances error");
console.log(error);
console.log(jQxhr);
},
complete : function (jQxhr, status) {
console.log("### describeInstances complete");
console.log(jQxhr);
}
});
}
}
I programmed like above and the node web server is operating well.
awsControl.js is server-side javascript and control.js is client-side javascript.
When I connect to my web server, Control class is called first.
Here is the trouble. When I send request with startInstance (AWS-SDK API), it's working in server-side.
However, I can't receive response with 503 error(service unavailable).
On client-side, I always receive error callback with 503 error.
I don't know why I can't receive response.
I set security groups(EC2) and NACL(VPC) up so I don't think that it's firewall trouble.
Is there anybody can tell me how I can find the solution out?
I have done this.
ec2.describeInstances(params, function (err, data) {
if (err) { // an error occurred
console.log(err, err.stack);
} else { // successful response
console.log(data);
res.send(data); <-- this line
}
});
I added just a line I focus to awsControl.js file, then done.

MemoryStore in node.js

I am working on implementing social network application using node.js and the source that I use is Building Node Application with MongoDB and Backbone' by Mike Wilson.However, I cannot figure out the how to resolve the error of the MemoryStore --var MemoryStore = require('connect').session.MemoryStore;
Also, I tried to comment it but the error with middleware appear
var Session = require('connect').middleware.session.Session;
Can I get your help please ?
Thanks in advance
Here is the code of app.js
var express = require('express');
var http = require('http');
var app = express();
var nodemailer = require('nodemailer');
var MemoryStore = require('connect').session.MemoryStore;
var dbPath = 'mongodb://10.168.122.123:27017/socialnet';
var fs = require('fs');
var events = require('events');
// Create an http server
app.server = http.createServer(app);
// Create an event dispatcher
var eventDispatcher = new events.EventEmitter();
app.addEventListener = function (eventName, callback) {
eventDispatcher.on(eventName, callback);
};
app.removeEventListener = function (eventName, callback) {
eventDispatcher.removeListener(eventName, callback);
};
app.triggerEvent = function (eventName, eventOptions) {
eventDispatcher.emit(eventName, eventOptions);
};
// Create a session store
app.sessionStore = new MemoryStore();
// Import the data layer
var mongoose = require('mongoose');
var config = {
mail: require('./config/mail')
};
// Import the model
var models = {
Account: require('./models/Account')(app, config, mongoose, nodemailer)
}
// Configure the application
app.configure(function(){
app.sessionSecret = 'SocialNet secret key';
app.set('view engine', 'jade');
app.use(express.static(__dirname + '/public'));
app.use(express.limit('1mb'));
app.use(express.bodyParser());
app.use(express.cookieParser());
app.use(express.session({
secret: app.sessionSecret,
key: 'express.sid',
store: app.sessionStore
}));
mongoose.connect(dbPath, function onMongooseError(err) {
if (err) throw err;
});
});
// Import the routes located in ./routes
fs.readdirSync('routes').forEach(function(file) {
if (file[0] == '.') return;
var routeName = file.substr(0, file.indexOf('.'));
require('./routes/' + routeName)(app, models);
});
// -----
// GET /
// -----
app.get('/', function(req, res){
res.render("index.jade", {layout: false});
});
// -------------------
// POST /contacts/find
// -------------------
app.post('/contacts/find', function(req, res) {
var searchStr = req.param('searchStr', null);
if (null == searchStr) {
res.send(400);
return;
}
models.Account.findByString(searchStr, function onSearchDone(err, accounts) {
if (err || accounts.length == 0) {
res.send(404);
} else {
// TODO: Check if these accounts were already contacts
// if so, mark them as isContact so the views/Contact
// knows not to add a addButton
res.send(accounts);
}
});
});
// Let the server listen to 8000 (instead of the app)
app.server.listen(8000);
console.log('SocialNet listening to port 8000');
Your problem:
app.use(app.router)
, mounts your routes in that position in the call chain. You have it before your session middleware, so there is no req.session yet. When you leave it out, your routes will be positioned whenever you do your first app.get (or app.post and so on). If you still wish to control where your routes should be,
you can just:
move app.use(app.router) below the session middleware.

Resources