I am working with a very small piece of nodeJS code thats using express. The app receives a payload as a POST and sends it to another method which submits it to kafka. Once this happens, I need to return a response back to express so that it can close the connection based on the status (return a 200 response for example).
// producer.js
// Defined variables
var kafka = require('kafka-node'),
Producer = kafka.Producer;
module.exports = {
sendToProducer: function (payload) {
// Define our kafka settings
var client = new kafka.KafkaClient(),
producer = new Producer(client);
// On producer ready
producer.on('ready', function () {
// Send our payload to our topic
producer.send(payload, function (err, data) {
console.log(data);
// Return our response back to `app.js` so express can handle it.
});
});
// On producer error
producer.on('error', function (err) {
console.log('Producer Error', err)
})
}
};
// app.js
app.post('/payload', function (req, res) {
// Define our route
const payload = req.body;
const route = payload.route;
// Check to see if we have a route
if(!route){
logError('no route');
}
let mappedData = [{ topic: 'test', messages: 'hi... ', partition: 0 }]
// Based on the route
switch(route){
// Receive data
case 'ingest':
producer.sendToProducer(mappedData);
// How can I tell express the status of the above method so that we can close the request? (ex 200 response for example?)
break;
// Handle undefined routes
default:
break;
}
});
In my app.js case statement, how can I wait for a response/callback from producer.sendToProducer(mappedData); so that I can handle the express status appropriately ?
Make sure your sendToProducer() is a promise / async function :
app.post('/payload', async function (req, res) {
// Define our route
const payload = req.body;
const route = payload.route;
// Check to see if we have a route
if(!route){
logError('no route');
}
let mappedData = [{ topic: 'test', messages: 'hi... ', partition: 0 }]
// Based on the route
switch(route){
// Receive data
case 'ingest':
await producer.sendToProducer(mappedData);
// How can I tell express the status of the above method so that we can close the request? (ex 200 response for example?)
break;
// Handle undefined routes
default:
break;
}
});
Related
Running into the well documented "Error [ERR_HTTP_HEADERS_SENT]: Cannot set headers after they are sent to the client" in my node app. I have reviewed existing pages dedicated to this issue and tried their remedies - mainly making sure to "return" all other responses; however the error still persists. Code below.
You'll notice I have a bunch of different paths calling unique python functions via subprocess. Interestingly the http error only shows up for certain paths - typically those with very large output. In all cases the response arrives and renders on the client side just fine, but with the larger responses I get the http error and my app shuts down. I can think of a few potential non-middleware causes for this - Potentially res.send() behaves differently as the responses get large? Or perhaps it's subprocess.stdout causing issues with the larger responses? Alternatively the longer response time could be causing the browser to resend the request before the response is delivered?... Would like to rule these out before going deeper into potential middleware issues. Thanks
router.get('/element/chart', ensureAuthenticated, (req,res) => {
const path = require('path')
const {spawn} = require('child_process')
console.log(current_data_page_id)
console.log(typeof current_data_page_id)
let runScript;
runScript = (current_data_page_id) => {
switch(current_data_page_id) {
case "100":
return spawn('python', ["-u", path.join(process.cwd(),'/python/python_func_1.py')]);
break;
case "101":
return spawn('python', ["-u", path.join(process.cwd(),'/python/python_func_2.py')]);
break;
case "102":
return spawn('python', ["-u", path.join(process.cwd(),'/python/python_func_3.py')]);
break;
case "103":
return spawn('python', ["-u", path.join(process.cwd(),'/python/python_func_4.py')]);
break;
case "104":
return spawn('python', ["-u", path.join(process.cwd(),'/python/python_func_5.py')]);
break;
case "200":
return spawn('python', ["-u", path.join(process.cwd(),'/python/python_func_6.py')]);
break;
case "201":
return spawn('python', ["-u", path.join(process.cwd(),'/python/python_func_7.py')]);
break;
case "202":
return spawn('python', ["-u", path.join(process.cwd(),'/python/python_func_8.py')]);
break;
default:
console.log("Data page ID does not match current options")
};
};
const subprocess = runScript(current_data_page_id)
// print output of script
subprocess.stdout.on('data', (data) => {
var dataToSend = data.toString();
console.log(dataToSend);
res.send(dataToSend);
res.end('end')
return;
});
subprocess.stderr.on('data', (data) => {
console.log(`error:${data}`);
return;
});
subprocess.stderr.on('close', () => {
console.log("Closed");
return;
});
});
// ensureAuthenticated middleware
module.exports = {
ensureAuthenticated: function(req, res, next) {
if(req.isAuthenticated()) {
return next();
}
req.flash('error_msg', 'Please log in to view this resource');
res.sendFile(process.cwd() + '/views/login.html');
return;
}
}
Console output below. Stack trace points to the "res.send(dataToSend);" line.
_http_outgoing.js:536
throw new ERR_HTTP_HEADERS_SENT('set');
^
Error [ERR_HTTP_HEADERS_SENT]: Cannot set headers after they are sent to the client
The reason of the error is you call "end" request to many time, a http request just need finish only once time.
You call res.send(dataToSend); or res.end('end') this mean you finish the request 2 times.
When you remove res.end('end'); line, the error still appear, because res.send(dataToSend); has been called more than once time, because I think your "python" command more than 1 line of data, then subprocess.stdout.on('data' has been call many time.
The idea for that case is just end the request when the "command" finish (done), you can catch "done" event with close event of spawn process (not subprocess.stdout).
With the data (output of the command) we have 2 ways:
Append output to a string variable and send it to client when the command done
var data = ''; // init data store
subprocess.stdout.on('data', (data) => {
var dataToSend = data.toString();
console.log(dataToSend);
data += dataToSend + '\n'; // append data with new line char :)
// res.send(dataToSend);
// res.end('end')
// return;
});
// subscribe to close event
subprocess.on('close', (code) => {
if (code !== 0) {
console.log(`grep process exited with code ${code}`);
}
res.send(data); // send data to client and finish the request
});
Use express api to write response to res object (I like this way)
subprocess.stdout.on('data', (data) => {
var dataToSend = data.toString();
console.log(dataToSend);
res.write(dataToSend); // write data to response stream
// res.send(dataToSend);
// res.end('end')
// return;
});
// subscribe to close event
subprocess.on('close', (code) => {
if (code !== 0) {
console.log(`grep process exited with code ${code}`);
}
res.end(); // finish the request, `end` not `send`
});
I have a NODE.JS api using expressjs that connects to an SQL Server, and I want to use it in an angular project. I make use two files, a route file and a controllers file. My route file is as follows:
module.exports = (app) => {
const UsrContrllr = require('../Controllers/users.controllers');
//1. GET ALL USERS
app.get('/api/users', UsrContrllr.func1);
//2. POST NEW USER
app.post('/api/user/new', UsrContrllr.func2);
};
And my controllers file is given below:
const mssql = require('mssql');
exports.func1 = (req, res) =>
{
// Validate request
console.log(`Fetching RESPONSE`);
// create Request object
var request = new mssql.Request();
// query to the database and get the records
const queryStr = `SELECT * FROM USERS`;
request.query(queryStr, function (err, recordset) {
if (err) console.log(err)
else {
if (recordset.recordset.toString() === '') {
res.send('Oops!!! Required data not found...');
}
else {
// send records as a response
res.send(recordset);
}
};
});
};
exports.func2 = (req, res) =>
{
// Validate request
console.log(`INSERTING RECORD ${req}`);
// create Request object
var request = new mssql.Request();
// query to the database and get the records
const queryStr = `INSERT INTO GDUSERS (USERCODE, PASSWORD, LANGUAGE, USERCLASS, FIRSTNAME, LASTNAME, CONTACTNO) VALUES ('${req.body.usercode}', '${req.body.password}', 'EN', '0', '${req.body.firstname}', '${req.body.lastname}', '${req.body.contactno}');`;
request.query(queryStr, function (err, recordset) {
if (err) console.log(err)
else {
if (recordset.recordset.toString() == '') {
res.send('Oops!!! Required data not found...');
}
else {
// Send records as response
res.send(recordset);
}
};
});
};
The GET request works well, but when I try to run the POST request directly from the angular application, I get an error stating
Cannot GET URL/api/user/new
The angular code in my angular project is:
signup() {
let headers = new Headers({ 'Content-Type': 'application/json' });
let options = new RequestOptions({ headers: headers });
console.log(this.user); //User details come from a form
this.http.post(“URL", this.user, options)
.subscribe(
(err) => {
if(err) console.log(err);
console.log("Success");
});
}
I’m not sure whether the angular code I’m using, is right or not, and I don’t know where I’m going wrong. How does one exactly send a http POST request from an Angular project?
this i the way i handled my user signup with http.post calls. my approach is slightly different when signing up user because i am using a promise instead of observable (which i normally use for my servicecalls). but i will show you both ways.
createUser(user: User): Promise < string > {
const promise = new Promise < string > ((resolve, reject) => {
const userForPost = this.createUserForPost(user);
this.http.post(environment.backendUrl + '/api/user/signup', userForPost, this.config).toPromise < HttpConfig > ()
.then(createdUser => {
}).catch(error => {
console.log(error);
});
});
return promise;
}
here another example with an observable
createForumPost(forumPost: ForumPost) {
this.http.post < { message: string, forumPostId: string } > (environment.backendUrl + '/api/forumPosts', forumPost).subscribe((responseData) => {
const id = responseData.forumPostId;
forumPost.id = id;
});
}
i defined my URL somewhere else and then just use the environment.backedUrl + 'path' to define my path (the same as the path in your backend controller)
this is one of my first answers here on SO. i am sry if it is a bit messy
i hope i was able to help with my examples :)
I'm currently having problems figuring out how to capture my MQTT message event back to my REST API body which is written in NodeJS. My current setup is App -> NodeJS REST API -> MQTT broker inside RPi 3.
This is my MQTTHandler.js class where I have put all my reusable MQTT functions
const mqtt = require('mqtt')
class MQTTHandler {
constructor (host) {
this.client = null
this.host = host
}
connect () {
this.client = mqtt.connect(this.host)
this.client.on('error', function (err) {
console.log(err)
this.client.end()
})
this.client.on('connect', function () {
console.log('MQTT client connected...')
})
// I need this to send message back to app.js
this.client.on('message', function (topic, message) {
if (!message.toString()) message = 'null'
console.log(JSON.parse(message.toString()))
})
this.client.on('close', function () {
console.log('MQTT client disconnected...')
})
}
subscribeTopic (topic) {
this.client.subscribe(topic)
}
unsubscribeTopic (topic) {
this.client.unsubscribe(topic)
}
sendMessage (topic, message) {
this.client.publish(topic, message)
}
}
module.exports = MQTTHandler
And below is a short snippet of my app.js
const MQTTHandler = require('./mqtt.handler')
...
var mqttClient = new MQTTHandler('mqtt://127.0.0.1')
mqttClient.connect()
app.get('/hello', function (req, res) {
mqttClient.subscribeTopic('topic')
mqttClient.sendMessage('topic', 'hello world')
// I need to return the MQTT message event here
// res.json(<mqtt message here>)
res.end()
})
I have already tried using NodeJS' event emitter but it doesn't seem to work. Any help or suggestions would be much appreciated, thank you!
You are trying to mix a synchronous protocol (HTTP) with and asynchronous protocol (MQTT). These 2 paradigm don't easily mix.
When you publish an MQTT message you have no idea how many clients may be subscribed to that topic, it could be zero, it could be many. There is also no guarantee that any of them will send a reply so you need to include a timeout. (You also need to include a request id in the payload so you can coordinate any response with the request as you can't say anything about what order responses may come in.)
Your example code is only using 1 topic, this is very bad as you will end up needing to filter out request messages from response messages. Better to use 2 different topics (MQTT v5 even has a msg header to specify the topic the response should be sent on).
Having said all that it is possible to build something that will work (I will use request and reply topics.
var inflightRequests = {};
// interval to clear out requests waiting for a response
// after 3 seconds
var timer = setInterval(function() {
var now = new Date.now();
var keys = Object.keys(inflightRequests);
for (var key in keys) {
var waiting = inflightRequests[keys[key]];
var diff = now = waiting.timestamp;
// 3 second timeout
if (diff > 3000) {
waiting.resp.status(408).send({});
delete(inflightRequests[keys[key]]);
}
}
},500);
// on message handler to reply to the HTTP request
client.on('message', function(topic, msg){
if (topic.equals('reply')) {
var payload = JSON.parse(msg);
var waiting = inflightRequest[payload.requestId];
if (waiting) {
waiting.res.send(payload.body);
delete(inflightRequests[payload.requestId]);
} else {
// response arrived too late
}
}
});
// HTTP route handler.
app.get('/hello', function(req, res) {
//using timestamp as request Id as don't have anything better in this example.
var reqId = Date.now();
var waiting = {
timeStamp: reqId,
res: res
}
inflightRequests[reqId] = waiting;
var message = {
requestId: reqId,
payload: 'hello world'
}
client.publish('request',JSON.stringify(message));
});
It may be a wrong way to use bull queue but here is what I want to do:
var Promise = require('bluebird');
var redis = require('redis');
var Queue = require('bull');
var redisClient = redis.createClient(6379);
var pdfQueue = new Queue('msg');
function check(resolve, reject,i) {
console.log('check called');
//Or if it is in Router then I want to send request, response in queue so that I can call them in on complete function
pdfQueue.add('msg',{'msg':'Hello', 'resolve':resolve,'reject':reject}).then(job=>{
console.log('added to the pdf')
});
}
pdfQueue.on('completed', function (job, result) {
//Here I want to call request.send('some msg');
//and resolve('complete');
resolve('final callback');
})
pdfQueue.process('msg',100,function (job,done) {
console.log('process');
done(null,'job done ')
})
function check2 () {
return new Promise(function(resolve, reject){
check(resolve,reject);
})
}
check2().then(data => {
console.log('got the value ', data)
});
In my real project I want to implement queue where I will be sending pdf to the user. Like res.download(pdf path); but this function should be in pdf.on('completed',()=>{ res.download(pdf path); }); or in resolve(pdfPath) but I am not able to find anyway to send pdf to the user using queue because I don't know how to call response or resolve in other functions by using queue jobs.
Please help me. Thanks you
I have an http server with a handleRequest callback that runs another script in vm.runInNewContext for each request. The script that runs inside vm.runInNewContext makes some asynchronous http post requests and writes the server response only after getting the responses from the posts.
As a result, the code of handleRequest callback ends before the server response is written.
Is it safe? or is there a way to avoid this situation?
Here is some code:
var server = http.createServer(handleRequest);
server.listen(8080);
var handleRequest = function (request, response) {
// get request data...
var context = {
ServerRequest : request,
ServerResponse : response
};
var stringScript = // a string with the script that posts data
var script = vm.createScript(stringScript);
script.runInNewContext({ context: context });
}
the script string does this:
var request = require('request');
var options = {....}
var req = request.get(options);
req.on('response', function (res) {
var chunks = [];
res.on('data', function(chunk) {
chunks.push(chunk);
});
res.on('end', function() {
var buffer = Buffer.concat(chunks);
var encoding = res.headers['content-encoding'];
if (encoding == 'gzip') {
zlib.gunzip(buffer, function(err, decoded) {
// set response headers and write the response
context.ServerResponse.end(decoded.toString());
});
} else if (encoding == 'deflate') {
zlib.inflate(buffer, function(err, decoded) {
// set response headers and write the response
context.ServerResponse.end(decoded.toString());
})
} else {
// set response headers and write the response
context.ServerResponse.end(buffer.toString());
}
});
});
Simple solution: Return a promise (e.g. use the Q-library) from the VM-script.
script.runInNewContext will return whatever you return from the VM-script. That way you have a "callback" for when the VM code finishes.
// Script for VM
// I simplified it. Just resolve or reject the promise whenever you are done with your work
'use strict';
var defer = q.defer();
doABarrelRoll(function() {
defer.resolve('RESULT');
});
defer.promise; // This line will return the promise.
When returning a value from a VM-script, you do not need any return construction. Just write the thing you want and let the magic happen.
// Script for current context
'use strict';
var server = http.createServer(handleRequest);
server.listen(8080);
var handleRequest = function (request, response) {
// get request data...
var context = {
ServerRequest : request,
ServerResponse : response
};
var stringScript = // a string with the script that posts data
var script = vm.createScript(stringScript);
var prom = script.runInNewContext({
context: context,
q: require('q'),
});
prom.done(function ($result) {
console.log('VM finished with result: ' + $result);
});
}