It may be a wrong way to use bull queue but here is what I want to do:
var Promise = require('bluebird');
var redis = require('redis');
var Queue = require('bull');
var redisClient = redis.createClient(6379);
var pdfQueue = new Queue('msg');
function check(resolve, reject,i) {
console.log('check called');
//Or if it is in Router then I want to send request, response in queue so that I can call them in on complete function
pdfQueue.add('msg',{'msg':'Hello', 'resolve':resolve,'reject':reject}).then(job=>{
console.log('added to the pdf')
});
}
pdfQueue.on('completed', function (job, result) {
//Here I want to call request.send('some msg');
//and resolve('complete');
resolve('final callback');
})
pdfQueue.process('msg',100,function (job,done) {
console.log('process');
done(null,'job done ')
})
function check2 () {
return new Promise(function(resolve, reject){
check(resolve,reject);
})
}
check2().then(data => {
console.log('got the value ', data)
});
In my real project I want to implement queue where I will be sending pdf to the user. Like res.download(pdf path); but this function should be in pdf.on('completed',()=>{ res.download(pdf path); }); or in resolve(pdfPath) but I am not able to find anyway to send pdf to the user using queue because I don't know how to call response or resolve in other functions by using queue jobs.
Please help me. Thanks you
Related
I'm currently having problems figuring out how to capture my MQTT message event back to my REST API body which is written in NodeJS. My current setup is App -> NodeJS REST API -> MQTT broker inside RPi 3.
This is my MQTTHandler.js class where I have put all my reusable MQTT functions
const mqtt = require('mqtt')
class MQTTHandler {
constructor (host) {
this.client = null
this.host = host
}
connect () {
this.client = mqtt.connect(this.host)
this.client.on('error', function (err) {
console.log(err)
this.client.end()
})
this.client.on('connect', function () {
console.log('MQTT client connected...')
})
// I need this to send message back to app.js
this.client.on('message', function (topic, message) {
if (!message.toString()) message = 'null'
console.log(JSON.parse(message.toString()))
})
this.client.on('close', function () {
console.log('MQTT client disconnected...')
})
}
subscribeTopic (topic) {
this.client.subscribe(topic)
}
unsubscribeTopic (topic) {
this.client.unsubscribe(topic)
}
sendMessage (topic, message) {
this.client.publish(topic, message)
}
}
module.exports = MQTTHandler
And below is a short snippet of my app.js
const MQTTHandler = require('./mqtt.handler')
...
var mqttClient = new MQTTHandler('mqtt://127.0.0.1')
mqttClient.connect()
app.get('/hello', function (req, res) {
mqttClient.subscribeTopic('topic')
mqttClient.sendMessage('topic', 'hello world')
// I need to return the MQTT message event here
// res.json(<mqtt message here>)
res.end()
})
I have already tried using NodeJS' event emitter but it doesn't seem to work. Any help or suggestions would be much appreciated, thank you!
You are trying to mix a synchronous protocol (HTTP) with and asynchronous protocol (MQTT). These 2 paradigm don't easily mix.
When you publish an MQTT message you have no idea how many clients may be subscribed to that topic, it could be zero, it could be many. There is also no guarantee that any of them will send a reply so you need to include a timeout. (You also need to include a request id in the payload so you can coordinate any response with the request as you can't say anything about what order responses may come in.)
Your example code is only using 1 topic, this is very bad as you will end up needing to filter out request messages from response messages. Better to use 2 different topics (MQTT v5 even has a msg header to specify the topic the response should be sent on).
Having said all that it is possible to build something that will work (I will use request and reply topics.
var inflightRequests = {};
// interval to clear out requests waiting for a response
// after 3 seconds
var timer = setInterval(function() {
var now = new Date.now();
var keys = Object.keys(inflightRequests);
for (var key in keys) {
var waiting = inflightRequests[keys[key]];
var diff = now = waiting.timestamp;
// 3 second timeout
if (diff > 3000) {
waiting.resp.status(408).send({});
delete(inflightRequests[keys[key]]);
}
}
},500);
// on message handler to reply to the HTTP request
client.on('message', function(topic, msg){
if (topic.equals('reply')) {
var payload = JSON.parse(msg);
var waiting = inflightRequest[payload.requestId];
if (waiting) {
waiting.res.send(payload.body);
delete(inflightRequests[payload.requestId]);
} else {
// response arrived too late
}
}
});
// HTTP route handler.
app.get('/hello', function(req, res) {
//using timestamp as request Id as don't have anything better in this example.
var reqId = Date.now();
var waiting = {
timeStamp: reqId,
res: res
}
inflightRequests[reqId] = waiting;
var message = {
requestId: reqId,
payload: 'hello world'
}
client.publish('request',JSON.stringify(message));
});
I think the rendering takes place before the searching of the string on the files, i have tried different methods but don't seems to get this working. any help will be appreciated. im a noob on to the nodejs. im trying to get the id of the user and query and get all the data and there after see if he is in any of the lists given and finally render the page.
const j = [];
let name = '';
const filename = [];
var ext = '';
module.exports = function(app, express) {
app.use(bodyParser.urlencoded({ extended: false }));
app.use(bodyParser.json());
app.post('/cusdetails', isLoggedIn, function (req, res) {
var cusid=req.body.cusid;
var insertQuerys = "SELECT * FROM customer WHERE cusid=? ORDER BY rowid DESC LIMIT 1";
connection.query(insertQuerys,[cusid],
function(err, rows){
rows.forEach( (row) => {
name=row.fncus;
});
fs.readdir('./views/iplist', function(err, files) {
if (err)
throw err;
for (var index in files) {
j.push(files[index])
}
j.forEach(function(value) {
var k = require('path').resolve(__dirname, '../views/iplist/',value);
fs.exists(k, function(fileok){
if(fileok) {
fs.readFile(k, function(err, content) {
if (err) throw err;
if (content.indexOf(name) > -1) {
ext = path.extname(k);
filename.push(path.basename(k, ext));
}
});
}
else {
console.log(" FileNotExist ");
}
});
});
});
console.log(filename);
res.render('cusdetails.ejs', {rows: rows, user:req.user , aml: filename });
});
})
You can create simple Promise wrapper and then use it inside async/await function to pause execution until resolved.
// use mysql2 package as it provides promise, less work to write promise wrappers
const mysql = require('mysql2/promise');
// create the connection to database
const connection = mysql.createConnection({
host: 'localhost',
user: 'root',
database: 'test'
});
// sample wrapper
function some(k) {
// more advisable to have local variables, why do you need this to be array?
var filename = [];
return new Promise((resolve, reject) => {
// doing this is also not recommended check nodejs documentation **fs.exists** for more info
fs.exists(k, function(fileok){
if(fileok) {
fs.readFile(k, function(err, content) {
if (err) reject(err);
if (content.indexOf(name) > -1) {
ext = path.extname(k);
filename.push(path.basename(k, ext));
resolve(filename)
}
});
}
else {
// reject(new Error("FileNotExist"))
console.log(" FileNotExist ");
}
});
})
}
// note the use of async
app.post('/cusdetails', isLoggedIn, async function (req, res) {
var cusid=req.body.cusid;
var insertQuerys = "SELECT * FROM customer WHERE cusid=? ORDER BY rowid DESC LIMIT 1";
// using await to pause excution, waits till query is finished
const [rows] = await connection.query(insertQuerys,[cusid])
rows.forEach( (row) => {
name=row.fncus;
});
// then you can
var result = await some(k)
...
Note however this way you loose the advantage of concurrent execution, as it's kindoff blocking. If the result of one call is not used in another, you can execute in parallel and await for result to achieve sequencing like
const [rows] = connection.query(insertQuerys,[cusid])
var result = some(k)
console.log(await rows) // do something
console.log(await result) // do something
JavaScript is asynchronous. This means that if you have a function with a callback (i.e. your query), the callback will be called asynchronously, at an unknown time, while the other code executes.
You need to look up some tutorials how to deal with callbacks, to get a proper understanding of it. Another method is using async/await and/or promises.
Basically, if you take the following code:
console.log("this will print first");
setTimeout(function () {
console.log("this will print last");
}, 1000);
console.log("this will print second");
If you run the code above, the top level is executed synchronously, so, it first calls console.log, then it executes setTimeout, which is synchronous. It sets a timeout, then says "I'm ready", and the code continues to the other console.log. After 1 second (1000 milliseconds), the callback in the setTimeout function is executed, and only then that console.log is called. You can not make the rest of the code wait this way, you need to restructure your code or read into promises.
I am making use of "socket.io-client" and "socket.io stream" to make a request and then stream some data. I have the following code that handles this logic
Client Server Logic
router.get('/writeData', function(req, res) {
var io = req.app.get('socketio');
var nameNodeSocket = io.connect(NAMENODE_ADDRESS, { reconnect: true });
var nameNodeData = {};
async.waterfall([
checkForDataNodes,
readFileFromS3
], function(err, result) {
if (err !== null) {
res.json(err);
}else{
res.json("Finished Writing to DN's");
}
});
function checkForDataNodes(cb) {
nameNodeSocket.on('nameNodeData', function(data) {
nameNodeData = data;
console.log(nameNodeData);
cb(null, nameNodeData);
});
if (nameNodeData.numDataNodes === 0) {
cb("No datanodes found");
}
}
function readFileFromS3(nameNodeData, cb) {
for (var i in nameNodeData['blockToDataNodes']) {
var IP = nameNodeData['blockToDataNodes'][i]['ipValue'];
var dataNodeSocket = io.connect('http://'+ IP +":5000");
var ss = require("socket.io-stream");
var stream = ss.createStream();
var byteStartRange = nameNodeData['blockToDataNodes'][i]['byteStart'];
var byteStopRange = nameNodeData['blockToDataNodes'][i]['byteStop'];
paramsWithRange['Range'] = "bytes=" + byteStartRange.toString() + "-" + byteStopRange.toString();
//var file = require('fs').createWriteStream('testFile' + i + '.txt');
var getFileName = nameNodeData['blockToDataNodes'][i]['key'].split('/');
var fileData = {
'mainFile': paramsWithRange['Key'].split('/')[1],
'blockName': getFileName[1]
};
ss(dataNodeSocket).emit('sendData', stream, fileData);
s3.getObject(paramsWithRange).createReadStream().pipe(stream);
//dataNodeSocket.disconnect();
}
cb(null);
}
});
Server Logic (that gets the data)
var dataNodeIO = require('socket.io')(server);
var ss = require("socket.io-stream");
dataNodeIO.on('connection', function(socket) {
console.log("Succesfully connected!");
ss(socket).on('sendData', function(stream, data) {
var IP = data['ipValue'];
var blockName = data['blockName'];
var mainFile = data['mainFile'];
dataNode.makeDir(mainFile);
dataNode.addToReport(mainFile, blockName);
stream.pipe(fs.createWriteStream(mainFile + '/' + blockName));
});
});
How can I properly disconnect the connections in function readFileFromS3. I have noticed using dataNodeSocket.disconnect() at the end does not work as I cannot verify the data was received on the 2nd server. But if I comment it out, I can see the data being streamed to the second server.
My objective is to close the connections in Client Server side
It appears that the main problem with closing the socket is that you weren't waiting for the stream to be done writing before trying to close the socket. So, because the writing is all asynchronous and finishes sometime later, you were trying to close the socket before the data had been written.
Also because you were putting asynchronous operations inside a for loop, you were also running all your operations in parallel which may not be exactly what you want as it makes error handling more difficult and server load more difficult.
Here's the code I would suggest that does the following:
Create a function streamFileFromS3() that streams a single file and returns a promise that will notify when it's done.
Use await in a for loop with that streamFileFromS3() to serialize the operations. You don't have to serialize them, but then you would have to change your error handling to figure out what to do if one errors while the others are already running and you'd have to be more careful about concurrency issues.
Use try/catch to catch any errors from streamFileFromS3().
Add error handling on the stream.
Change all occurrences of data['propertyName'] to data.propertyName. The only time you need to use brackets is if the property name contains a character that is not allowed in a Javascript identifier or if the property name is in a variable. Otherwise, the dot notation is preferred.
Add socket.io connection error handling logic for both socket.io connections.
Set returned status to 500 when there's an error processing the request
So, here's the code for that:
const ss = require("socket.io-stream");
router.get('/writeData', function(req, res) {
const io = req.app.get('socketio');
function streamFileFromS3(ip, data) {
return new Promise((resolve, reject) => {
const dataNodeSocket = io.connect(`http://${ip}:5000`);
dataNodeSocket.on('connect_error', reject);
dataNodeSocket.on('connect_timeout', () {
reject(new Error(`timeout connecting to http://${ip}:5000`));
});
dataNodeSocket.on('connection', () => {
// dataNodeSocket connected now
const stream = ss.createStream().on('error', reject);
paramsWithRange.Range = `bytes=${data.byteStart}-${data.byteStop}`;
const filename = data.key.split('/')[1];
const fileData = {
'mainFile': paramsWithRange.Key.split('/')[1],
'blockName': filename
};
ss(dataNodeSocket).emit('sendData', stream, fileData);
// get S3 data and pipe it to the socket.io stream
s3.getObject(paramsWithRange).createReadStream().on('error', reject).pipe(stream);
stream.on('close', () => {
dataNodeSocket.disconnect();
resolve();
});
});
});
}
function connectError(msg) {
res.status(500).send(`Error connecting to ${NAMENODE_ADDRESS}`);
}
const nameNodeSocket = io.connect(NAMENODE_ADDRESS, { reconnect: true });
nameNodeSocket.on('connect_error', connectError).on('connect_timeout', connectError);
nameNodeSocket.on('nameNodeData', async (nameNodeData) => {
try {
for (let item of nameNodeData.blockToDataNodes) {
await streamFileFromS3(item.ipValue, item);
}
res.json("Finished Writing to DN's");
} catch(e) {
res.status(500).json(e);
}
});
});
Other notes:
I don't know what paramsWithRange is as it is not declared here and when you were doing everything in parallel, it was getting shared among all the connections which is asking for a concurrency issue. In my serialized implementation, it's probably safe to share it, but the way it is now bothers me as it's a concurrency issue waiting to happen.
I am trying to ask a question to the user and using AMAZON.yesIntent and noIntent to capture the response. In case I get response, I am calling other function which should emit something back to user. But the emit is not being called. Code below:
'use strict';
var Alexa = require("alexa-sdk");
var https = require('https');
var alexa;
exports.handler = function(event, context, callback) {
alexa = Alexa.handler(event, context);
alexa.registerHandlers(handlers);
alexa.execute();
};
var handlers = {
'LaunchRequest': function() {
alexa.emit(':ask', 'my text');
},
'oneShotIntent': function() {
var self = this;
// removed the code for simplicity
self.attributes['yes_no'] = true;
alexa.emit(':ask', 'sample yes no question');
})
},
'AMAZON.YesIntent': function () {
if (this.attributes['yes_no']) {
delete this.attributes['yes_no'];
alexa.emit('oneShotSecond');
}
},
'oneShotSecond': function() {
delete this.attributes['yes_no']; // already deleted
// removed code for simplicity
var self = this;
// removed code for simplicity
console.log(info);
if (info) {
console.log('here in if condition');
self.emit(':tell', 'sample text');
} else {
self.emit(':tell', 'sorry failed');
}
});
}
};
The first emit is working. The console.log in other function are happening. But emit is failing.
No logs in cloudwatch also. It just ends after printing the console.log data points.
Any help?
I am not sure if I understand
alexa.emit('oneShotSecond');
However I would have done it like below
oneShotSecond(this);
And then
'oneShotSecond': function(self) { // use self.emit}
I have a weird problem where my callback is never published and the message goes to timeout, even though the method runs in the queue. This happens in some specific queues and after it happens once, i cannot make any other requests from client which even previously worked, they all timeout. Have to restart the client and sever to make it working again.
This is the code, where its happening, and i cant seem to understand whats wrong.
Server.js file where i am creating the queues. I have several such queues, this is one of them.
var amqp = require('amqp');
var util = require('util');
var cnn = amqp.createConnection({host:'127.0.0.1'});
var getCart = require('./services/getCart');
cnn.on('ready', function() {
cnn.queue('getCart_queue', function(q){
q.subscribe(function(message, headers, deliveryInfo, m){
// util.log(util.format( deliveryInfo.routingKey, message));
// util.log("Message: "+JSON.stringify(message));
// util.log("DeliveryInfo: "+JSON.stringify(deliveryInfo));
getCart.handle_request(message, function(err,res){
cnn.publish(m.replyTo, res, {
contentType:'application/json',
contentEncoding:'utf-8',
correlationId:m.correlationId
});
});
});
});
});
Here, the handle request function is completed successfully, but the callback never goes through and its always timeout on the other end
var cart = require('../models/cart');
function handle_request(msg, callback) {
var user_id = msg.id;
cart
.find({id:user_id})
.populate('users ads')
.exec(function(err, results){
// This works, just the callback doesnt
if(!err){
console.log(results);
callback(null, results);
} else {
console.log(err);
callback(err, null);
}
});
}
exports.handle_request = handle_request;
this is how i am calling the request
var msg_payload = {"id":id};
mq_client.make_request('getCart_queue', msg_payload, function(err, results){
console.log(results); // never prints
//stuff that is never reached
});
These are my rpc files, i dont think there should be anything wrong with these, as some other queues work fine.
And this is the error shown on client
GET /getCart - - ms - -
Error: timeout 6ee0bd2a4b2ba1d8286e068b0f674d8f
at Timeout.<anonymous> (E:\Ebay_client\rpc\amqprpc.js:32:18)
at Timeout.ontimeout [as _onTimeout] (timers.js:341:34)
at tryOnTimeout (timers.js:232:11)
at Timer.listOnTimeout (timers.js:202:5)
Hope the information is not vague, if you need more, please let me know. Thanks!
I Think the error is in this file, because i tried debugging and from the rabbitmq server, the callback is being called and it has the correlation id as well as the reply to variable, so the request is not getting picked up here.
var amqp = require('amqp')
, crypto = require('crypto');
var TIMEOUT=8000;
var CONTENT_TYPE='application/json';
var CONTENT_ENCODING='utf-8';
var self;
exports = module.exports = AmqpRpc;
function AmqpRpc(connection){
self = this;
this.connection = connection;
this.requests = {};
this.response_queue = false;
}
AmqpRpc.prototype.makeRequest = function(queue_name, content, callback){
self = this;
var correlationId = crypto.randomBytes(16).toString('hex');
var tId = setTimeout(function(corr_id){
callback(new Error("timeout " + corr_id));
delete self.requests[corr_id];
}, TIMEOUT, correlationId);
var entry = {
callback:callback,
timeout: tId
};
self.requests[correlationId]=entry;
self.setupResponseQueue(function(){
self.connection.publish(queue_name, content, {
correlationId:correlationId,
contentType:CONTENT_TYPE,
contentEncoding:CONTENT_ENCODING,
replyTo:self.response_queue});
});
};
AmqpRpc.prototype.setupResponseQueue = function(next){
if(this.response_queue) return next();
self = this;
self.connection.queue('', {exclusive:true}, function(q){
self.response_queue = q.name;
q.subscribe(function(message, headers, deliveryInfo, m){
var correlationId = m.correlationId;
if(correlationId in self.requests){
var entry = self.requests[correlationId];
clearTimeout(entry.timeout);
delete self.requests[correlationId];
entry.callback(null, message);
}
});
return next();
});
};
This is the code for your make_request() in client.js file:
var amqp = require('amqp');
var connection = amqp.createConnection({host:'127.0.0.1'});
var rpc = new (require('./amqprpc'))(connection);
function make_request(queue_name, msg_payload, callback){
rpc.makeRequest(queue_name, msg_payload, function(err, response){
if(err)
console.error(err);
else{
console.log("response", response);
callback(null, response);
}
});
}
exports.make_request = make_request;
Look at what happens when you have an err on rpc.makeRequest():
rpc.makeRequest(queue_name, msg_payload, function(err, response){
if(err)
console.error(err);
//
//HERE: should be a callback call here.
//
else{
console.log("response", response);
callback(null, response);
}
});
This could be why you are getting a timeout. I hope it helps.
There wasn't a problem with rabbitMQ but with my queries in the handle request and after responding to the request.
For others coming with this problem, check and double check every statement, as the error will not show in the console, but will only show a timeout