I'm working on child process using fork. but got totally confused on few things
• will it (process)pass app object instance eg:- let app = express(); using IPC
I m trying to explain my senario, first I have server.js where I initialize (starting point) server and other file is my task.js from where I am doing heavy task like reading a big file data and sending data back to other server. For send I had require authorization from that server whose logic is present in main.js and if any error occur I'm send email with few detail to client. Below provide code for email and authorization in main.js
Let task = require('./task.js')
app.sendEmail = function (message, emailinfo, attachment){
// my email logic
}
app.auth= function(host,port)
// Authorization logic
}
New task(app).run()
In task.js (sample code)
Class Task {
constructor(app){
this.app =app
}
run(){
fs.readfile('myfile',function(err,data){
if(err){ let msg =err;
let clientinf; clientinf.to = "client email";
clientinf.cc = " other user in CC";
this.app.sendEmail(msg, clientinf, attach);
}else{
let host='other server url';
let port='port';
this.app.auth(host,port);
}
})
}
}
I want to run task.js in one more thread . note cluster and worker(because I m using node 10.19 so not confident that worker works properly) I don't want to use . It is possible to use folk or spawn to share data between each other. If not how I can achieve my requirement using thread?
Here are two solutions. The first is using the Worker class from the worker_threads module but since you don't want to update the node version the second solution is using fork function from child_process module. They do pretty much the same thing to be honest I can't tell which is better but the worker_threads solution is more recent.
Solution 1:
const { Worker } = require('worker_threads')
const task_script = path.join(__dirname, "./task.js")
const obj = {data:"data"}
const worker = new Worker(task_script, {
workerData: JSON.stringify(obj)
})
worker.on("error", (err) => console.log(err))
worker.on("exit", () => console.log("exit"))
worker.on("message", (data) => {
console.log(data)
res.send(data)
})
and you have to change the task.js code slightly.Here it is
const { parentPort, workerData, isMainThread } = require('worker_threads')
class Task {
constructor(app){
this.app = app
}
run(){
if (!isMainThread) {
console.log("workerData: ", workerData) //you have worker data here
fs.readfile('myfile',function(err,data){
if(err){ let msg = err;
let clientinf; clientinf.to = "client email";
clientinf.cc = " other user in CC";
this.app.sendEmail(msg, clientinf, attach);
parentPort.postMessage(msg) //use can send message to parent like this
} else {
let host='other server url';
let port='port';
this.app.auth(host,port);
}
})
}
}
}
And here is the second solution
const { fork } = require('child_process');
const forked = fork('task.js');
forked.on('message', (msg) => {
console.log('Message from child', msg);
});
forked.send({ hello: 'world' });
and the taks.js way of sending and recieving data with this method
class Task {
constructor(app){
this.app = app
}
run(){
//receive
process.on('message', (msg) => {
console.log('Message from parent:', msg);
});
fs.readfile('myfile',function(err,data){
if(err){ let msg = err;
let clientinf; clientinf.to = "client email";
clientinf.cc = " other user in CC";
this.app.sendEmail(msg, clientinf, attach);
process.send(msg); //send method
} else {
let host='other server url';
let port='port';
this.app.auth(host,port);
}
})
}
}
Related
I am using redis for the notification service, the redis library is Node-Redis with latest version 4.5.1
const Redis = require('redis');
const redisClient = Redis.createClient({ url:'redis://127.0.0.1'});
class NotificationService {
async getNotificationCount(userId){
let llen = 0;
try {
await redisClient.connect();
const notificationKey = `user:notification:${userId}`;
llen = await redisClient.lLen(notificationKey);
redisClient.quit();
} catch(err) {
console.log("Err:", err);
}
return llen;
}
async getNotifications(userId, pageNum, pageSize){
let offset = (pageNum - 1) * pageSize;
let lpopList1 = [];
try {
await redisClient.connect();
const notificationKey = `user:notification:${userId}`;
lpopList1 = await redisClient.lRange(notificationKey, 0, -1);
redisClient.quit();
} catch(err) {
console.log("Err:", err);
}
return lpopList1;
}
...
}
As noted, there is only one client and is being reused for every connection. In each new request, the new connection is built by 'await redisClient.connect()' and closed by 'redisClient.quit()'
However, sometimes the app throws out the error saying 'Err: Error: Socket already opened'.
why is it happening , a better way to deal with this?
Update:
the error happened in circumstance that getNotifications() and getNotificationCount() are being called at the same time.
however, when I only called one function at each time, the error is not happening.
So it seems one redis client in this library can not deal with concurrency, any best practice?
Connecting to Redis can be nontrivial, especially when dealing with multi-node topologies, and you should try to reuse your connections as much as possible: instead of connecting and disconnecting at every run, I would suggest to just leave the client connected: node-redis handles reconnections automatically in the event of a network failure, so you don't have to worry about that.
Of course, whether you choose to bind the lifespan of the connection to the entire process or to, perhaps, your NotificationService class is up to you and your business needs.
const Redis = require('redis');
const redisClient = Redis.createClient({ url:'redis://127.0.0.1'});
// ...
await redisClient.connect();
// ...
class NotificationService {
async getNotificationCount(userId){
let llen = 0;
try {
const notificationKey = `user:notification:${userId}`;
llen = await redisClient.lLen(notificationKey);
} catch(err) {
console.log("Err:", err);
}
return llen;
}
async getNotifications(userId, pageNum, pageSize){
let offset = (pageNum - 1) * pageSize;
let lpopList1 = [];
try {
const notificationKey = `user:notification:${userId}`;
lpopList1 = await redisClient.lRange(notificationKey, 0, -1);
} catch(err) {
console.log("Err:", err);
}
return lpopList1;
}
// ...
}
I have a socket connection using zmq.js client:
// routerSocket.ts
const zmqRouter = zmq.socket("router");
zmqRouter.bind(`tcp://*:${PORT}`);
zmqRouter.on("message", async (...frames) => {
try {
const { measurementData, measurementHeader } =
await decodeL2Measurement(frames[frames.length - 1]);
addHeaderInfo(measurementHeader);
// Add cell id to the list
process.send(
{ measurementData, measurementHeader, headerInfoArrays },
(e: any) => {
return;
},
);
} catch (e: any) {
return;
}
});
I run this socket connection within a forked process in index.ts:
// index.ts
const zmqProcess = fork("./src/routerSocket");
zmqProcess.on("message", async (data: ZmqMessage) => {
if (data !== undefined) {
const { measurementData, measurementHeader, headerInfoArrays } = data;
headerInfo = headerInfoArrays;
emitHeaderInfo(headerInfoArrays);
// Emit the message to subscribers of the rnti
const a = performance.now();
io.emit(
measurementHeader.nrCellId,
JSON.stringify({ measurementData, measurementHeader }),
);
// Emit the message to the all channel
io.emit("all", JSON.stringify({ measurementData, measurementHeader }));
const b = performance.now();
console.log("time to emit: ", a - b);
}
});
There is data coming in rapidly, about one message per ms, to the zmqRouter object, which it then processes and sends onto the main process where I use socket.io to distribute the data to clients. But as soon as the stream begins, node can't do anything else. Even a setInterval log stops working when the stream begins.
Thank you for your help!
Basically, my code worked completely fine in replit, but now it doesnt work in a vsc folder. my replit version also suddenly can’t send any messages anymore. it sends all the console.logs but the client.say stuff it just skips without an error.
const tmi = require('tmi.js');
// Define configuration options
const opts = {
identity: {
username: 'BormBot',
password: 'cut out for a reason'
},
channels: [
'Epicurious__'
]
};
// Create a client with our options
const client = new tmi.client(opts);
const jsonFile = require('./link.json');
const fs = require('fs');
const mongoose = require('mongoose');
// Register our event handlers (defined below)
client.on('connected', onConnectedHandler);
// Connect to Twitch:
client.connect();
client.on('message', (channel, tags, msg, self, target) => {
if (self) return;
//start of geoguessr commands
const link = {
"link": ""
};
if (msg.startsWith('!geolink')) {
if (tags.badges.broadcaster == 1) {
const arguments = msg.split(/[ ]+/)
if (arguments[1]) {
let link = arguments[1];
const data = JSON.stringify(link);
fs.writeFile('./link.json', data, (err) => {
if (err) {
throw err;
}
console.log("JSON data is saved.");
});
client.say(channel, link);
} else {
console.log("no args");
}
}
}
if (msg.startsWith('!game')) {
// read JSON object from file
fs.readFile('./link.json', 'utf-8', (err, data) => {
if (err) {
throw err;
}
// parse JSON object
const linkDone = JSON.parse(data.toString());
// print JSON object
client.say(channel, `The link for the current geoguessr game is: ${linkDone}`);
console.log(`${linkDone}`);
});
}
//end of geoguessr commands
});
// Called every time the bot connects to Twitch chat
function onConnectedHandler(addr, port) {
console.log(`* Connected to ${addr}:${port}`);
}
console
Also on twitch developer forum: 2
On twitch developer forum there hasn't been an answer yet, hence why I'm also putting it on here. Hopefully I can find an answer, also, maybe add a tag for tmi.js
I have written a unit test case,but it is giving error.
Please find the code below
index.js
const { ServiceBusClient, ReceiveMode } = require("#azure/service-bus");
module.exports = async function (context, myTimer) {
// Define connection string and related Service Bus entity names here
const connectionString = process.env['serviceBusConnectionString'];
const queueName = process.env['serviceBusQueueName'];
const sbClient = ServiceBusClient.createFromConnectionString(connectionString);
const queueClient = sbClient.createQueueClient(queueName);
//const receiver = queueClient.createReceiver(ReceiveMode.receiveAndDelete);
const receiver = queueClient.createReceiver(ReceiveMode.peekLock);
const messages = await receiver.receiveMessages(1);
try {
let payloads = [];
messages.forEach((msg) => {
payloads.push(msg.body);
})
await queueClient.close();
} catch (err) {
context.log('Queue message status settle: abandon');
await messages[0].abandon();
console.log('Error ', err);
} finally {
await sbClient.close();
context.done();
}
};
This is the unit test file and I am getting error.Please let me know why I am getting this errorenter image description here
indexTest.js:
beforeEach(() => {
const sbClientStub = {
createQueueClient: sinon.stub().returnsThis(),
createReceiver: sinon.stub().returnsThis(),
receiveMessages:sinon.stub(),
close: sinon.stub(),
};
sinon.stub(ServiceBusClient, 'createFromConnectionString').callsFake(() => sbClientStub);
const ctx = {};
// const actual = await pushToQueue(message, ctx);
// sinon.assert.match(actual, 2);
sinon.assert.calledWithExactly(ServiceBusClient.createFromConnectionString, undefined);
sinon.assert.calledWithExactly(sbClientStub.createQueueClient, undefined);
sinon.assert.calledOnce(sbClientStub.createReceiver, undefined );
//sinon.assert.calledWithExactly(sbClientStub.send.firstCall, { body: 'a' });
//sinon.assert.calledWithExactly(sbClientStub.send.secondCall, { body: 'b' });
sinon.assert.calledTwice(sbClientStub.close);
});
You should replace every sinon.stub() with sinon.spy(). The stub will prevent calling the original implementation of methods, but spies will do. They basically have the same APIs.
In order to call the original methods of #azure/service-bus, make sure the resources of #azure/service-bus are ready such as environment variables, service account, queue and so on.
If you do this, the unit tests are no longer isolated. In fact, they are no longer unit tests, but integration tests, or e2e tests.
I am writing one node js service which receives messages using rabbitmq. But I am facing one issue when I am trying to send concurrent requests to my node js service.
Here is amqp subscriber I have written,
const amqp = require('amqplib/callback_api')
let AmqpConnection = {
// some other methods to make connection
// ....
//....
subscribe: function(){
this.withChannel((channel) => {
let defaultQueueName = "my_queue";
channel.assertQueue(defaultQueueName, { durable: true }, function(err, _ok) {
if (err) throw err;
channel.consume(defaultQueueName, AmqpConnection.processMessage);
Logger.info("Waiting for requests..");
});
})
},
processMessage: function(payload){
debugger
try {
Logger.info("received"+(payload.content.toString()))
}
catch(error){
Logger.error("ERROR: "+ error.message)
//Channel.ack(payload)
}
}
}
And now I am trying to publish messages to this using publisher,
const amqp = require('amqplib/callback_api')
let Publisher = {
// some other methods to make connection
// ....
// ....
sendMessage: function(message){
this.withChannel((channel) => {
let exchangeName = 'exchange';
let exchangeType = 'fanout';
let defaultQueueName = 'my_queue';
channel.assertExchange(exchangeName, exchangeType)
channel.publish(exchangeName, defaultQueueName, new Buffer(message));
})
}
}
let invalidMsg = JSON.stringify({ "content": ""})
let correctMsg = JSON.stringify({ "content": "Test message"})
setTimeout(function () {
for(let i=0; i<2; i++){
Publisher.sendMessage(correctMsg)
Publisher.sendMessage(invalidMsg)
}
}, 3000)
But when I execute both publisher and subscriber, I get following output on subscriber side
2017-02-18T11:27:55.368Z - info: received{"content":""}
2017-02-18T11:27:55.378Z - info: received{"content":""}
2017-02-18T11:27:55.379Z - info: received{"content":""}
2017-02-18T11:27:55.380Z - info: received{"content":""}
It seems like concurrent requests are overriding message received. Can someone help here?