I have an express app which uses bullmq queues, schedulers and workers. Even after pressing Ctrl + C I can still see the node process running inside my Activity manager but my server on the terminal shuts down. I know this because the bullmq task starts outputting console.log statements even after the server is down to the terminal.
This is what my server.js file looks like
// eslint-disable-next-line import/first
import http from 'http';
import { app } from './app';
import { sessionParser } from './session';
import { websocketServer } from './ws';
import 'jobs/repeatable';
const server = http.createServer(app);
server.on('upgrade', (request, socket, head) => {
sessionParser(request, {}, () => {
websocketServer.handleUpgrade(request, socket, head, (ws) => {
websocketServer.emit('connection', ws, request);
});
});
});
server.on('listening', () => {
websocketServer.emit('listening');
});
server.on('close', () => {
websocketServer.emit('close');
});
// https://stackoverflow.com/questions/18692536/node-js-server-close-event-doesnt-appear-to-fire
process.on('SIGINT', () => {
server.close();
});
export { server };
Notice that I have a SIGINT handler defined above. Is this the reason my jobs are not exiting? Do I have to manually close every queue, worker and scheduler inside my SIGINT? My jobs/repeatable.js file looks as shown below
const { scheduleJobs } = require('jobs');
if (process.env.ENABLE_JOB_QUEUE === 'true') {
scheduleJobs();
}
Here is my jobs.js file
import { scheduleDeleteExpiredTokensJob } from './delete-expired-tokens';
import { scheduleDeleteNullVotesJob } from './delete-null-votes';
export async function scheduleJobs() {
await scheduleDeleteExpiredTokensJob();
await scheduleDeleteNullVotesJob();
}
Here is my delete-expired-tokens.js file, other one is quite similar
import { processor as deleteExpiredTokensProcessor } from './processor';
import { queue as deleteExpiredTokensQueue } from './queue';
import { scheduler as deleteExpiredTokensScheduler } from './scheduler';
import { worker as deleteExpiredTokensWorker } from './worker';
export async function scheduleDeleteExpiredTokensJob() {
const jobId = process.env.QUEUE_DELETE_EXPIRED_TOKENS_JOB_ID;
const jobName = process.env.QUEUE_DELETE_EXPIRED_TOKENS_JOB_NAME;
await deleteExpiredTokensQueue.add(jobName, null, {
repeat: {
cron: process.env.QUEUE_DELETE_EXPIRED_TOKENS_FREQUENCY,
jobId,
},
});
}
export {
deleteExpiredTokensProcessor,
deleteExpiredTokensQueue,
deleteExpiredTokensScheduler,
deleteExpiredTokensWorker,
};
How do I shutdown bullmq task queues gracefully?
You have to call the close() method on the workers:
server.on('close', async () => {
websocketServer.emit('close');
// Close the workers
await worker.close()
});
Docs
Related
I have a file called socket_io.js where I created a single instance of a socket io client in my react app as shown below:
socket_io.js
import EndPoints from './http/endpoints';
import io from "socket.io-client";
const socketUrl = EndPoints.SOCKET_BASE;
let socketOptions = { transports: ["websocket"] }
let socket;
if (!socket) {
socket = io(socketUrl, socketOptions);
socket.on('connect', () => {
console.log(`Connected to Server`);
})
socket.on('disconnect', () => {
console.log(`Disconnected from Server`);
})
}
export default socket;
Then I imported the above singleton in many react components as shown below.
MessagePage.js
import socket from '../socket_io.js';
let messageHandler=(data)=>{
}
useEffect(()=>{
socket.on('message',messageHandler); //This event no longer fires When the singleton socket io instance is reconnected
return ()=>{
socket.off('message');
}
},[]);
which works well but the issue I'm facing now is that when the singleton instance reconnects, the components referencing it are no longer receiving events from their respective handlers.
Possible causes of reconnection are when I manually restart the server
How can this be resolved?
I just solved this after working on it for a project of my own. My method involves two parts: creating the socket in a useEffect hook and then managing it using useRef for reconnection situations.
In Summary:
I think there are two issues. One is that the socket is being initialized as a singleton and not using a hook/context. I've read other reports of strangeness in this case, so I suggest switching to using context and creating your socket in a hook. Secondly, we have to manually store reconnection logic (although by generating the socket properly, it seems as though the actual event listeners are kept through reconnect).
export const SocketContext = createContext();
export const SocketContextProvider = ({ children }) => {
const [socket, setSocket] = useState();
const reconnectEmits = useRef([]);
// Here's your basic socket init.
useEffect(()=>{
const newSocket = io(url);
setSocket(newSocket);
return () => {
newSocket.close();
}
}, []);
// Code used to rejoin rooms, etc., on reconnect.
newSocket.io.on('reconnect', e => {
console.log("it took " + e + " tries to reconnect.");
for (let action of reconnectEmits.current) {
newSocket.emit(action.event, action.data);
}
})
// Here I also define a setListener and removeListener function, which determine which listeners a socket listens to. I don't have the code in front of me now, but it's pretty simple:
const addListener = (event, function) => {
// I use socket.off(event) right here to make sure I only have one listener per event, but you may not want this. If you don't use it you will need to make sure you use hooks to remove the event listeners that your components add to your socket when they are removed from the DOM.
socket.on(event, function);
}
// I implement an emit function here that's a wrapper, but I'm not sure if it's necessary. You could just expose the socket itself in the context. I just choose not to.
return (
<SocketContext.Provider value={{ emit, setListener, removeListener, addReconnectEmit, removeReconnectEmit }}>
{children}
</SocketContext.Provider>
)
}
And then in my components, in addition to having the emits to join rooms or conduct actions, I also provide the add and remove ReconnectEmit functions:
const addReconnectEmit = (event, data) => {
reconnectEmits.current = ([...reconnectEmits.current, { event, data }]);
console.log(reconnectEmits.current);
}
const removeReconnectEmit = (event, data) => {
console.log('removing reconnect event');
reconnectEmits.current = reconnectEmits.current.filter(e =>
{ return e.event !== event && e.data !== data }
);
console.log(reconnectEmits.current);
};
With these, I can set it so that, after a reconnect, my socket knows to reconnect to a certain room, etc. See here:
const Chatroom = ({ convoId }) => {
console.log("RENDERED: Chatroom");
const { emit, addReconnectEmit, removeReconnectEmit } = useContext(SocketContext);
useEffect(() => {
emit('joinConvo', convoId);
console.log("Emitting joinConvo message.");
addReconnectEmit('joinConvo', convoId);
return () => {
emit('leaveConvo', convoId);
removeReconnectEmit('leaveConvo', convoId);
}
}, [convoId, emit, addReconnectEmit, removeReconnectEmit]);
return (
<div id="chatroom">
<ChatroomOutput />
<ChatroomStatus />
<ChatroomControls convoId={convoId} />
</div>
);
}
I hope that helps! Between useEffect and manual reconnection logic, I just fixed similar issues to the ones you were having, where I was losing data on reconnection.
Saw you just answered yourself but my approach might still be valuable for others or if you continue to build a socket-client.
You need to abstract the listening components away from the socket object. The socket object upon onMessage needs to retrieve the subscribers and publish the new message to them. You can of course add filtering based on id, type or other properties. Also each component can drop its subscription when un-mounting or based on another need.
In order to show case I used timers but would be easily converted to messages.
socket_io.js
let socket;
const subscribers = []
if (!socket) {
// socket initial connect
socket = true
setInterval(() => {
console.log('interval runs', { socket })
if (socket) {
subscribers.forEach((sub) => {
sub.onMessage()
})
}
}, 1000)
setTimeout(() => {
// socket disconnects
socket = false
setTimeout(() => {
// socket reconnects
socket = true
}, 4000)
}, 4000)
}
export default subscribers;
MessagePage.js
import React, { useEffect, useState } from 'react'
import subscribers from './socket_io.js'
const MessagePage = () => {
const [messageCount, setMessageCount] = useState(0)
let messageHandler = (data) => {
setMessageCount((current) => current + 1)
}
useEffect(() => {
subscribers.push({
id: '1',
onMessage: (data) => messageHandler(data)
})
return () => {
const subToRemove = subscribers.findIndex((sub) => sub.id === '1')
subscribers.splice(subToRemove, 1)
}
}, []);
return (
<div>
Messages received: {messageCount}
</div>
)
}
export default MessagePage
Hope I could help.
export default expects a Hoistable Declarative , i.e function,express
socket_oi.js
import EndPoints from './http/endpoints';
import io from "socket.io-client";
const socketUrl = EndPoints.SOCKET_BASE;
let socketOptions = { transports: ["websocket"] }
let socket;
class Socket {
constructor (){
if (!socket) {
socket = io(socketUrl, socketOptions);
socket.on('connect', () => {
console.log(`Connected to Server`);
})
socket.on('disconnect', () => {
console.log(`Disconnected from Server`);
})
}
socket = this
}
}
//Freeze the object , to avoid modification by other functions/modules
let newSocketInstance = Object.freeze(new Socket)
module.exports = newSocketInstance;
MessagePage.js
import socket from '../socket_io.js';
const MessagePage = (props){
const messageHandler=(data)=>{
}
useEffect(()=>{
socket.on('message',messageHandler); //This event no longer fires When the
singleton socket io instance is reconnected
return ()=>{
socket.off('message');
}
},[]);
}
I have a Node 14 server which is initialized like this:
import express, { Express } from 'express';
import kafkaConsumer from './modules/kafkaConsumer';
async function bootstrap(): Promise<Express> {
kafkaConsumer();
const app = express();
app.get('/health', (_req, res) => {
res.send('ok');
});
return app;
}
export default bootstrap;
kafkaConsumer code:
import logger from './logger.utils';
import KafkaConnector from '../connectors/kafkaConnector';
// singleton
const connectorInstance: KafkaConnector = new KafkaConnector('kafka endpoints', 'consumer group name');
// creating consumer and producer outside of main function in order to not initialize a new consumer producer per each new call.
(async () => {
await connectorInstance.createConsumer('consumer group name');
})();
const kafkaConsumer = async (): Promise<void> => {
const kafkaConsumer = connectorInstance.getConsumer();
await kafkaConsumer.connect();
await kafkaConsumer.subscribe({ topic: 'topic1', fromBeginning: true });
await kafkaConsumer.run({
autoCommit: false, // cancel auto commit in order to control committing
eachMessage: async ({ topic, partition, message }) => {
const messageContent = message.value ? message.value.toString() : '';
logger.info('received message', {
partition,
offset: message.offset,
value: messageContent,
topic
});
// commit message once finished all processing
await kafkaConsumer.commitOffsets([ { topic, partition, offset: message.offset } ]);
}
});
};
export default kafkaConsumer;
You can see that in the kafkaConsumer module there's an async function which is called at the begging to initialize the consumer instance.
How can I guarantee that it successfully passed when importing the module?
In addition, when importing the module, does this mean that the kafkaConsumer default function, is automatically called? won't it cause the server to be essentially stuck at startup?
Would appreciate some guidance here, thanks in advance.
Twicked the Kafka initalzation, and tested with local Kafka. Everything works as expected.
With top-level await accepted into ES2022, I wonder if it is save to assume that await import("./path/to/module") has no timeout at all.
Here is what I’d like to do:
// src/commands/do-a.mjs
console.log("Doing a...");
await doSomethingThatTakesHours();
console.log("Done.");
// src/commands/do-b.mjs
console.log("Doing b...");
await doSomethingElseThatTakesDays();
console.log("Done.");
// src/commands/do-everything.mjs
await import("./do-a");
await import("./do-b");
And here is what I expect to see when running node src/commands/do-everything.mjs:
Doing a...
Done.
Doing b...
Done.
I could not find any mentions of top-level await timeout, but I wonder if what I’m trying to do is a misuse of the feature. In theory Node.js (or Deno) might throw an exception after reaching some predefined time cap (say, 30 seconds).
Here is how I’ve been approaching the same task before TLA:
// src/commands/do-a.cjs
import { autoStartCommandIfNeeded } from "#kachkaev/commands";
const doA = async () => {
console.log("Doing a...");
await doSomethingThatTakesHours();
console.log("Done.");
}
export default doA;
autoStartCommandIfNeeded(doA, __filename);
// src/commands/do-b.cjs
import { autoStartCommandIfNeeded } from "#kachkaev/commands";
const doB = async () => {
console.log("Doing b...");
await doSomethingThatTakesDays();
console.log("Done.");
}
export default doB;
autoStartCommandIfNeeded(doB, __filename);
// src/commands/do-everything.cjs
import { autoStartCommandIfNeeded } from "#kachkaev/commands";
import doA from "./do-a";
import doB from "./do-b";
const doEverything = () => {
await doA();
await doB();
}
export default doEverything;
autoStartCommandIfNeeded(doEverything, __filename);
autoStartCommandIfNeeded() executes the function if __filename matches require.main?.filename.
Answer: No, there is not a top-level timeout on an await.
This feature is actually being used in Deno for a webserver for example:
import { serve } from "https://deno.land/std#0.103.0/http/server.ts";
const server = serve({ port: 8080 });
console.log(`HTTP webserver running. Access it at: http://localhost:8080/`);
console.log("A");
for await (const request of server) {
let bodyContent = "Your user-agent is:\n\n";
bodyContent += request.headers.get("user-agent") || "Unknown";
request.respond({ status: 200, body: bodyContent });
}
console.log("B");
In this example, "A" gets printed in the console and "B" isn't until the webserver is shut down (which doesn't automatically happen).
As far as I know, there is no timeout by default in async-await. There is the await-timeout package, for example, that is adding a timeout behavior. Example:
import Timeout from 'await-timeout';
const timer = new Timeout();
try {
await Promise.race([
fetch('https://example.com'),
timer.set(1000, 'Timeout!')
]);
} finally {
timer.clear();
}
Taken from the docs: https://www.npmjs.com/package/await-timeout
As you can see, a Timeout is instantiated and its set method defines the timeout and the timeout message.
I created a simple class for polling by Event Emitter of Nodejs
For example:
import EventEmitter from "events";
import config from "../config";
export class Poller extends EventEmitter {
constructor(private timeout: number = config.pollingTime) {
super();
this.timeout = timeout;
}
poll() {
setTimeout(() => this.emit("poll"), this.timeout);
}
onPoll(fn: any) {
this.on("poll", fn); // listen action "poll", and run function "fn"
}
}
But I don't know to write the right test for Class. This my unit test
import Sinon from "sinon";
import { Poller } from "./polling";
import { expect } from "chai";
describe("Polling", () => {
it("should emit the function", async () => {
let spy = Sinon.spy();
let poller = new Poller();
poller.onPoll(spy);
poller.poll();
expect(spy.called).to.be.true;
});
});
But It always false
1) Polling
should emit the function:
AssertionError: expected false to be true
+ expected - actual
-false
+true
Please tell me what's wrong with my test file. Thank you very much !
You can follow sinon doc
Quick fix
import Sinon from "sinon";
import { Poller } from "./polling";
import { expect } from "chai";
import config from "../config";
describe("Polling", () => {
it("should emit the function", async () => {
// create a clock to control setTimeout function
const clock = Sinon.useFakeTimers();
let spy = Sinon.spy();
let poller = new Poller();
poller.onPoll(spy);
poller.poll(); // the setTimeout function has been locked
// "unlock" the setTimeout function with a "tick"
clock.tick(config.pollingTime + 10); // add 10ms to pass setTimeout in "poll()"
expect(spy.called).to.be.true;
});
});
I'm working on an electron application that performs a screenshot capture every 3 seconds with the electron api, and writes it to a given target path. I've set up a separate BrowserWindow where the capturing code runs in (see code structure below) a setInterval() "loop", but whenever the capture happens, the app freezes for a moment. I think it is the call to source.thumbnail.toPng() or writeScreenshot() method in the file ScreenCapturer.jshtml.js.
I set up this structure as I though this was the way to go, but apparently this is not. WebWorkers won't help me either as I need node modules such as fs, path and desktopCapturer (from electron).
How would one do this type of task without blocking the main thread every time the interval code (as seen in file ScreenCapturer.jshtml.js) runs (because I thought the renderer processes were separate processes?)
My code as reference
main.js (main process)
// all the imports and other
// will only show the import that matters
import ScreenCapturer from './lib/capture/ScreenCapturer';
app.on('ready', () => {
// Where I spawn my main UI
mainWindow = new BrowserWindow({...});
mainWindow.loadURL(...);
// Other startup stuff
// Hee comes the part where I call function to start capturing
initCapture();
});
function initCapture() {
const sc = new ScreenCapturer();
sc.startTakingScreenshots();
}
ScreenCapturer.js (module used by main process)
'use strict';
/* ******************************************************************** */
/* IMPORTS */
import { app, BrowserWindow, ipcMain } from 'electron';
import url from 'url';
import path from 'path';
/* VARIABLES */
let rendererWindow;
/*/********************************************************************///
/*///*/
/* ******************************************************************** */
/* SCREENCAPTURER */
export default class ScreenCapturer {
constructor() {
rendererWindow = new BrowserWindow({
show: true, width: 400, height: 600,
'node-integration': true,
webPreferences: {
webSecurity: false
}
});
rendererWindow.on('close', () => {
rendererWindow = null;
});
}
startTakingScreenshots(interval) {
rendererWindow.webContents.on('did-finish-load', () => {
rendererWindow.openDevTools();
rendererWindow.webContents.send('capture-screenshot', path.join('e:', 'temp'));
});
rendererWindow.loadURL(
url.format({
pathname: path.join(__dirname, 'ScreenCapturer.jshtml.html'),
protocol: 'file:',
slashes: true
})
);
}
}
/*/********************************************************************///
/*///*/
ScreenCapturer.jshtml.js (the thml file loaded in the renderer browser window)
<html>
<body>
<script>require('./ScreenCapturer.jshtml.js')</script>
</body>
</html>
ScreenCapturer.jshtml.js (the js file loaded from the html file in the renderer process)
import { ipcRenderer, desktopCapturer, screen } from 'electron';
import path from 'path';
import fs from 'fs';
import moment from 'moment';
let mainSource;
function getMainSource(mainSource, desktopCapturer, screen, done) {
if(mainSource === undefined) {
const options = {
types: ['screen'],
thumbnailSize: screen.getPrimaryDisplay().workAreaSize
};
desktopCapturer.getSources(options, (err, sources) => {
if (err) return console.log('Cannot capture screen:', err);
const isMainSource = source => source.name === 'Entire screen' || source.name === 'Screen 1';
done(sources.filter(isMainSource)[0]);
});
} else {
done(mainSource);
}
}
function writeScreenshot(png, filePath) {
fs.writeFile(filePath, png, err => {
if (err) { console.log('Cannot write file:', err); }
return;
});
}
ipcRenderer.on('capture-screenshot', (evt, targetPath) => {
setInterval(() => {
getMainSource(mainSource, desktopCapturer, screen, source => {
const png = source.thumbnail.toPng();
const filePath = path.join(targetPath, `${moment().format('yyyyMMdd_HHmmss')}.png`);
writeScreenshot(png, filePath);
});
}, 3000);
});
I walked away from using the API's delivered by electron. I'd recommend using desktop-screenshot package -> https://www.npmjs.com/package/desktop-screenshot. This worked cross platform (linux, mac, win) for me.
Note on windows we need the hazardous package, because otherwise when packaging your electron app with an asar it won't be able to execute the script inside desktop-screenshot. More info on the hazardous package's page.
Below is how my code now roughly works, please don't copy/paste because it might not fit your solution!! However it might give an indication on how you could solve it.
/* ******************************************************************** */
/* MODULE IMPORTS */
import { remote, nativeImage } from 'electron';
import path from 'path';
import os from 'os';
import { exec } from 'child_process';
import moment from 'moment';
import screenshot from 'desktop-screenshot';
/* */
/*/********************************************************************///
/* ******************************************************************** */
/* CLASS */
export default class ScreenshotTaker {
constructor() {
this.name = "ScreenshotTaker";
}
start(cb) {
const fileName = `cap_${moment().format('YYYYMMDD_HHmmss')}.png`;
const destFolder = global.config.app('capture.screenshots');
const outputPath = path.join(destFolder, fileName);
const platform = os.platform();
if(platform === 'win32') {
this.performWindowsCapture(cb, outputPath);
}
if(platform === 'darwin') {
this.performMacOSCapture(cb, outputPath);
}
if(platform === 'linux') {
this.performLinuxCapture(cb, outputPath);
}
}
performLinuxCapture(cb, outputPath) {
// debian
exec(`import -window root "${outputPath}"`, (error, stdout, stderr) => {
if(error) {
cb(error, null, outputPath);
} else {
cb(null, stdout, outputPath);
}
});
}
performMacOSCapture(cb, outputPath) {
this.performWindowsCapture(cb, outputPath);
}
performWindowsCapture(cb, outputPath) {
require('hazardous');
screenshot(outputPath, (err, complete) => {
if(err) {
cb(err, null, outputPath);
} else {
cb(null, complete, outputPath);
}
});
}
}
/*/********************************************************************///