I have this monitor object that raises an event every time data is found, by means of an event handler:
monitor.on("data", data => { /* do something */ })
I would like to replace this pattern by using a generator:
for await(const data of monitor.iterate()) { /* do something */ }
I know I can do this:
async function monitorWrapper() {
const allData = await new Promise( resolve => {
const _allData = []
monitor.on("data", d => _allData.push(d))
monitor.on("end", () => resolve(_allData))
} )
yield *allData
}
and call it this way:
for await (const data of monitorWrapper()) { /* do something */ }
But this negates the whole of point of using yield since I have to wait for all elements to be available before processing.
In this context, my question is : Is there any pattern that allows to yield as the data events are triggered ?
Just ran into this problem myself.
const stream = new class { // mock event stream
async start() {
while (true) {
const ms = Math.random() * 100;
await new Promise(resolve => setTimeout(resolve, ms));
if (this.handler) this.handler(ms);
}
}
on(handler) { this.handler = handler; }
}
class Monitor {
events = []; // store events
resolve = () => {}; // resolves current outstanding promise
constructor(handler) {
handler(event => { // binds event listener
this.events.push(event); // add event
this.resolve(); // call resolve
});
}
async *stream() { // generator
while (true) {
let event;
while (event = this.events.shift()) yield event; // remove and yield
await new Promise(r => this.resolve = r); // wait for next
}
}
}
const monitor = new Monitor(handler => stream.on(handler)); // bind events
stream.start();
for await (const event of monitor.stream()) { console.log({ event }); }
I am not sure, but I think you can try this and let me know what happened.
//start is a thunk creator
function start(cb){
var c = cb();
c.next(); //start up!
return function(data){
c.next(data);
};
}
monitor.on('data', start(function(message){
var m = yield message;
console.log(m);
}));
Related
I am using a pool of workers to complete some CPU intensive tasks in Node.
However, I have a problem in my code below. When a task is first run, everything goes as expected, the pool is created, then is called which sends a message to a thread that runs the task successfully and responds with a message. However, on a second call, it seems to run the callback function of the worker (via the parentPort.postMessage) before the work in the threaded file is done. This seems to be an issue with using promises here? I see the "This run was complete in X ms" log before the "done" message shows. Why is this happening? Is something wrong with my workerPool class?
Am I not able to use any async/promise logic within the worker?
The version of Node I'm using is 14.
I followed largely the example set in this doc: https://nodejs.org/api/async_context.html#class-asyncresource
workerPool.js:
const { AsyncResource } = require("async_hooks");
const { EventEmitter } = require("events");
const path = require("path");
const { Worker } = require("worker_threads");
const kTaskInfo = Symbol("kTaskInfo");
const kWorkerFreedEvent = Symbol("kWorkerFreedEvent");
const { MONGODB_URI } = process.env;
class WorkerPoolTaskInfo extends AsyncResource {
constructor(callback) {
super("WorkerPoolTaskInfo");
this.callback = callback;
}
done(err, result) {
console.log("<<<<<<<<<<<");
this.runInAsyncScope(this.callback, null, err, result);
this.emitDestroy(); // TaskInfos are used only once.
}
}
class WorkerPool extends EventEmitter {
constructor(numThreads, workerFile) {
super();
this.numThreads = numThreads;
this.workerFile = workerFile;
this.workers = [];
this.freeWorkers = [];
for (let i = 0; i < numThreads; i++) this.addNewWorker();
}
addNewWorker() {
const worker = new Worker(path.resolve(__dirname, this.workerFile), {
workerData: { MONGODB_URI },
});
worker.on("message", (result) => {
// In case of success: Call the callback that was passed to `runTest`,
// remove the `TaskInfo` associated with the Worker, and mark it as free
// again.
worker[kTaskInfo].done(null, result);
worker[kTaskInfo] = null;
this.freeWorkers.push(worker);
this.emit(kWorkerFreedEvent);
});
worker.on("error", (err) => {
// In case of an uncaught exception: Call the callback that was passed to
// `runTest` with the error.
if (worker[kTaskInfo]) worker[kTaskInfo].done(err, null);
else this.emit("error", err);
// Remove the worker from the list and start a new Worker to replace the
// current one.
this.workers.splice(this.workers.indexOf(worker), 1);
this.addNewWorker();
});
this.workers.push(worker);
this.freeWorkers.push(worker);
this.emit(kWorkerFreedEvent);
}
runTest(data, callback) {
if (this.freeWorkers.length === 0) {
// No free threads, wait until a worker thread becomes free.
console.log("No free threads. Process queued.");
this.once(kWorkerFreedEvent, () => this.runTest(data, callback));
return;
}
const worker = this.freeWorkers.pop();
worker[kTaskInfo] = new WorkerPoolTaskInfo(callback);
worker.postMessage(data);
}
close() {
for (const worker of this.workers) worker.terminate();
}
}
module.exports = WorkerPool;
The index.js file where workers are called:
return new Promise(async (resolve, reject) => {
threadPool.runTest(
{ ...some data },
(err, result) => {
if (err) {
console.error("Bad error from test:", err);
reject(err)
}
const endTime = new Date();
// this callback is fired before the message is supposed to be posted...
console.log("Run Test Complete");
console.log(`This run took ${endTime - startTime} ms`);
resolve(true);
}
);
});
};
The code that actually runs in the worker:
const { parentPort, threadId, workerData } = require("worker_threads");
parentPort.on("message", async (data) => {
// await func1(...)
// await func2(...)
console.log("Done");
parentPort.postMessage('Done stuff);
});
Async programming in node.js is usually done with callbacks. I find callback-based code hard to read and reason about, which is why I'm using async & await whenever I can. This almost always works well and leads to robust code. However, rarely I'm wondering whether I'm making things more difficult than necessary. For example, how do you create a stream such that you can await its creation? More specifically, the result of the await should of course be the stream itself when things go well. When they don't, an appropriate exception should be thrown.
The best I could come up with is the function below. It feels very clunky and I'm wondering whether there is an easier way to do this?
import type { EventEmitter } from "events";
export const createStreamAsync = async <T extends EventEmitter>(create: () => T): Promise<T> => {
const result = create();
let onOpen = (): void => void 0;
let onError = (reason?: unknown): void => void reason;
try {
await new Promise<void>((resolve, reject) => {
onOpen = resolve;
onError = reject;
result.on("open", onOpen).on("error", onError);
});
return result;
} finally {
result.removeListener("open", onOpen);
result.removeListener("error", onError);
}
};
You'd use the function as follows:
import { createWriteStream } from "fs";
import { createStreamAsync } from "./createStreamAsync.js";
try {
const stream = await createStreamAsync(() => createWriteStream("~/whatever.txt"));
// Use stream ...
} catch (error: unknown) {
// Handle error
}
Readable streams have an AsyncIterator symbol, so they can be processed with for await ... of:
const readable = fs.createReadStream('file.txt');
for await (const chunk of readable) {
console.log(chunk);
}
You can listen to both an event and catch the error event (if any) with events.once:
const { once } = require('node:events');
const writable = fs.createWriteStream('file.txt');
try {
await once(writable, 'open');
// If the event emits data, `once` returns it as an array:
// const x = await once(...);
} catch (err) {
console.error('Could not open file', err);
}
This will automatically remove the listener, just like EventEmitter.once does.
event.on returns an AsyncIterator to handle multiple events:
const { on } = require('node:events');
const { exec } = require('node:child_process');
const child = exec(...);
try {
for await (const event of on(child, 'message')) {
// `event` is an array of 0 or more values.
console.log(event);
}
} catch (err) {
console.error(err);
}
Ima rookie using async/await but must now to use Redis-om. NN_walkd walks through a Redis database looking for loop-chains and does this by recursion. So the 2 questions I have is:
Am I calling the inner recursive NN_walkd calls correctly via async/await?
At runtime, the compSearchM proc is called first and seems to work (it gets 5 entries so it has to call NN_walkd 5 times). A NN_walkd is then recursively called, and then when it loops the 1st time it then calls compSearchK where the problems are. It seems to sit on the first Redis call in compSearchK (.search). Yet the code for compSearchK and compSearchM look basically identical.
main call
NN_walk = async function(req, db, cnode, pnode, chain, cb) {
var vegas, sneaker;
req.session.walk = [];
await NN_walkd(req, cnode, pnode, [], 1);
req.session.walk = null;
console.log('~~~~~~~~~~~~ Out of Walk ~~~~~~~~~~~~~~~');
cb();
};
redis.mjs
export class RedisDB {
constructor() {
...
this._companyRepo = ...
}
compSearchK(ckey) { // doesn't matter if I have a async or not here
return new Promise(async (resolve) => {
const sckey = await this._companyRepo.search()
.where('COMPANYKEY').equals(ckey)
.return.all();
if (sckey.length) {
const ttrr = await this._companyRepo.fetch(sckey[0].entityId);
resolve(ttrr.toJSON());
} else
resolve(null);
});
}
compSearchM(mkey) {
var tArr=[];
return new Promise(async (resolve) => {
const smkey = await this._companyRepo.search()
.where('MASTERKEY').equals(mkey)
.and('TBLNUM').equals(10)
.return.all();
if (smkey.length) {
for (var spot in smkey) {
const ttrr = await this._companyRepo.fetch(smkey[spot].entityId);
tArr.push(ttrr.toJSON());
}
resolve(tArr);
} else {
resolve(null);
}
});
}
walk.js
NN_walkd = async function(req, cnode, pnode, chain, lvl) {
...
if (cnode[1]) {
const sObj = await req.app.get('redis').compSearchK(cnode[1]);
if (sObj) {
int1 = (sObj.TBLNUM==1) ? null : sObj.CLIENTKEY;
(async () => await NN_walkd(req, [sObj.COMPANYKEY,int1], cnode, Array.from(chain), tlvl))()
}
} else {
const sArr = await req.app.get('redis').compSearchM(cnode[0]);
if (sArr.length) {
for (sneaker in sArr) {
(async () => await NN_walkd(req, [sArr[sneaker].COMPANYKEY,sArr[sneaker].CLIENTKEY], cnode, Array.from(chain), tlvl))()
}
} else {
console.log('no more links on this chain: ',cnode);
}
}
}
"doesn't matter if i have async or not here"
compSearchK(ckey) { // doesn't matter if I have a async or not here
return new Promise(async (resolve) => {
const sckey = await this._companyRepo.search()
.where('COMPANYKEY').equals(ckey)
.return.all();
if (sckey.length) {
const ttrr = await this._companyRepo.fetch(sckey[0].entityId);
resolve(ttrr.toJSON());
} else
resolve(null);
});
}
Of course it doesn't matter, because you're not using await inside of compSearchK!
You are using the explicit promise contructor anti-pattern. You should avoid it as it demonstrates lack of understanding. Here is compSearchK rewritten without the anti-pattern -
async compSearchK(ckey) {
// await key
const sckey =
await this._companyRepo.search()
.where('COMPANYKEY').equals(ckey)
.return.all();
// return null if key is not found
if (sckey.length == 0) return null;
// otherwise get ttrr
const ttrr = await this._companyRepo.fetch(sckey[0].entityId);
// return ttrr as json
return ttrr.toJSON();
}
I want to supply plugin to my services that will measure each call to redis and send metric with the response time.
how can I wrap ioredis to do this?
I thought to use Proxy but it's work to me only for async (promise) methods.
let handler = {
get: (target, name, receiver) => {
const startTime = Date.now();
return (...args) => new Promise((resolve, reject) => {
const apiMethod = Reflect.get(target, name, receiver);
const boundApiMethod = apiMethod.bind(target);
boundApiMethod(...args).then((data) => {
const duration = Date.now() - startTime;
metric({duration});
resolve(data)
}, (err) => {
rawMetering.reportError(err);
reject(err);
});
});
}
};
return new Proxy(redis, handler);
We were facing a very similar requirement recently, what we ended up doing is creating a proxy-class around ioredis as follows:
class RedisClient {
constructor() {
this.initClient(); // init your ioredis here
}
#MeasureTime()
async get(key) {
return this._ioredis.get(key);
}
// other methods like set here
}
As you can see we've defined a decorator MeasureTime which is really just a higher order function wrapping the call and performing the measurement:
function MeasureTime = () => (target, _propertyKey, descriptor) => {
const originalMethod = descriptor.value;
descriptor.value = async function (...args) {
const start = moment.now();
const result = await originalMethod.apply(this, args);
const duration = moment.now() - start;
// do something with the duration here
return result;
};
return descriptor;
}
I have to make a JSON RPC API that will have to support a big traffic and manage a postgreSQL database.
To do it I chose 'http' for the server and pg-promise for the database.
My problem is that I have some difficulty understanding and using the promises and async/wait, so I'm not sure I did it correctly
I put some code below
What I did
./server/server.js create an http server with requestHandler() as request handler. It does some checks and then calls async requestProcessor() to execute the method
The methods are defined in the repos (here a transaction in devices.js) as async and in my example below use await to wait for the required results
Some questions :
I have to define as async only the methods that should use await ?
In my SystemRepository, do I need to define 'InsertOneSystem' as async ?
How can I do a simple test script to test the load ? Like requests per second, ... ?
Thanks in advance !
A bit of code
server.js
const http = require('http');
const Database = require('../db');
const path = '/api/v1', port = 9000;
const methods = Database.methods;
/* hidden for brevity */
function sendResponse(res, response) {
if (response) {
const responseStr = JSON.stringify(response);
res.setHeader('Content-Type', 'application/json');
res.setHeader('Content-Length', responseStr.length);
res.write(responseStr);
} else {
/* hidden for brevity */
}
res.end();
}
const requestHandler = (req, res) => {
/* some checks, hidden for brevity */
const body = [];
req.on('data', (chunk) => {
body.push(chunk);
}).on('end', () => {
const bodyStr = Buffer.concat(body).toString();
// parse body en JSON
let request = JSON.parse(bodyStr);
requestProcessor(request).then((response) => {
sendResponse(res, response);
});
});
}
async function requestProcessor(request) {
let response = {
id: request.id,
jsonrpc: '2.0',
};
try {
response.result = await Promise.resolve(methods[request.method](request.params));
} catch (err) {
/* hidden for brevity */
}
return response;
}
const server = http.createServer(requestHandler);
server.listen(port, (err) => { /* hidden for brevity */ });
devices.js
'use strict';
/* hidden for brevity */
async function InsertOne(params) {
return Database.tx('Insert-New-Device', async function(transaction) {
let system = null, disks = null, cpus = null;
const query = pgp.helpers.insert(params.data.device, Collections.insert) + " RETURNING *";
let device = await transaction.one(query);
// if a system is present, insert with diviceId and return
if(params.data.system) {
params.data.system.deviceid = device.deviceid;
system = transaction.systems.InsertOne(params);
}
// same as system
if(params.data.disks) {
params.data.disks.deviceid = device.deviceid;
disks = transaction.disks.InsertOne(params);
}
// same as system
if(params.data.cpus) {
params.data.cpus.deviceid = device.deviceid;
cpus = transaction.cpus.InsertOne(params);
}
return {
device: device,
system: await system,
disks: await disks,
cpus: await cpus
}
})
.then(data => {
return data;
})
.catch(ex => {
console.log(ex)
throw new Error(ex);
});
}
/* hidden for brevity */
const DevicesRepository = {
InsertOne: InsertOne
};
module.exports = (db, pgpLib) => {
/* hidden for brevity */
return DevicesRepository;
}
systems.js
'use strict';
/* hidden for brevity */
async function InsertOneSystem(params) {
var system = params.data.system;
system.archid=2;
system.distributionid=3;
var query = pgp.helpers.insert(system, Collections.insert);
if(params.return) query += " RETURNING *";
return Database.one(query)
.then(data => {
return data;
})
.catch(ex => {
throw new Error(ex);
});
}
/* hidden for brevity */
const SystemsRepository = {
InsertOne: InsertOneSystem
};
module.exports = (db, pgpLib) => {
/* hidden for brevity */
return SystemsRepository;
}
I have to define as async only the methods that should use await ?
Have to - yes. But you should use async on all methods that return a promise, it's just a nice coding style, especially in TypeScript.
In my SystemRepository, do I need to define InsertOneSystem as async ?
You don't have to, but the same as above, it's a good coding style ;)
How can I do a simple test script to test the load ? Like requests per second, ... ?
I'm not answering that right now, as it is a whole separate area that deserves a separate question. You should investigate it yourself, how to test HTTP services load.
A little code improving, as you have plenty of redundancies:
async function InsertOne(params) {
return Database.tx('Insert-New-Device', async t => {
let system = null, disks = null, cpus = null;
const query = pgp.helpers.insert(params.data.device, Collections.insert) + " RETURNING *";
let device = await t.one(query);
// if a system is present, insert with diviceId and return
if(params.data.system) {
params.data.system.deviceid = device.deviceid;
system = await t.systems.InsertOne(params);
}
// same as system
if(params.data.disks) {
params.data.disks.deviceid = device.deviceid;
disks = await t.disks.InsertOne(params);
}
// same as system
if(params.data.cpus) {
params.data.cpus.deviceid = device.deviceid;
cpus = await t.cpus.InsertOne(params);
}
return {device, system, disks, cpus};
})
.catch(ex => {
console.log(ex); // it is better use "pg-monitor", or handle events globally
throw ex;
});
}