NodeJS Cluster: how to reduce data from workers in master? - node.js

Question

i'm new to nodejs, and what i want is to read data from database and compute.To make it faster, i use the nodejs cluster module.
there are tow global variables: pairMap and nameSet, and i allocate the jobs to worker in master process, and they do some computation works(to modify the map and set, just like the map-reduce )
however, it seems that the pairMap and nameSet are not modified and empty . (code in the doMasterAction )
(another strange thing is i console the data, and it did modified but in the end ,return to empty in the master process).
the data is as follows(i extract the main idea):
const Promise = require('bluebird');
const cluster = require('cluster');
const numCPUs = require('os').cpus().length;
const fs = Promise.promisifyAll(require('fs'))
const utils = {
mergeMap:(source,dest)=>{
for(let [key,value] of Object.entries(source)){
if(!dest.has(key)) dest.set(key,value);
for(let [type,arr] of Object.entries(value)){
const final = new Set([...dest.get[key][type],...arr])
dest.get[key][type] = final;
}
}
}
}
/**
* key: name1#group.com||name2#group.com
* value: {to: [id1,id2,id3],cc,bcc}
* #param row
* #param map
* #param nameSet
*/
function countLinks(res,map,nameSet) {
nameSet.add(res);
map.set(res,{ 'test': Math.floor(Math.random()*10+1)});
}
class hackingTeamPrepare {
constructor(bulk=100000,total = 1150000){
this.bulk = bulk;
this.count = Math.ceil(total / this.bulk);
const parallelArr = new Array(this.count).fill(0).map((v,i)=> i);
this.jobs = parallelArr.map(v=> 'key'+v);
this.pairMap = new Map();
this.nameSet = new Set();
this.bindThis();
}
bindThis(){
this.doWorkerAction = this.doWorkerAction.bind(this);
this.doMasterAction = this.doMasterAction.bind(this);
}
doMasterAction() {
const workers = [],result = {};
const self = this;
let count = 0,timeout;
for(let i=0;i<numCPUs;i++){
const worker = cluster.fork();
workers[i] = worker;
}
cluster.on('online', (worker) => {
worker.send(self.jobs.shift());
});
cluster.on('exit', function() {
if(self.jobs.length===0) return;
console.log('A worker process died, restarting...');
});
cluster.on('message',function (senderWorkder,info) {
const { workerId,jobIndex } = info;
result[jobIndex] = true;
console.log(`----worker ${workerId} done job: ${jobIndex}----`);
const finish = !self.jobs.length && Object.keys(result).length===self.count;
if(finish){
// -----------------!!here!!--------------------------**
console.log('-------finished-------',self.pairMap,self.nameSet); // Map {}, Set {}
for(let id in cluster.workers){
const curWorker = cluster.workers[id];
curWorker.disconnect();
}
}else{
if(!self.jobs.length) return;
senderWorkder.send(self.jobs.shift());
}
})
}
/**
* {[person1,person2]: {to,cc,bcc}}
*/
doWorkerAction() {
//Process为worker, receive from master
const self = this;
process.on('message',(sql)=>{
const jobPromise = Promise.resolve(sql).then(res => {
countLinks(res,self.pairMap,self.nameSet);
const data = {
workerId: process.pid,
jobIndex: sql,
}
// send to master
process.send(data);
}).catch(err=> {
console.log('-----query error----',err)
});
})
}
readFromPG(){
if(cluster.isMaster){
this.doMasterAction();
}else if (cluster.isWorker){
this.doWorkerAction();
}
}
init(){
this.readFromPG();
}
}
const test = new hackingTeamPrepare(2,10);
test.init();
anyone can help me with this?
i have tried to merge data manually in the master process,however the data sent by the worker.send seems to ignore the object in it.

Answer 1

In Node.js cluster, objects in memory are not shared between master and workers.
pairMap and nameSet exist separately in master and in every worker. When a worker modifies these objects, they change in the same worker (process), while remain unchanged in master and other workers.
To make your idea work, you need to maintain a single pairMap and a single nameSet inside the master process, send messages containing whatever data you need from workers to master, and update these objects using the received data.
Note that you cannot pass any object as a message from worker to master. If you need somewhat complex data, you'll need to send plain javascript objects (key-value pairs). For example, if you need to send a Map instance from worker to master, see the following functions taken from here:
// source - http://2ality.com/2015/08/es6-map-json.html
function mapToJson(map) {
return JSON.stringify([...map]);
}
function jsonToMap(jsonStr) {
return new Map(JSON.parse(jsonStr));
}
// send message using this example:
process.send(mapToJson(pairMap));
// receive message:
worker.on('message', message => console.log(jsonToMap(message)))