NodeJS no code running after worker code finishes - node.js

I'm having an issue with my worker code, currently i have some code that scans through two database tables and finds some matches and then adds some data from one table to the other creating a new table. This is a large set of data so i'm using worker threads to speed this up.
This is all working fine however once the worker threads are complete no other code runs i've tried adding the function LogData everywhere i can and it will not run i've even add the console.log("Finished building merge table") and that doesn't run either. Even the parentResolve does happen as i don't see the console.log("parentResolve") message.
if anyone can help me I would really appreciate it.
const calculateFactorialwithWorker = async () => {
const SCCM = await ProgramDev.find({ "program name": { $not: { $regex: ".*[(]KB*[)]*" } } }).limit(8000)
const sccmLength = SCCM.length
mongoose.connection.close()
return new Promise(async (parentResolve, parentReject) => {
const numbers = [...new Array(sccmLength)].map((_, i) => i);
const segmentSize = Math.ceil(sccmLength / userCPUCount);
const segments = [];
for (let segmentIndex = 0; segmentIndex < userCPUCount; segmentIndex++) {
const start = segmentIndex * segmentSize;
const end = start + segmentSize;
const segment = numbers.slice(start, end)
segments.push(segment);
}
try {
const results = await Promise.all(
segments.map(
segment =>
new Promise((resolve, reject) => {
const worker = new Worker(workerPath, {
workerData: segment,
});
worker.on('message', resolve);
worker.on('error', reject);
worker.on('exit', (code) => {
if (code !== 0)
reject(new Error(`Worker stopped with exit code ${code}`));
});
})
));
parentResolve(() => {
console.log("parentResolve")
})
} catch (e) {
parentReject(e)
}
});
};
calculateFactorialwithWorker().then(() => {
console.log("Finished building merge table")
LogData
})

Add if else block in worker exit event. When exit fired with code === 0 , there is no resolve/reject to handle it. The promises will not be resolved/rejected.
Ref. https://nodejs.org/api/worker_threads.html#worker_threads_event_exit
Also, I rewrite your codes a bit because some promises wrapper is unnecessary.
const calculateFactorialwithWorker = async () => {
try {
const SCCM = await ProgramDev.find({
"program name": { $not: { $regex: ".*[(]KB*[)]*" } },
}).limit(8000);
const sccmLength = SCCM.length;
const numbers = [...new Array(sccmLength)].map((_, i) => i);
const segmentSize = Math.ceil(sccmLength / userCPUCount);
const segments = [];
for (let segmentIndex = 0; segmentIndex < userCPUCount; segmentIndex++) {
const start = segmentIndex * segmentSize;
const end = start + segmentSize;
const segment = numbers.slice(start, end);
segments.push(segment);
}
const promises = segments.map(
segment =>
new Promise((resolve, reject) => {
const worker = new Worker(workerPath, {
workerData: segment,
});
worker.on("message", resolve);
worker.on("error", reject);
worker.on("exit", code => {
if (code !== 0) {
reject(new Error(`Worker stopped with exit code ${code}`));
} else {
resolve();
}
});
})
);
await Promise.all(promises);
} catch (err) {
throw new Error(err);
}
};
calculateFactorialwithWorker()
.then(() => {
console.log("Finished building merge table");
LogData();
})
.catch(console.log)
.finally(() => {
mongoose.connection.close();
});

Related

how to work with response object in nodejs stream, exceljs and worker thread

I am using worker thread and stream at same time in node JS project. At initial I was not able to pass res object through main process to worker thread. I saw many stackoverflow question and solution and wrote a solution which works great. I created a Readable stream in main thread and writable stream in worker thread. while doing this, I have done a huge calculation from more than 10 table and export data which takes nearly 1 minutes for processing.
code:
router.get("/downloadAll", (req, res) => {
new Promise((resolve, reject) => {
const promise = [];
promise.push(Dashboard.DUser());
promise.push(Dashboard.DDUser());
promise.push(Dashboard.DDLUser());
promise.push(Dashboard.Din());
promise.push(Dashboard.Str());
promise.push(Dashboard.R());
promise.push(Dashboard.Q());
Promise.all(promise).catch(err => err)
.then(results => {
const worker = new Worker(`${process.cwd()}/src/route/modules/dashboard/worker.js`, {
workerData: { results }
});
const fileHeaders = [
{
name: "Content-Type",
value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
}
];
res.setHeader("Content-Disposition", `attachment; filename="Stream.xlsx`);
fileHeaders.forEach(header => res.setHeader(header.name, header.value));
const readStream = new Readable({
read() {}
});
readStream.pipe(res);
worker.on("message", message => {
readStream.push(message);
});
worker.on("exit", code => {
console.log("exit", code);
resolve(true);
//if (code !== 0) reject(new Error(`stopped with ${code} exit code`));
});
});
})
.then(() => res.end())
.catch(err => console.log(err));
});
WORKER THREAD:
const { workerData, parentPort } = require("worker_threads");
const { Writable } = require("stream");
const Excel = require("exceljs");
const writableStream = new Writable();
// writableStream.on("message", () => {});
writableStream._write = (chunk, encoding, next) => {
parentPort.postMessage(chunk);
next();
};
const createWorkbook = () => {
const workbook = new Excel.stream.xlsx.WorkbookWriter({
stream: writableStream, // stream to server response
useStyles: true // not sure about this one, check with it turned off.
});
workbook.title = "Serious";
workbook.creator = "SS";
workbook.created = new Date();
return workbook;
};
const createSheet = workbook => {
workerData.results.forEach((result, index) => {
const worksheet = workbook.addWorksheet(result.title, {
properties: { outlineLevelCol: 1 }
});
worksheet.columns = Object.keys(result.data[0]).map(item => {
return { header: item, key: item };
});
result.data.forEach(row => worksheet.addRow(row).commit);
});
};
const workbook = createWorkbook();
createSheet(workbook);
workbook.commit();
The above code works fine and is fast for small calculation. when I have huge computation it is showing processing for 1 minutes and finish processing and download the xls file. so i updated the code to:
router.get("/downloadAll", (req, res) => {
const worker = new Worker(`${process.cwd()}/src/worker/worker.js`);
const fileHeaders = [
{
name: "Content-Type",
value: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
}
];
const today = new Date();
res.setHeader(
"Content-Disposition",
`attachment; filename=Q-${today.getFullYear()}${String(today.getMonth() + 1).padStart(2, "0")}${String(
today.getDate()
).padStart(2, "0")}.xlsx`
);
fileHeaders.forEach(header => res.setHeader(header.name, header.value));
const readStream = new Readable({
read() {}
});
readStream.pipe(res);
worker.on("message", message => {
readStream.push(message);
});
worker.on("exit", code => {
console.log("exit", code);
res.end();
//if (code !== 0) reject(new Error(`stopped with ${code} exit code`));
});
});
and worker thread code:
const { workerData, parentPort } = require("worker_threads");
const { Writable } = require("stream");
const Excel = require("exceljs");
const { resolve } = require("path");
const db = require(`${process.cwd()}/src/modules/db.module`);
const Dashboard = require(`${process.cwd()}/src/route/modules/dashboard.model`);
const promise = [];
promise.push(Dashboard.DUser());
promise.push(Dashboard.DDUser());
promise.push(Dashboard.DDLUser());
promise.push(Dashboard.Din());
promise.push(Dashboard.Str());
promise.push(Dashboard.R());
promise.push(Dashboard.Q());
Promise.all(promise).catch(err => err)
.then(results => { const writableStream = new Writable();
// writableStream.on("message", () => {});
writableStream._write = (chunk, encoding, next) => {
console.log(chunk.toString());
parentPort.postMessage(chunk);
next();
};
const createWorkbook = () => {
const workbook = new Excel.stream.xlsx.WorkbookWriter({
stream: writableStream, // stream to server response
useStyles: true // not sure about this one, check with it turned off.
});
workbook.creator = "ss";
workbook.created = new Date();
return workbook;
};
const createSheet = workbook => {
results.forEach((result, index) => {
// console.log(result);
const worksheet = workbook.addWorksheet(result.title, {
properties: { outlineLevelCol: 1 }
});
worksheet.columns = Object.keys(result.data[0]).map(item => {
return { header: item, key: item };
});
result.data.forEach(row => worksheet.addRow(row).commit);
});
};
The above code doesnot work correctly. I can get the data from callback from promise but when its downloading its shows 300kb , 200b,1byte and ends to 0 but it does not download.
if I try to insert the promise inside createsheet then i am getting error:
Error [ERR_UNHANDLED_ERROR]: Unhandled error. ({ message: 'queue closed', code: 'QUEUECLOSED', data: undefined })
code:
const createSheet = workbook => {
let promise = [];
/**
* get count of all the user list
*/
promise.push(Dashboard.DDPro());
Promise.all(promise)
.then(results => {
results.forEach((result, index) => {
console.log(result);
const worksheet = workbook.addWorksheet(result.title, {
properties: { outlineLevelCol: 1 }
});
worksheet.columns = Object.keys(result.data[0]).map(item => {
return { header: item, key: item };
});
result.data.forEach(row => worksheet.addRow(row).commit);
});
})
.catch(err => console.log(err));
};
can any body helps me solve the problem.

Node.js split file lines

I want to write a script that divides the lines read from the file into packages of 25, unfortunately the sample package returns 40 codes. I would like to do so that, for example, he divided me into packages of 25 items. I mean, I have, for example, 60 codes, this creates 2 packages of 25, and one with 10 codes. Unfortunately, I can't handle it.
const fs = require('fs');
fs.readFile('code.txt', function (err, data) {
if (err) throw err;
const array = data.toString().split("\n");
let count = 0;
let items = [];
for (let i in array) {
items.push({
PutRequest: {
Item: {
code: array[i]
}
}
});
let params = {
RequestItems: {
'TABLE_NAME': items
}
};
if (count === 25) {
dynamoDB.batchWrite(params, function (err, data) {
if (err) {
console.log(err);
} else {
count = 0;
items = [];
}
});
}else{
count++;
}
}
});
code.txt content
https://0bin.net/paste/NA8-4hkq#1Ohwt5uUkQqE0YscwnxTX2gxEqlvAUVKp1JRipBCsZg
Any idea what I do wrong?
Your dynamoDB.batchWrite() is asynchronous. Thus its callback is executed only after the loop has completed. So items and count are never reset ...
The easiest would be, if you could switch to an promise based approach like the following
const BATCHSIZE = 25;
const fs = require('fs').promises;
async function batchLoad() {
const lines = (await fs.readFile("code.txt", "utf-8")).split("\n");
while (lines.length > 0) {
const items = lines.splice(0, BATCHSIZE).map(l => ({PutRequest: {Item: { code: l }}}));
const params = { RequestItems: { TABLE_NAME: items}};
await new Promise((resolve, reject) => {
dynamoDb.batchWrite(params, (err) => {
if (err) return reject(err);
resolve();
});
});
}
}
A callback based approach could look like this
const BATCHSIZE = 25;
fs.readFile("code.txt", "utf-8", (err, data) => {
const lines = data.split("\n");
function writeBatch() {
if (!lines.length) return;
const items = lines.splice(0, BATCHSIZE).map(l => ({PutRequest: {Item: { code: l }}}));
const params = { RequestItems: { TABLE_NAME: items}};
dynamoDb.batchWrite(params, err => {
if (err) ...
else writeBatch();
});
}
writeBatch();
}
The function writeBatch takes a certain number of lines from your original array and writes them into the database. Only afer the write into the DB was successful, it recursively calls itself and handles the next batch. But be aware, that this approach may exceed the maximum call stack size and throw an error.
You can also make either of this approaches not manipulate the lines array (which may be quite expensive), but just get out the current slice
const BATCHSIZE = 25;
const fs = require('fs').promises;
async function batchLoad() {
const lines = (await fs.readFile("code.txt", "utf-8")).split("\n");
let currentIndex = 0;
while (currentIndex < lines.length) {
const items = lines.slice(currentIndex, currentIndex + BATCHSIZE).map(l => ({PutRequest: {Item: { code: l }}}));
const params = { RequestItems: { TABLE_NAME: items}};
await new Promise((resolve, reject) => {
dynamoDb.batchWrite(params, (err) => {
if (err) return reject(err);
resolve();
});
});
currentIndex += BATCHSIZE;
}
}
and
const BATCHSIZE = 25;
fs.readFile("code.txt", "utf-8", (err, data) => {
const lines = data.split("\n");
function writeBatch(currentIndex) {
if (currentIndex >= lines.length) return;
const items = lines.slice(currentIndex, currentIndex + BATCHSIZE).map(l => ({PutRequest: {Item: { code: l }}}));
const params = { RequestItems: { TABLE_NAME: items}};
dynamoDb.batchWrite(params, err => {
if (err) ...
else writeBatch(currentIndex + BATCHSIZE);
});
}
writeBatch(0);
}
To prevent stumbling into a maximum callstack exception you may also add the next batch to the eventloop and not call it recursively. Ie
dynamoDb.batchWrite(params, err => {
if (err) ...
else setTimeout(()=> { writeBatch(currentIndex + BATCHSIZE);}, 0);
});
This way you won't build up a massive callstack from recursive calls.
To keep track of how many records are already saved to the db you could simply store the current counter in a file. When you restart the process, load that file and check how many lines to skip. Don't forget to remove the file, once all records have been saved ... For example with the first approach:
const BATCHSIZE = 25;
const fs = require('fs').promises;
async function batchLoad() {
const lines = (await fs.readFile("code.txt", "utf-8")).split("\n");
const skipLines = 0;
try {
skipLines = +(await fs.readFile("skip.txt", "utf-8"));
if (isNaN(skipLines)) skipLines = 0;
lines.splice(0, skipLines);
} catch (e) {
skipLines = 0;
}
while (lines.length > 0) {
const items = lines.splice(0, BATCHSIZE).map(l => ({PutRequest: {Item: { code: l }}}));
const params = { RequestItems: { TABLE_NAME: items}};
await new Promise((resolve, reject) => {
dynamoDb.batchWrite(params, (err) => {
if (err) return reject(err);
resolve();
});
});
skipLines += BATCHSIZE;
await fs.writeFile("skip.txt", `${skipLines}`);
}
try {
await fs.unlink("skip.txt");
} catch (e) {
}
}

Socket Hangup error in Nodejs for multiple API calls

I am trying to fetch a list of all companies listed in stock market from an external API, and after getting the list, I am trying to fetch all details regarding individual companies including graph data. It was all working fine. However, today I am getting socket hangup error. I have tried going through other posts here in stackoverflow. However, none of them works.
const request = require('request');
const fetchAPI = apiPath => {
return new Promise(function (resolve, reject) {
request(apiPath, function (error, response, body) {
if (!error && response.statusCode == 200) {
resolve(body);
} else {
reject(error);
}
});
});
}
// get list of all companies listed in
const fetchCompanyDetails = () => {
return new Promise(function (resolve, reject) {
let details = [];
fetchAPI('https://api//')
.then(res => {
res = JSON.parse(res)
details.push(res);
resolve(details);
})
.catch(err => {
console.log("error at fetchcompany details" + err);
})
});
}
const getDateAndPriceForGraphData = (graphData) => {
let res = []
graphData.forEach(data => {
let d = {}
d["x"] = new Date(data.businessDate).getTime() / 1000
d["y"] = data.lastTradedPrice
res.push(d)
})
return res
}
// get graph data for individual assets
const getGraphDataForAssets = (assetID) => {
return new Promise((resolve, reject) => {
let details = {};
fetchAPI(`https://api/${assetID}`)
.then(async (res) => {
res = JSON.parse(res)
let data = await getDateAndPriceForGraphData(res)
details = data
resolve(details);
})
.catch(err => {
console.log("error at getGraphDataForAssets" + err);
})
});
}
// fetch data about individual assets
const fetchAssetDetailsOfIndividualCompanies = (assetID) => {
return new Promise((resolve, reject) => {
let details = {"assetData" : {}, "graphData": {}};
fetchAPI(`https://api/${assetID}`)
.then(async (res1) => {
res1 = JSON.parse(res1)
details["assetData"] = res1
// get graph data
var graphData = await getGraphDataForAssets(assetID)
details["graphData"] = graphData
resolve(details);
})
.catch(err => {
console.log("error at fetchAssetDetailsOfIndividualCompanies" + err);
reject(err)
})
});
}
// returns list of details of all tradeable assets (Active and Suspended but not delisted)
const fetchDetailsForEachCompany = async (companyList) => {
let result = []
await Promise.all(companyList.map(async (company) => {
try {
// return data for active and suspended assets
if(company.status != "D") {
let companyData = await fetchAssetDetailsOfIndividualCompanies(company.id)
result.push(companyData)
}
} catch (error) {
console.log('error at fetchDetailsForEachCompany'+ error);
}
}))
return result
}
exports.fetchAssetDetails = async () => {
let companyDetails = await fetchCompanyDetails()
let det = await fetchDetailsForEachCompany(companyDetails[0])
return det
}
To expand on what I meant with not needing those new Promise()s, this would be an idiomatic async function refactoring for the above code.
I eliminated getGraphDataForAssets, since it was eventually not used; fetchAssetDetailsOfIndividualCompanies fetched the same data (based on URL, anyway), and then had getGraphDataForAssets fetch it again.
const request = require("request");
function fetchAPI(apiPath) {
return new Promise(function (resolve, reject) {
request(apiPath, function (error, response, body) {
if (!error && response.statusCode === 200) {
resolve(body);
} else {
reject(error);
}
});
});
}
async function fetchJSON(url) {
return JSON.parse(await fetchAPI(url));
}
async function fetchCompanyDetails() {
return [await fetchAPI("https://api//")];
}
function getDateAndPriceForGraphData(graphData) {
return graphData.map((data) => ({
x: new Date(data.businessDate).getTime() / 1000,
y: data.lastTradedPrice,
}));
}
// fetch data about individual assets
async function fetchAssetDetailsOfIndividualCompanies(assetID) {
const assetData = await fetchJSON(`https://api/${assetID}`);
const graphData = getDateAndPriceForGraphData(assetData);
return { assetID, assetData, graphData };
}
// returns list of details of all tradeable assets (Active and Suspended but not delisted)
async function fetchDetailsForEachCompany(companyList) {
const promises = companyList.map(async (company) => {
if (company.status === "D") return null;
return fetchAssetDetailsOfIndividualCompanies(company.id);
});
const results = await Promise.all(promises);
return results.filter(Boolean); // drop nulls
}
async function fetchAssetDetails() {
const companyDetails = await fetchCompanyDetails();
return await fetchDetailsForEachCompany(companyDetails[0]);
}
exports.fetchAssetDetails = fetchAssetDetails;

Handling promises inside the forEach loop

I am trying to run a series of tasks. Each task is dynamic, and could have different rules to follow. This will be executed on AWS-Lambda.
I have an array of JSON. It has a body with task name in it, and it also has attributes.
I need to dynamically load a javascript file with the name inside the body.
I need to wait until all is finished inside that task. Or it failed (regardless where). If the fail happens, I will need to write that data inside the current record inside the forEach loop.
I have old issue, where my forEach is finished first without waiting for the task to complete.
This is the forEach loop:
const jobLoader = require('./Helpers/jobLoader');
event.Records.forEach(record => {
const { body: jobName } = record;
const { messageAttributes } = record;
const job = jobLoader.loadJob(jobName);
job.runJob(messageAttributes).then(res => {
console.log('Show results');
return; // resume another record from forEach
}).catch(err => {
record.failed = true;
record.failureMessage = err.message;
console.log('I errored');
});
console.log('All Done');
});
The problem is that message All Done is printed, and then the message show results is printed. I get results from the database once it comes for execution.
This is the file that loads a task:
exports.loadJob = (jobName) => {
const job = require(`../Tasks/${jobName}`);
return job;
};
This is the file that contains actual task:
const mySqlConnector = require('../Storage/mySql');
exports.runJob = async (params) => {
let payload = {};
let dataToSend = await getUserName(params.userId.stringValue);
payload.dataToSend = dataToSend;
let moreDataToSend = await getEvenMoreData(params.userId.stringValue);
payload.moreDataToSend = moreDataToSend;
return await sendData(payload);
};
const getUserName = async (userId) => {
const query = 'SELECT * FROM user_data';
return await mySqlConnector.handler(query);
};
const getEvenMoreData = async (userId) => {
const query = 'SELECT * FROM user_data';
return await mySqlConnector.handler(query);
};
const sendData = (payload) => {
//this should be Axios sending data
};
And this is the mySql connector itself:
const mysql = require('promise-mysql');
exports.handler = async (query) => {
return mysql.createConnection({
host : '127.0.0.1',
user : 'root',
password : '',
database: 'crm'
}).then(conn =>{
let result = conn.query(query);
conn.end();
return result;
}).then(rows => {
//console.log("These are rows:" + rows);
return rows;
}).catch(error => {
return error;
});
};
The task file can have any number of things it needs to complete, which will be different when I start adding tasks.
I need that job.runJob completes, or that it catches an error, from whatever location it originated, so I can continue with the forEach.
I have tried using map and what not, but the end result is always the same.
What am I doing wrong?
You can use Promise.all method :
const promises = event.Records.map(record => {
const { body: jobName } = record;
const { messageAttributes } = record;
const job = jobLoader.loadJob(jobName);
return job.runJob(messageAttributes).then(res => {
console.log('Show results', res);
}).catch(err => {
record.failed = true;
record.failureMessage = err.message;
console.log('I errored');
throw new Error('Your error !');
});
});
try {
const results = await Promise.all(promises);
console.log('All done');
} catch (e) {
console.log('Something has an error', e);
}
don't forget to make your function async !
I managed to solve it, and still keep details about the execution:
Something like this:
for (let prop in event.Records){
const { body: jobName } = event.Records[prop];
const { messageAttributes } = event.Records[prop];
const job = jobLoader.loadJob(jobName);
await job.runJob(messageAttributes).then(res => {
console.log('Show results', res);
}).catch(err => {
event.Records[prop].failed = true;
event.Records[prop].failed = err.message;
console.log('I errored');
});
}

how to stop process nodejs within promises

I've created nodejs to trigger(with cronjobs) firebase realtime database as follow:
var db = admin.database();
var ref = db.ref('myusers');
var promises = [];
function updateUnlocked(isLocked, locked, msisdn) {
return new Promise(function (resolve, reject) {
if (isLocked === 1) {
var startDate = moment(locked);
var endDate = moment();
var result = endDate.diff(startDate, 'minutes');
if (result > 5) {
var ref = db.ref('myusers/' + msisdn);
ref.update({isLocked: 2});
}
}
resolve('done');
});
}
ref.once('value', function(snapshot) {
snapshot.forEach(childSnapshot => {
promises.push(updateUnlocked(childSnapshot.val().isLocked, childSnapshot.val().locked, childSnapshot.key));
});
});
Promise.all(promises).then(function(data) {
console.log(data);
}).catch(function(err) {
console.log('error');
});
Please let me know where can I add process.exit(). Thanks.
You must wait for the "once" callback to get executed. Else the promise array is empty and the process could exit immediately.
var db = admin.database();
var ref = db.ref('myusers');
function updateUnlocked(isLocked, locked, msisdn) {
...
}
ref.once('value', function(snapshot) {
const promises = snapshot.map(childSnapshot => {
return updateUnlocked(childSnapshot.val().isLocked, childSnapshot.val().locked, childSnapshot.key);
})
Promise.all(promises).then(() => {
console.log('done')
process.exit(0)
}).catch(err => {
console.log('error', err)
process.exit(1)
})
});
Demonstrating the control flow.
setTimeout(() => {
const x = [1, 2, 3]
const promises = x.map(i => {
return new Promise(resolve => resolve(i))
})
Promise.all(promises).then(() => {
console.log('done. process.exit(0) here')
})
}, 200)
If you want to exit on successful completion then refer below code:
Promise.all(promises).then(function(data) {
console.log(data);
process.exit(0);
}).catch(function(err) {
console.log('error');
});
If you want to exit on error as well then add process.exit(1) in catch block.

Resources