PM2 & Puppeteer Watch Restarting - node.js

I have a puppeteer script which I am using to produce an export from a reporting tool that I use (called pivot.js):
const fs = require('fs');
const path = require('path');
const events = require('events');
const puppeteer = require('puppeteer');
let eventEmitter = new events.EventEmitter();
const directoryPath = "./storage/"; /* A path to the storage of exported files */
((directoryPath) => {
fs.mkdir(path.resolve(path.resolve(),
directoryPath.replace(/^\.*\/|\/?[^\/]+\.[a-z]+|\/$/g, '')), { recursive: true }, error => {
if (error) console.error(error);
});
})(directoryPath); /* Creating a storage folder for exported files (if such a folder doesn't exist yet) */
(async () => {
eventEmitter.once('reportcomplete', () => {
/*
All changes should be made within this function.
Available methods:
- setReport (https://www.flexmonster.com/api/setreport/)
- exportTo (https://www.flexmonster.com/api/exportto/)
The exportTo method takes two parameters: type and params.
Callback function will be ignored.
Possible destination types:
- plain (the file will be saved by the path defined as a value of the "directoryPath" variable)
- server (the file will be exported to the server)
Available events (use "eventEmitter" to manage events):
- ready (https://www.flexmonster.com/api/ready/)
- reportcomplete (https://www.flexmonster.com/api/reportcomplete/)
- exportcomplete (https://www.flexmonster.com/api/exportcomplete/)
Additional methods and events can be added using the template.
*/
eventEmitter.once('reportcomplete', () => { /* Exporting when the report is ready */
exportTo("csv");
exportTo("html");
exportTo("pdf");
exportTo("image");
exportTo("excel");
});
let exportCount = 0;
eventEmitter.on('exportcomplete', () => {
exportCount++;
if (exportCount == 5) browser.close(); /* Closing the browser when all the exports are complete */
});
setReport({
dataSource: {
filename: 'https://cdn.flexmonster.com/data/data.json'
}
});
});
const browser = await puppeteer.launch(); /* Launching the headless browser */
const page = await browser.newPage(); /* Creating a new page */
/* A function to set a report for the component dynamically */
function setReport(report) {
page.evaluate(report => {
flexmonster.setReport(report);
}, report)
}
/* This code is responsible for the export itself. It supports five export formats:
.html, .xlsx, .pdf, .csv, and .png. */
function exportTo(type, params) {
page.evaluate((type, params) => {
type = type.toLowerCase();
if (params) {
if (params.destinationType != "plain" && params.destinationType != "server")
params.destinationType = "plain";
}
else params = { destinationType: "plain" };
if (!params.filename) params.filename = "pivot";
flexmonster.exportTo(type, params, (result) => {
switch (type) {
case "pdf":
result.data = result.data.output();
break;
case "excel":
result.data = Array.from(result.data);
break;
case "image":
result.data = result.data.toDataURL();
break;
}
exportHandler(result);
});
}, type, params);
}
await page.exposeFunction('exportHandler', (result) => {
switch (result.type) {
case "excel":
result.data = Buffer.from(result.data);
result.type = "xlsx";
break;
case "image":
result.data = Buffer.from(result.data.replace(/^data:image\/\w+;base64,/, ""), 'base64');
result.type = "png";
break;
}
fs.writeFile(`${directoryPath}${result.filename}.${result.type}`, result.data, 'utf8', error => {
if (error) console.log(error);
});
});
/* This code adds functions to emit ready, reportcomplete, and exportcomplete events for the browser
when called. This approach allows us to handle the component's events in the browser's scope. */
await page.exposeFunction('onReady', () => {
eventEmitter.emit('ready')
});
await page.exposeFunction('onReportComplete', () => {
eventEmitter.emit('reportcomplete')
});
await page.exposeFunction('onExportComplete', () => {
eventEmitter.emit('exportcomplete')
});
/* Reading the file with the component and setting it as the browser page's contents */
await page.setContent(fs.readFileSync('index.html', 'utf8'));
/* This code runs in the page's scope, subscribing the browser window to the component's ready,
reportcomplete, and exportcomplete events */
await page.evaluate(() => {
window.addEventListener('ready', () => window.onReady());
window.addEventListener('reportcomplete', () => window.onReportComplete());
window.addEventListener('exportcomplete', () => window.onExportComplete());
});
})();
I am then using PM2 to watch the file, allowing me to swap out the code used to produce different reports, using:
pm2 start pivot.js --watch
The issue I have is that whenever I delete the contents of the storage folder (which the script writes into), a new export appears straight away. Almost as if the script is continuously being called, or PM2 is being restarted.
Both of the logs for PM2 are completely blank. But after running:
pm2 show 0
I receive the following:
│ status │ stopping │
│ name │ pivot │
│ namespace │ default │
│ version │ 1.0.0 │
│ restarts │ 1957 │
│ uptime │ 0 │
│ script path │ C:\Users\admin\Documents\Windows Puppeteer\pivot.js │
│ script args │ N/A │
│ error log path │ C:\Users\admin\.pm2\logs\pivot-error.log │
│ out log path │ C:\Users\admin\.pm2\logs\pivot-out.log │
│ pid path │ C:\Users\admin\.pm2\pids\pivot-0.pid │
│ interpreter │ node │
│ interpreter args │ N/A │
│ script id │ 0 │
│ exec cwd │ C:\Users\admin\Documents\Windows Puppeteer │
│ exec mode │ fork_mode │
│ node.js version │ 14.15.1 │
│ node env │ N/A │
│ watch & reload │ ✔ │
│ unstable restarts │ 0 │
│ created at │ 2020-11-30T01:24:27.461Z │
I hope you can help.

The issue is that the puppeteer script (named pivot.js) dumps the returning file in a folder called "storage". Storage is within the same directory as pivot.js, meaning that whilst PM2 monitors that directory, it is creating an infinite loop. The solution is to use the ignore watch option.
Creating a ecosystem file as such:
module.exports = {
apps : [{
script: 'pivot.js',
watch: '.',
ignore_watch : ["node_modules", "storage"]
}],
...
};
Or using:
pm2 start pivot.js --watch --ignore-watch="storage"
In my examples above will resolve the problem.

Related

How can I print Cypress base Url in the `Git bash CLI console`

How can I print Cypress base Url in the Git bash CLI console while running the Cypress test ? Could someone please advise ?
When running headless, the browser logs don't show in the terminal, but those from the Cypress node process (aka cy.task()) do show up.
I assume the baseUrl is changing during the test, here is an example that does that and logs the current value using a task.
The configured value is http://localhost:3000, the test changes it to http://localhost:3001.
cypress.config.js
const { defineConfig } = require('cypress')
module.exports = defineConfig({
e2e: {
setupNodeEvents(on, config) {
console.log('Printing baseUrl - during setup', config.baseUrl)
on('task', {
logBaseUrl(baseUrl) {
console.log('Printing baseUrl - from task', baseUrl)
return null
}
})
},
baseUrl: 'http://localhost:3000'
},
})
test
it('logs baseUrl to the terminal in run mode', () => {
console.log('Printing baseUrl - directly from test', Cypress.config('baseUrl')) // no show
Cypress.config('baseUrl', 'http://localhost:3001')
cy.task('logBaseUrl', Cypress.config('baseUrl'))
})
terminal
Printing baseUrl - during setup http://localhost:3000
====================================================================================================
Running: log-baseurl.cy.js (1 of 1)
Printing baseUrl - from task http://localhost:3001
√ logs baseUrl to the terminal in run mode (73ms)
1 passing (115ms)
(Results)
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Tests: 1 │
│ Passing: 1 │
│ Failing: 0 │
│ Pending: 0 │
│ Skipped: 0 │
│ Screenshots: 0 │
│ Video: true │
│ Duration: 0 seconds │
│ Spec Ran: log-baseurl.cy.js │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
====================================================================================================
(Run Finished)
Spec Tests Passing Failing Pending Skipped
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ √ log-baseurl.cy.js 108ms 1 1 - - - │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
√ All specs passed! 108ms 1 1 - - -
Done in 18.49s.
You can use a nodejs command console.log(Cypress.config().baseUrl).
That does not require Git Bash though, only nodejs installed on Your Windows.
Or you can add in your tests cy.config().baseUrl.

Jobs doesn't get picked up by #Processor in Nest.js BullQueue

I'm trying to implement a queue using the BullQueue package in nest.js. I have two queues, one is a file queue and another is an email queue.
I use the file queue to regenerate thumbnails in the background after the thumbnail is uploaded. And email queue to send the email. The problem is that my file queue is working completely fine but the email queue I have created gives me an inconsistent result. Sometimes the jobs are getting picked up by the #Process('JobName') function and sometimes it doesn't.
More importantly, I have noticed that when the problem occurs, I tried removing the dist folder and restarting the server which regenerated the dist directory. It worked until it didn't after a few trials.
This is my folder structure.
my-nest-app/
├─ src/
│ ├─ bull-queue/
│ │ ├─ bull-queue.module.ts
│ │ ├─ file-queue/
│ │ │ ├─ file-queue-producer.service.ts
│ │ │ ├─ file-queue-consumer.service.ts
│ │ ├─ email-queue/
│ │ │ ├─ email-producer.service.ts
│ │ │ ├─ email-processor.service.ts
bull-queue.module.ts
export enum BullQueue {
FileQueue = 'file-queue',
EmailQueue = 'email-queue',
}
#Global()
#Module({
imports: [
BullModule.forRoot({
redis: {
host: process.env.REDIS_HOST,
port: parseInt(process.env.REDIS_PORT) || 6379
}
}),
BullModule.registerQueue(
{ name: BullQueue.FileQueue },
{ name: BullQueue.EmailQueue }
),
ProjectModule,
FileModule
],
providers: [
BullQueueService,
FileQueueConsumerService,
FileQueueProducerService,
EmailProcessorService,
EmailProducerService
],
controllers: [],
exports: [FileQueueProducerService, EmailProducerService]
})
export class BullQueueModule {}
email-producer.service.ts
export enum EmailJobs {
PasswordReset = 'password_reset',
}
#Injectable()
export class EmailProducerService {
constructor(#InjectQueue(BullQueue.EmailQueue) private queue:Queue){}
async password_reset_email(payload: PasswordResetPayload){
await this.queue.add(EmailJobs.PasswordReset, payload);
}
}
email-processor.service.ts
#Processor(BullQueue.EmailQueue)
export class EmailProcessorService {
constructor(
private readonly loggerService: LoggerService,
private readonly helperService: HelperService
) {}
#Process(EmailJobs.PasswordReset)
async password_reset(job: Job<PasswordResetPayload>) {
console.log('password_reset Job Processor')
}
}
I was wondering if I violated any design principle or if this was just an issue with the queue package manager.

Getting error when running Postman Script via Newman

I am trying to run the following Postman Script via Newman to write the response to file:
//verify http response code
pm.test("Report Generated", function () {
pm.response.to.have.status(200);
});
var fs = require('fs');
var outputFilename = 'C:/Users/archit.goyal/Downloads/spaceReport.csv';
fs.writeFileSync(outputFilename, pm.response.text());
The request gives a response but getting the following error when writing to file:
1? TypeError in test-script
┌─────────────────────────┬──────────┬──────────┐
│ │ executed │ failed │
├─────────────────────────┼──────────┼──────────┤
│ iterations │ 1 │ 0 │
├─────────────────────────┼──────────┼──────────┤
│ requests │ 20 │ 0 │
├─────────────────────────┼──────────┼──────────┤
│ test-scripts │ 20 │ 1 │
├─────────────────────────┼──────────┼──────────┤
│ prerequest-scripts │ 0 │ 0 │
├─────────────────────────┼──────────┼──────────┤
│ assertions │ 2 │ 0 │
├─────────────────────────┴──────────┴──────────┤
│ total run duration: 1m 48.3s │
├───────────────────────────────────────────────┤
│ total data received: 1.24MB (approx) │
├───────────────────────────────────────────────┤
│ average response time: 5.3s │
└───────────────────────────────────────────────┘
# failure detail
1. TypeError fs.writeFileSync is not a function
at test-script
inside "3i_BMS_Amortization_Schedule / GetReport"
Please help
Postman itself cannot execute scripts like that. To save your responses of all the api requests, you can create a nodeJS server which will call the api through newman and then save the response to a local file. Here is an example -
var fs = require('fs'),
newman = require('newman'),
allResponse=[],
outputFilename = 'C:/Users/archit.goyal/Downloads/spaceReport.csv';
newman.run({
collection: '//your_collection_name.json',
iterationCount : 1
})
.on('request', function (err, args) {
if (!err) {
//console.log(args); // --> args contain ALL the data newman provides to this script.
var responseBody = args.response.stream,
response = responseBody.toString();
allResponse.push(JSON.parse(response));
}
})
.on('done', function (err, summary) {
fs.writeFile(outputFilename,"");
for(var i =0;i<allResponse.length;i++)
{
fs.appendFileSync(outputFilename,JSON.stringify(allResponse[i],null,5));
}
});
Note that, the above code will save only the responses. Other data like request , or URl can be extracted in a similar manner. To run this, install newman in the same directory as the script, then run the script using -
node file_name.js

how to set a variable automatically zero for each midnight in node js

how to set a variable automatically zero for each midnight in node js is it possible to use node-schedule-npm ? if so please help me how to do it
initially var count= 0; when action is performed, it will get incremented throughout the day, for next day it should be automatically set to zero.
You can use node-cron from npm.
$ npm install --save node-cron
Here is an example code
var cron = require('node-cron');
cron.schedule('* * * * *', function(){
console.log('running a task every minute');
});
# ┌────────────── second (optional)
# │ ┌──────────── minute
# │ │ ┌────────── hour
# │ │ │ ┌──────── day of month
# │ │ │ │ ┌────── month
# │ │ │ │ │ ┌──── day of week
# │ │ │ │ │ │
# │ │ │ │ │ │
# * * * * * *
you can try this
var cron = require('node-schedule');
cron.scheduleJob('00 00 12 * * 1-7',function setVotesToZero(req, res, next) {
db.users.find().toArray(function(err, vote) {
if (err) return next(err);
db.users.update({}, {
$set: {
count : 0
},
}, function(err, result) {
if (err) return next(err);
res.send({
status: 'success',
});
});
})
})

Node.js formatted console output

Is there a simple built-in way to output formatted data to console in Node.js?
Indent, align field to left or right, add leading zeros?
Two new(1) built in methods String.Prototype.padStart and String.Prototype.padEnd were introduced in ES2017 (ES8) which perform the required padding functions.
(1) node >= 8.2.1 (or >= 7.5.0 if run with the --harmony flag)
Examples from the mdn page:
'abc'.padStart(10); // " abc"
'abc'.padStart(10, "foo"); // "foofoofabc"
'abc'.padStart(6,"123465"); // "123abc"
'abc'.padStart(8, "0"); // "00000abc"
'abc'.padStart(1); // "abc"
'abc'.padEnd(10); // "abc "
'abc'.padEnd(10, "foo"); // "abcfoofoof"
'abc'.padEnd(6, "123456"); // "abc123"
'abc'.padEnd(1); // "abc"
For indenting a json onto the console try using JSON.stringify. The third parameter provides the indention required.
JSON.stringify({ a:1, b:2, c:3 }, null, 4);
// {
// "a": 1,
// "b": 2,
// "c": 3
// }
If the data is tabular, then the simplest way would be to do it with console.table
https://nodejs.org/dist/latest-v10.x/docs/api/console.html#console_console_table_tabulardata_properties
This is the code.
console.table(
COMMANDS.map(command => {
return {
"Long Option": command.long_option,
"Short Option": command.short_option,
Description: command.description
};
})
);
You don't need external libraries for it.
Here is sample output. You only need to pass an array object.
Not only in Nodejs, but it also works in chrome.
https://developer.mozilla.org/en-US/docs/Web/API/Console/table
There's nothing built into NodeJS to do this. The "closest" you'd come is util.format, which still doesn't do much unfortunately (reference).
You'll need to look into other modules to provide a richer formatting experience. For example: sprintf.
Sprintf-js allows both positional (0, 1, 2) arguments and named arguments.
A few examples of padding and alignment:
var sprintf=require("sprintf-js").sprintf;
console.log(sprintf("Space Padded => %10.2f", 123.4567));
console.log(sprintf(" _ Padded => %'_10.2f", 123.4567));
console.log(sprintf(" 0 Padded => %010.2f", 123.4567));
console.log(sprintf(" Left align => %-10.2f", 123.4567));
Results:
Space Padded => 123.46
_ Padded => ____123.46
0 Padded => 0000123.46
Left align => 123.46
If you have simpler needs you can look into util.format. It can generate string from various parameters. If you want printf like formatting you can use either sprintf package or sprintf-js package.
Take a look at Log4JS, which is an attempt at a functional port of Log4j
You might also like string-kit and terminal-kit.
https://www.npmjs.com/package/string-kit
https://www.npmjs.com/package/terminal-kit
https://blog.soulserv.net/terminal-friendly-application-with-node-js-part-ii-moving-and-editing/
If I combine util.format and "".padStart/"".padEnd together,as described in posts above, then I get what I wated:
> console.log(util.format("%s%s","Name:".padEnd(10), "John Wall"))
Name: John Wall
A version of console.table for NodeJS with align.
⚠ This is only for console version! ⚠
/**
* #param {NonNullable<{
* [key: string]: string
* }>} data
*/
const consoleTable = (data) => {
if (typeof data === 'object') {
const e = Object.entries(data);
let kl = 0;
let vl = 0;
for (const [k, v] of e) {
if (k.length > kl) {
kl = k.length;
}
const s = JSON.stringify(v);
const l = s ? s.length : 0;
if (l > vl) {
vl = l;
}
}
/** #type {{ [key: string] : string }} */
const result = {};
for (const [k, v] of e) {
const s = JSON.stringify(v);
result[k.padStart(kl)] = s ? s.padEnd(vl) : v;
}
console.table(result);
} else {
console.table(data);
}
};
Output example:
┌─────────────────┬────────────────────────────────────────────────────────────────────────────────────────┐
│ (index) │ Values │
├─────────────────┼────────────────────────────────────────────────────────────────────────────────────────┤
│ CONFIG_FILE │ '"/Users/ecuomo/projects/zzzzz/xxxxxxxx/tttttttttt-web/jjjjjjjjjj-commons/config.env"' │
│ NODE_ENV │ '"development" ' │
│ APP_ENV │ '"local-dev-edu" ' │
│ APP_VERSION │ '"0-local-ecuomo" ' │
│ BASE_URL │ '"http://localhost:3000" ' │
│ CDN_BASE_URL │ '"http://localhost:3000/static" ' │
│ API_BASE_URL │ '"http://localhost:3000/api" ' │
│ MONGO │ '"mongodb://mongo:27017/xxxxxxxxxxprod" ' │
│ MYSQL_HOST │ '"mysql" ' │
│ MYSQL_PORT │ '3306 ' │
│ MYSQL_DB_ETL │ '"xxxxxxxxxx_etl" ' │
│ MYSQL_DB_STATS │ '"xxxxxxxxxx_stats" ' │
│ MYSQL_DB_ZAPIER │ '"xxxxxxxxxx_yyyyyy" ' │
│ POSTGRES_HOST │ '"postgres" ' │
│ POSTGRES_PORT │ '5432 ' │
│ POSTGRES_DB │ '"xxxxxxxxxx" ' │
│ REDIS_HOST │ '"redis" ' │
│ REDIS_PORT │ '"6379" ' │
│ REDIS_MASTER │ undefined │
│ REDIS_MODE │ '"single" ' │
└─────────────────┴────────────────────────────────────────────────────────────────────────────────────────┘

Resources