Batching in nodejs - node.js

Question

Very new to nodejs, I have a file which contains only single column jobid. And I am iterating one by one job id and sending requesting to a service which give me the status of the job in a json format. Reading the json response and fetching few values from it and writing to the database. And I want write when it reach the 100th(with the help of a counter) jobid(because I am expecting more than 100 jobid in the file, also this is dynamic).
For example if I have 234 records, then it will write 3 times, first two 100 each and third one with 34. And the jobStatusMetrics array should be cleaned every write.
const fileStatusprocess = require('../controller/readResultFile');
const config = require('../config/config');
const https = require('https');
const uuid = require('uuid-random');
async function jobProcesser() {
const iterator = (await fileStatusprocess.processResultFile("C:\Support\result.csv"))
console.log('Total jobs are',iterator[0].length);
var counter = 0 ;
for (i = 0; i < iterator[0].length; i++) {
counter++;
const formJobStatusURL = "https://localhost:8091/api/job/" + iterator[0][i] + "/status";
const option = {
method: 'GET' ,
headers: {
'X-Message-Created-Ts': `${new Date().toISOString()}`,
'X-Transaction-Created-Ts': `${new Date().toISOString()}`,
'X-User-Id': 'PerformanceExecuter',
'X-Client-Id': `${uuid()}`,
'X-Message-Id': `${uuid()}`,
'X-Transaction-Id': `${uuid()}`,
'Content-Type': 'application/json'
}
}
let content = '';
let reqGet = https.request(formJobStatusURL,option, function (response) {
response.on('data', function (data) {
content += data;
});
response.on('end', function () {
const jsonPayload = JSON.parse(content);
const jobStatusMetrics = {};
for ( var key in jsonPayload){
if(jsonPayload.hasOwnProperty(key)){
jobStatusMetrics.job_id = jsonPayload.id;
jobStatusMetrics.status = jsonPayload.status;
jobStatusMetrics.initiatedBy = jsonPayload.initiatedBy;
jobStatusMetrics.product = jsonPayload.product;
jobStatusMetrics.operation = jsonPayload.operation;
jobStatusMetrics.startTimestamp = jsonPayload.startTimestamp;
jobStatusMetrics.endTimestamp = jsonPayload.endTimestamp;
jobStatusMetrics.totalRecords = jsonPayload.file.totalRecords;
jobStatusMetrics.failedRecords = jsonPayload.file.totalFailedRecords;
jobStatusMetrics.sucessRecords = jsonPayload.file.totalSuccessRecords;
jobStatusMetrics.inprogressRecords = jsonPayload.file.totalInProgressRecords;
jobStatusMetrics.sucessStatus = jsonPayload.results.successFileAvailable;
jobStatusMetrics.failureStatus = jsonPayload.results.failureFileAvailable;
jobStatusMetrics.uploadJob = jsonPayload.actionsAvailable.dataUploadAllowed;
jobStatusMetrics.abortJob = jsonPayload.actionsAvailable.abortJob;
}
if ( counter%100 == 0) {
console.log('Writting to the database')
// logic for influxdb writter
}
else {
}
}
//console.log(jobStatusMetrics);
})
});
reqGet.end();
}
}
jobProcesser()

Answer 1

Rather to use https for http request please use another library like phin-retry. Please check the below code, added the logic. Hope this work.
const rp = require('phin-retry');
const fileStatusprocess = require('../controller/readResultFile');
const config = require('../config/config');
const uuid = require('uuid-random');
const influxDBWrite = require('../controller/influxdbWritter')
async function checkFileStatus() {
const iterator = (await fileStatusprocess.processResultFile("C:\Support\result.csv"));
const jobIds = iterator[0];
console.log(jobIds)
let jobDetailsList = [];
for (i = 0; i < jobIds.length; i++) {
console.log('JobId', i)
const jobId = jobIds[i];
const formJobStatusURL = "https://localhost:8091/api/job/" + jobId + "/status";
const jobDetails = await rp.get({
url: formJobStatusURL,
headers: {
'X-Message-Created-Ts': `${new Date().toISOString()}`,
'X-Transaction-Created-Ts': `${new Date().toISOString()}`,
'X-User-Id': 'PerformanceExecuter',
'X-Client-Id': `${uuid()}`,
'X-Message-Id': `${uuid()}`,
'X-Transaction-Id': `${uuid()}`,
'Content-Type': 'application/json'
}
});
jobDetailsList.push(jobDetails);
if ((i + 1) % 5 === 0) {
await saveResultsToInfux(jobDetailsList);
jobDetailsList = [];
}
}
await saveResultsToInfux(jobDetailsList);
}
async function saveResultsToInfux(jobDetailsList) {
const metrics = [];
for (i = 0; i < jobDetailsList.length; i++) {
console.log('Metrics', i)
const jobDetail = jobDetailsList[i];
const tags = {
Id: jobDetail.id,
Status: jobDetail.status.overall,
Product: jobDetail.product,
Entity: jobDetail.entity,
Operation: jobDetail.operation
}
const fields = {
startTimestamp:jobDetail.startTimestamp,
endTimestamp:jobDetail.endTimestamp,
totalRecords:jobDetail.file.totalRecords,
totalFailedRecords:jobDetail.file.totalFailedRecords,
totalSuccessRecords:jobDetail.file.totalSuccessRecords,
totalInProgressRecords:jobDetail.file.totalInProgressRecords,
successFileAvailable:jobDetail.results.successFileAvailable,
failureFileAvailable:jobDetail.results.failureFileAvailable,
dataUploadAllowed:jobDetail.actionsAvailable.dataUploadAllowed,
abortJob:jobDetail.actionsAvailable.abortJob,
}
metrics.push({
measurement: 'BulkData',
tags,
fields
});
}
influxDBWrite.dbWritter(metrics);
}
checkFileStatus()