Synchronous NodeJS batch job - node.js

I'm trying to write a batch script that will
Read XMLs from a directory
Parse each XML and find a value to use for DB(database) Lookup
Use the parsed value to DB lookup additional metadata
Populate XML with the metadata retrieved from DB lookup (step 4)
Write updated XML to complete directory
Close DB connection
The issue I'm running into is that I cannot control the code execution order so that I can close the DB connection at the end of the script. If I attempt to close the connection, I get a 'connection undefined' error. Below is my code for reference. Is there a good way to accomplish something like this in NodeJs, or should I look at doing this in Java or some other language?
enter code here
'use strict';
let fs = require('fs');
let xml2js = require('xml2js');
const oracledb = require('oracledb');
const dbConfig = require('./dbconfig.js');
function pad(number, length)
{
var str = '' + number;
while (str.length < length)
{
str = '0' + str;
}
return str;
}
async function run() {
try {
// Get a non-pooled connection
let connection;
if (!connection)
{
connection = await oracledb.getConnection(dbConfig);
console.log('Connection was successful!');
}
let directory = "EDI_XMLS";
let dirBuf = Buffer.from(directory);
//var f = 0;
let files = fs.readdirSync(directory);
console.log(files);
for (let f = 0; f < files.length; f++)
{
let parser = new xml2js.Parser();
var xml_json_data = "";
// read the file
await fs.readFile(directory + "/" + files[f], async function(err, data) {
// parse the file
await parser.parseString(data, async function(err, result) {
let results;
var line_count = result.page.GLLines[0].GLLine.length;
console.dir('Invoice: ' + result.page.InvoiceNumber[0]);
console.dir('Line Total: ' + line_count);
console.log('File: ' + f);
try
{ // Lookup Data
results = await connection.execute('SELECT BUSINESS_UNIT, OPERATING_UNIT, DEPTID,PRODUCT,a.effdt FROM SYSADM.PS_A_NSS_SHPTO_ACC#FDEV a where(a.a_ship_to_customer = :shipTo) order by a.effdt
desc',[pad(result.page.VoucherDescription[0], 10)], {
maxRows: 2
});
console.log(results.metaData);
console.log(results.rows);
}
catch (err)
{
console.error(err);
}
for (let i = 0; i < line_count; i++) // Populate data
{
result.page.GLLines[0].GLLine[i].GLBU[0] = results.rows[0][0];
result.page.GLLines[0].GLLine[i].OpUnit[0] = results.rows[0[1];
result.page.GLLines[0].GLLine[i].Department[0] = results.rows[0][2];
result.page.GLLines[0].GLLine[i].Product[0] = results.rows[0][3];
}
// Write to File
var builder = new xml2js.Builder();
var xml = builder.buildObject(result);
await fs.writeFile("complete/" + files[f], xml, function(err, data) {
if (err) console.log(err);
console.log("successfully written our update xml to file");
console.dir('BUs: ' + JSON.stringify(result.page));
}); //end write
}); //end parser
}); //end readfile
console.log('End');
} // async for
}
catch (err)
{
console.error(err);
}
finally
{
await connection.close();
console.log('Finally Done');
}
}
run();
console.log('completely Done');

Related

Parse excel file and create JSON format in exceljs ON nodejs

I have this excel file
I need to convert the data excel from file to JSON like below
[
{
"no_pemohonan": "PNL-202109200826210023105",
"sumber_data": "HOSTS",
"tgl_permohonan": "2021-09-20",
},
{
"no_pemohonan": "PNL-202109200845131363376",
"sumber_data": "HOSTS",
"tgl_permohonan": "2021-09-20",
},
...
]
I could make the data with this comment but i have to set the object again like below syntax
const excel = require('exceljs');
const workbook = new excel.Workbook();
await workbook.xlsx.load(objDescExcel.buffer);
let json = workbook.model;
let worksheetsArr = json.worksheets.length;
const arrRow = [];
const arrIdPembatalan = [];
// looping per worksheet
for (let index = 0; index < worksheetsArr; index++) {
let worksheet = workbook.worksheets[index];
// console.log("worksheet " + worksheet);
const rowlast = worksheet.lastRow;
// looping semua row untuk
worksheet.eachRow(async (row, rowNumber) => {
let new_row = row.values
// console.log(new_row);
let no_permohonan= new_row[2]
let sumber_data = new_row[3]
let tgl_permohonan = new_row[4]
let header = {
no_permohonan: no_permohonan,
sumber_data : sumber_data ,
tgl_permohonan : tgl_permohonan ,
};
arrIdPembatalan.push(header)
}
})
}
I want to set the header automatically without have to set the header again like above syntax.
I have seen this solution but it was written in xlsx library, while i am using exceljs.
Here is a nodejs implement.
(async function() {
const excel = require('exceljs');
const workbook = new excel.Workbook();
// use readFile for testing purpose
// await workbook.xlsx.load(objDescExcel.buffer);
await workbook.xlsx.readFile(process.argv[2]);
let jsonData = [];
workbook.worksheets.forEach(function(sheet) {
// read first row as data keys
let firstRow = sheet.getRow(1);
if (!firstRow.cellCount) return;
let keys = firstRow.values;
sheet.eachRow((row, rowNumber) => {
if (rowNumber == 1) return;
let values = row.values
let obj = {};
for (let i = 1; i < keys.length; i ++) {
obj[keys[i]] = values[i];
}
jsonData.push(obj);
})
});
console.log(jsonData);
})();
test result
$ node ./test.js ~/Documents/Book1.xlsx
[
{
no_pemohonan: 'PNL-202109200826210023105',
sumber_data: 'HOSTS',
tgl_permohonan: 2021-09-20T00:00:00.000Z
},
{
no_pemohonan: 'PNL-202109200845131363376',
sumber_data: 'HOSTS',
tgl_permohonan: 2021-09-20T00:00:00.000Z
}
]
If dealing with large files, I would explore stream processing using the following libraries:
Use exceljs to read .xlsx file as stream and write to .csv as stream:
// read from a stream
const readStream = fs.createReadStream('largeWorkbook.xlsx');
const workbook = new Excel.Workbook();
await workbook.xlsx.read(readStream);
// write to stream
const writeStream = fs.createWriteStream('largeWorksheet.csv');
await workbook.csv.write(writeStream, { sheetName: 'Page name' });
Then use csvtojson to transform CSV to JSON:
import csvToJson from 'csvtojson'
const readStream = fs.createReadStream('largeWorksheet.csv')
const writeStream = fs.createWriteStream('largeWorksheet.json')
readStream
.pipe(csvToJson())
.pipe(writeStream)
This will work for large files even on hardware with low memory.
Full code snippet:
import fs from 'fs'
import Excel from 'exceljs'
import csvToJson from 'csvtojson'
const xlsxRead = fs.createReadStream('largeWorkbook.xlsx')
const csvWrite = fs.createWriteStream('largeWorksheet.csv')
const csvRead = () => fs.createReadStream('largeWorksheet.csv')
const jsonWrite = fs.createWriteStream('largeWorksheet.json')
(async function process() {
const workbook = new Excel.Workbook()
await workbook.xlsx.read(xlsxRead)
await workbook.csv.write(csvWrite, { sheetName: 'Worksheet Name' })
csvRead()
.pipe(csvToJson())
.pipe(jsonWrite)
})() // this immediately-invoked wrapper function is just for Node.js runtimes
// that don't support top-level await yet
// if running via `--esm` or from `.mjs` file, it can be ommitted
var Excel = require('exceljs');
var ReadExcelCSV = function (fileType, fileName, filePath, delimeter, textQualifier) {
var deffered = q.defer();
var workbook = new Excel.Workbook();
var finalFilePath = filePath + fileName;
if (fileType == "excel") {
console.log("File Type: Excel");
workbook.xlsx.readFile(finalFilePath).then(function () {
ParseExcelCSVFile(workbook).then(function (resp) {
deffered.resolve(resp);
}, function (error) {
logger.info("Error in Parsing Excel/CSV");
});
}, function (err) {
logger.info("Error In Read Excel: " + JSON.stringify(err));
});
} else {
if (delimeter != undefined && textQualifier != undefined) {
var options = {};
options.delimiter = delimeter;
options.quote = textQualifier;
options.dateFormats = [];
workbook.csv.readFile(finalFilePath, options).then(function () {
ParseExcelCSVFile(workbook).then(function (resp) {
// fs.unlink(finalFilePath);
deffered.resolve(resp);
}, function (error) {
logger.info("Error in Parsing Excel/CSV");
deffered.reject(error);
});
}, function (error) {
logger.info("Error In Read CSV: " + JSON.stringify(error));
deffered.reject(error);
});
} else {
workbook.csv.readFile(finalFilePath).then(function () {
ParseExcelCSVFile(workbook).then(function (resp) {
deffered.resolve(resp);
}, function (error) {
logger.info("Error in Parsing Excel/CSV");
deffered.reject(error);
});
}, function (error) {
logger.info("Error In Read CSV: " + JSON.stringify(error));
deffered.reject(error);
});
}
}
return deffered.promise;
};
var ParseExcelCSVFile = function (workbook) {
try {
var deffered = q.defer();
var objresult = [];
var objheaders = [];
var worksheet = workbook.getWorksheet(1);
worksheet.eachRow(function (row, rowNumber) {
var currentobj = {};
row.eachCell({
includeEmpty: true
}, function (cell, colNumber) {
if (rowNumber == 1) {
objheaders.push(cell.value);
} else {
currentobj[objheaders[colNumber - 1]] = cell.value == null ? '' : cell.value;
}
});
if (rowNumber != 1) {
objresult.push(currentobj);
}
});
deffered.resolve(objresult);
return deffered.promise;
} catch (ex) {
logger.error("Error in ParseExcel: " + ex.stack);
}
};
I wrote this code quite a long time ago so you will see an old module like deffered which you can change easily, but it will help in what you are trying to achieve. It can read and parse excel and csv both.

Combine PDF files in a loop ( Hummus-Recipe )

I work with Hummus-Recipe library and it's work fine but I want to make a function that accept array of files to append all files to one.
This is my code that work:
const filesRoot = './uploads';
router.route('/')
.get( async (request, response) => {
const src = filesRoot + '/one.pdf';
const appendedFile = filesRoot + '/two.pdf';
const appendedFile2 = filesRoot + '/three.pdf';
const output = filesRoot + '/new.pdf';
const recipe = new HummusRecipe(src, output);
recipe
.appendPage(appendedFile)
.appendPage(appendedFile2)
.endPDF();
});
How can I take this code and make it accept array??
Something like that:
let combinePdfFiles = (array) => {
for (let i = 0; i < array.length; i++) {
}
};
thanks.
You can use easy-pdf-merge package that let you merge an array of some pdf files.
Usage:
const merge = require('easy-pdf-merge');
merge(source_files,dest_file_path,function(err){
if(err) {
return console.log(err)
}
console.log('Success')
});
Example:
merge(['File One.pdf', 'File Two.pdf'], 'File Ouput.pdf', function(err){
if(err) {
return console.log(err)
}
console.log('Successfully merged!')
});
I create this function and it work.
const combinePdfFiles = async (files, companyID, flowID) => {
const filesRoot = `./uploads/${companyID}/${flowID}`;
try {
const originalFile = `${filesRoot}/${files[0]}`;
const output = `${filesRoot}/combined.pdf`;
const recipe = new HummusRecipe(originalFile, output);
for (let i = 1; i < files.length; i++) {
recipe
.appendPage(`${filesRoot}/${files[i]}`);
}
recipe.endPDF();
} catch (error) {
throw error;
}
};

Nodejs Await function issue

I've make a translation service in nodejs :
module.exports = {GetcountryFR, GetRegionFR, GetChildrenFR, GetHomeownFR, GetHomestyleFR, GetIncomeFR, GetLstatusFR, GetWdomainFR, GetWtypeFR, GetSexFR}
async function GetcountryFR(country) {
var countrix = country;
switch (countrix) {
case 'createuser.france': countrix = 'FRANCE'; break;
case 'createuser.belgique': countrix = 'BELGIQUE'; break;
default: countrix = 'UNKNOWN'; break;
}
return countrix;
}
And now I make a function which uses translat function
const translate = require('../services/translate');
exports.AllUserToCSV = async function CSV() {
try {
let user = User.find();
var data = [];
len = user.length;
for (i = 0; i < len; i++) {
let sexix = await translate.GetSexFr(user[i].sex);
let regionix = await translate.GetRegionFR(user[i].region);
let countrix = await translate.GetcountryFR(user[i].country);
let wtypix = await translate.GetWtypeFR(user[i].wtype);
let wdomainix = await translate.GetWdomainFR(user[i].wdomain);
temp = {
sex: sexix,
region: regionix,
country: countrix,
wtype: wtypix,
wdomain: wdomainix,
}
data.push(temp);
}
const csvData = csvjson.toCSV(data, { headers: 'key' })
filename2 = '/assets/media/' + 'TEST' + '.csv';
writeFile(filename, csvData, (err) => {
if (err) {
console.log(err); // Do something to handle the error or just throw it
throw new Error(err);
}
console.log('Success!');
});
} catch (e) {
console.error(e);
}
}
In results CSV file is Empty [].
If I put values in temp it's OK.
Why my translate function didn't work ??
Thanks for Help Good Friends :)
Simply await your call to the User model i.e let user = await User.find();
Also for the loop, try
let users = await User.find();
await Promise.all(users.map(async (user) => {
let sexix = await translate.GetSexFr(user.sex);
...
})));
Writing to file, you may want to use await fs.writeFile(...);. This will make sure file is written before processing the next.

How to get code to execute in order in node.js

I am trying to finish my script, but for some reason i don't know, it refuses to execute in the order i put it in.
I've tried placing a 'wait' function between the JoinRequest update function and the following code, but when run, it acts as if the function call and wait function were the other way round, countering the point of the wait().
const Roblox = require('noblox.js')
var fs = require('fs');
var joinRequests = []
...
function wait(ms) {
var d = new Date();
var d2 = null;
do { d2 = new Date(); }
while(d2-d < ms*1000);
};
...
function updateJReqs() {
Roblox.getJoinRequests(4745601).then((array) => {
var i;
var final = [];
for(i = 0; i < array.length; i++) {
final.push(array[i].username);
};
if(final === '') {
final = '-None';
};
joinRequests = final
console.log('Updated join requests.')
});
}
function check() {
setTimeout(() => {
fs.readFile('Request.txt',encoding = 'utf-8', function(err, data) {
if (err) {
check();
} else {
updateJReqs(); //for some reason this function is executed alongside the below, not before it.
// Tried putting wait(x) in here.
console.log('Request received: ' + data)
var solution = joinRequests
console.log('Fuffiling request with ' + solution)
fufillRequest(solution)
fs.unlink('Request.txt', function(err) {
if(err) throw err;
});
check();
}
});
}, 400)
}
check();
The script is supposed to wait until a file is created (accomplished), update the list of join requests (accomplished) and then create a new file with the list of join requests in(not accomplished).
if I understand your code you work with async code, you need to return a promise in updateJReqs and add a condition of leaving from the function because you have an infinite recursion
function updateJReqs() {
return new Promise(resolve => {
Roblox.getJoinRequests(4745601).then((array) => {
var i;
var final = [];
for(i = 0; i < array.length; i++) {
final.push(array[i].username);
};
if(final === '') {
final = '-None';
};
joinRequests = final
console.log('Updated join requests.')
resolve();
});
}
}
async function check() {
setTimeout(() => {
fs.readFile('Request.txt',encoding = 'utf-8', function(err, data) {
if (err) {
await check();
} else {
await updateJReqs();
// Tried putting wait(x) in here.
console.log('Request received: ' + data)
var solution = joinRequests
console.log('Fuffiling request with ' + solution)
fufillRequest(solution)
fs.unlink('Request.txt', function(err) {
if(err) throw err;
});
// you dont have an exit from your function check();
return 'Success';
}
});
}, 400)
}
check().then(res => console.log(res));

nodejs + Mongodb: Inserting into two collections in sequence repeats last value in second

I am using the following to insert into MongoDB.
var tagData = JSON.parse(data);
var allTags = tagData.tags;
for (var j = 0; j < allTags.length; j++) {
var p = allTags[j].tagId.toString();
for (var k = 0; k < loggerParams.length; k++) {
var q = Object.keys(loggerParams[k]).toString();
if (p === q) {
// Prepare raw data tag
var tagRawDoc = {};
// Simple key-value assignment here
// Document prepared; ready to insert into MongoDB
database.addDocument('tagraw', tagRawDoc, function (err) {
if (err) {
log.info(util.format('Error adding document to tagrawdatas. %s', err.message));
throw err;
} else {
// Prepare history tag
var historyTagDoc = {};
historyTagDoc.tagNameAlias = tagRawDoc.tagNameAlias;
// Simple key-value assignment here
// Document prepared; ready to insert into MongoDB
database.addDocument('taghistory', historyTagDoc, function (err) {
if (err) {
log.info(util.format('Error adding document to tagrawdatas. %s', err.message));
throw err;
}
});
}
});
// Match found; exit loop
break;
}
}
}
The loggerParms is a simple JSON document read from file else-where. It allows for look-up in this code to build the document to be inserted. There will be 12 values in the allTags array. These 12 values are inserted successfully into the tagraw collection. However, in taghistory collection, the values from the last (or most recent) entry made into tagraw collection is repeated 12 times. Why does this happen?
The database.addDocument is shown below. It is a part of this article I am trying to replicate.
var MongoClient = require('mongodb').MongoClient;
var assert = require('assert');
var logger = require('../../util/logger');
var util = require('util');
function DB() {
this.db = "empty";
this.log = logger().getLogger('mongoMange-DB');
}
DB.prototype.connect = function(uri, callback) {
this.log.info(util.format('About to connect to DB'));
if (this.db != "empty") {
callback();
this.log.info('Already connected to database.');
} else {
var _this = this;
MongoClient.connect(uri, function(err, database) {
if (err) {
_this.log.info(util.format('Error connecting to DB: %s', err.message));
callback(err);
} else {
_this.db = database;
_this.log.info(util.format('Connected to database.'));
callback();
}
})
}
}
DB.prototype.close = function(callback) {
log.info('Closing database');
this.db.close();
this.log.info('Closed database');
callback();
}
DB.prototype.addDocument = function(coll, doc, callback) {
var collection = this.db.collection(coll);
var _this = this;
collection.insertOne(doc, function(err, result) {
if (err) {
_this.log.info(util.format('Error inserting document: %s', err.message));
callback(err.message);
} else {
_this.log.info(util.format('Inserted document into %s collection.', coll));
callback();
}
});
};
module.exports = DB;
That's because you are mixing a/multiple synchronous for and asynchronous code with database.addDocument which cause issues with function scope in nodejs.
A simple example of this kind of thing:
for(var i = 0; i < 10; i++){
setTimeout(() => console.log(i), 0);
}
You should use a package like async to handle flow control when iterating arrays/object asynchronously.
Simple example of your code refactored to use async:
var async = require('async');
var tagData = JSON.parse(data);
var allTags = tagData.tags;
async.each(allTags, function(tag, done){
var p = tag.tagId.toString();
var loggerParam = loggerParams.find(function(loggerParam){
var q = Object.keys(loggerParam).toString();
return p === q;
});
var tagRawDoc = {};
// Simple key-value assignment here
// Document prepared; ready to insert into MongoDB
return database.addDocument('tagraw', tagRawDoc, function (err){
if (err) return done(err);
// Prepare history tag
var historyTagDoc = {};
historyTagDoc.tagNameAlias = tagRawDoc.tagNameAlias;
// Simple key-value assignment here
// Document prepared; ready to insert into MongoDB
return database.addDocument('taghistory', historyTagDoc, done);
});
}, (err) => {
if(err) throw err;
console.log('All done');
});

Resources