How can node.js process large-capacity Excel files separately? - node.js

https://github.com/exceljs/exceljs
There is a problem in creating applications through electron and Exceljs.
I want to read the Excel file and show it to user in the html table. The problem is that it deals with xlsx files with a very large number of rows.
So I want to show the progress to the user as follows.
ex) 5/10000. (Read row / Total number of rows)
Also, if possible, I would like to return the data in the form of json or html whenever I read it. (If this is impossible, I'd like to indicate that it's loading until it's finished)
const filePath = "/Users/caglia/Documents/test.xlsx"
const ExcelJS = require('exceljs')
async function loadExcelFile(filePath) {
let start = new Date();
console.log('start :', Date.now())
const sheetData = []
const workbook = new ExcelJS.Workbook()
await workbook.xlsx.readFile(filePath)
const worksheet = workbook.worksheets[0] // first sheet
const options = {
includeEmpty: true
}
await worksheet.eachRow(options, (row, rowNum) => {
sheetData[rowNum] = []
//console.log('rowNum', rowNum)
row.eachCell(options, (cell, cellNum) => {
sheetData[rowNum][cellNum] = {
value: cell.value
}
})
})
let end = new Date();
console.log('finish ', Date.now(), 'timediff', end - start);
}
loadExcelFile(filePath)

Related

Is there a way to import a range selected in Excel to my web application?

I'm trying to import a selected range in excel to my web application (JSF with JavaScript).
I'm already able to import an entire excel file with file upload and I can also copy content of excel file, paste it in some kind of text input and parse it afterwards. But I would really love to have the behavior when one excel uses a selected range of another one.
I'm aware that this is maybe not possible because the browser's runtime is separated from the client (which runs excel) for very good reasons. Nevertheless I just wanted to check if there is a good idea outside which I didn't found in my web search.
excel range select:
You could use insertWorkbookFromBase64 API to copy the worksheet from another workbook. and then you could remove the other unrelated content by API call.
Below are the sample code for your reference.
async function run() {
await Excel.run(async (context) => {
//Excel.InsertWorksheetOptions
var workbook = context.workbook;
var options = {
sheetNamesToInsert: ["sample"],
positionType: Excel.WorksheetPositionType.end,
relativeTo: "Sheet1"
};
const sheets = context.workbook.worksheets;
sheets.addFromBase64(
workbookContents,
null, // get all the worksheets
Excel.WorksheetPositionType.end // insert them after the current workbook's worksheets
);
await context.sync();
});
}
async function getFileBase64() {
const myFile = <HTMLInputElement>document.getElementById("file");
const reader = new FileReader();
reader.onload = (event) => {
// strip off the metadata before the base64-encoded string
const startIndex = reader.result.toString().indexOf("base64,");
workbookContents = reader.result.toString().substr(startIndex + 7);
};
// read in the file as a data URL so we can parse the base64-encoded string
reader.readAsDataURL(myFile.files[0]);
}
async function insertWorksheetsFromBase64(workbookContents) {
Excel.run((context) => {
//Excel.InsertWorksheetOptions
var options = { sheetNamesToInsert: [], positionType: Excel.WorksheetPositionType.before, relativeTo: "Sheet2" };
const sheets = context.workbook.insertWorksheetsFromBase64(workbookContents);
return context.sync();
});
}

How to send values using labels of a field in Protractor

Can anyone please guide me on how can I send values using the labels of the field. I'm aware of the fact that one should not send values using labels as the labels changes but in my case it's not that case, i.e it's not gonna change.
I'm attaching the HTML Code screenshots of that label and the webpage screenshot as well. WebPage Screenshot and the HTML Code Screenshot.
The only code structure I can show is in this image.Code Structure
The codes which I have tried are as below,
The From Date and To Date is one set of code I tried. So, like first From and To Date variable is one way of implementation of code, second set is some other way of implementing the code and so on.
async selectDates(FromDate:string,ToDate:string){
console.log("$$$ From and To Date in selectDates function From: "+FromDate+" To: "+ToDate);
// var fromDate = "From Date";
// await browser.element(by.xpath("//label[. = '" + fromDate + "']/following-sibling::input"));
// await fromInput.sendKeys(FromDate);
// var toDate = "To Date";
// await browser.element(by.xpath("//label[. = '" + toDate + "']/following-sibling::input"));
// await toInput.sendKeys(ToDate);
// var TmpLabelName = "From Date";
// var TmpInput = await element(by.xpath("//label[contains(text(),'" + TmpLabelName + "')]/following-sibling::input"));
// await TmpInput.sendKeys(FromDate);
// var TmpLabelName2 = "To Date";
// var TmpInput2 = await element(by.xpath("//label[contains(text(),'" + TmpLabelName2 + "')]/following-sibling::input"));
// await TmpInput2.sendKeys(ToDate);
// var TmpLabelName = "From Date";
// var TmpInput = await element(by.xpath("//label[.,'" + TmpLabelName + "']/following-sibling::input"));
// await TmpInput.sendKeys(FromDate);
// var TmpLabelName2 = "To Date";
// var TmpInput2 = await element(by.xpath("//label[.,'" + TmpLabelName2 + "']/following-sibling::input"));
// await TmpInput2.sendKeys(ToDate);
// let FlabelName = "From Date";
// var Finput = await element(by.xpath("//label[. = '" + FlabelName + "']/following-sibling::input")).sendKeys(FromDate);
// let TlabelName = "To Date";
// var Tinput = await element(by.xpath("//label[. = '" + TlabelName + "']/following-sibling::input")).sendKeys(ToDate);
}
I have searched for many articles and other answers but neither of them gave the desired answer. can anyone please help me with this, It would be really helpful!!
Edits:
The Code structure (Sorry for identation issues)
function ele(label: string){
return element.all(by.css('td > label.fieldlabel')).filter((ele)=>{
return ele.getText().then((text: string) => {
return text === label;
});
}).get(0);
}
export class Reports extends ReportObjects {
async selectDates(FromDate:string,ToDate:string){
await browser.executeScript("arguments[0].value='" + FromDate + "';", ele('From Date'));
await browser.executeScript("arguments[0].value='" + ToDate + "';", ele('To Date'));
}
async generateReport(testDataRow:number){
let fromDate = excel.getColumnValue('FromDate',testDataRow).toString();
let toDate = excel.getColumnValue('ToDate',testDataRow).toString();
await this.selectDates(fromDate,toDate);
}
}
The Excel Screenshot From/To Date
PS: I cannot use ID because that is dynamic, it's changing for different scenarios
Even we not recommend to use XPATH, but in your case we need it.
cost ele = element(by.xpath('//label[text()="From Date"]/../../input'))
await ele.sendKeys(FromDate)
// if sendKeys() can't work, try as below
await browser.executeScript("arguments[0].value=arguments[1]", ele, FromDate);
Try below solution with java script executer
const FromDate= //your from date
function ele(label: string){
return element.all(by.css('td > label.feildlabel').filter((ele)=>{
return ele.getText().then((text: string) => {
return text === label;
});
}).get(0);
}
await browser.executeScript("arguments[0].value='" + FromDate + "';", ele('From Date'));
Refer https://www.protractortest.org/#/api?view=webdriver.WebDriver.prototype.executeScript

input function using user prompt

I've posted this question before without success.
I have some lines making one user menu getting data from exceljs.
1 - How to put while condition inside this function, to get all rows to menu? Looks like switch(input) doesn't accept loop.
The logic is:
* Read costumers.xlsx to get all costumers.
* Loop all costumers as menu choices.
* After the choice it'll open another xlsx file with the name of chosen costumer
ie:
1 - Costumer1
2 - Costumer2
If I chose 1 I'll open costumer1.xlsx
2 - How to take that choice and pass as string to open the xlsx?
wb_costumers.xlsx.readFile('costumers.xlsx').then(function(){
sh_costumers = wb_costumers.getWorksheet("Sheet1");
var ic = 2;
while (ic <= sh_costumers.rowCount){
console.log("Row " + sh_costumers.getRow(ic).getCell(1) + " - " + sh_costumers.getRow(ic).getCell(2));
ic++;
}
});
function processChoice(input){
return new Promise(function(resolve, reject) {
var error = undefined;
var func;
switch(input){
case sh_costumers.getRow(2).getCell(1).text :
func = addPreset;
break;
After some researches, I've found something about prompt (npm install prompt).
Now I can read my costumers configuration inside the xlsx file.
on workbook costumers.xlsx I have two columns:
cell1 = id
cell2 = costumer's name
on workbook check_(costumer's name).xlsx I have the informations what I want to place somewhere.
That's my code.
const prompt = require('prompt');
var Excel = require('exceljs');
var wb = new Excel.Workbook();
var wbc = new Excel.Workbook();
prompt.start();
var ic = 1;
wbc.xlsx.readFile('costumers.xlsx').then(function(){
shc = wbc.getWorksheet("Sheet1");
while (ic <= shc.rowCount){
console.log(shc.getRow(ic).getCell(1).value +" - "+ shc.getRow(ic).getCell(2).value);
ic++;
}
});
prompt.get(['costumer'], function (err, result) {
if (err) { return onErr(err); }
var costumer = shc.getRow(result.costumer).getCell(2).value;
wb.xlsx.readFile('check_'+costumer+'.xlsx').then(function(){
sh = wb.getWorksheet("Sheet1");
console.log(sh.getRow(2).getCell(3).value);
});
});
function onErr(err) {
console.log(err);
return 1;
}

GCF "No Such Object" when the Object in question was just created

I'm setting up a Google Cloud Functions (GCF) function that gets triggered often enough that there are multiple instances running at the same time.
I am getting errors from a readStream the source file of the stream does not exist, but at this point in my program I've actually just created it.
I've made sure the file exists before the start of the stream by console.log()-ing the file JSON, so the file does actually exist. I've also made sure that the file I'm trying to access has finished being written by a previous stream with an await, but no dice.
EDIT: The code now contains the entire script. The section that seems to be throwing the error is the function columnDelete().
var parse = require('fast-csv');
var Storage = require('#google-cloud/storage');
var Transform = require('readable-stream').Transform;
var storage = new Storage();
var bucket = storage.bucket('<BUCKET>');
const DMSs = ['PBS','CDK','One_Eighty','InfoBahn'];
class DeleteColumns extends Transform{
constructor(){
super({objectMode:true})
}
_transform(row, enc, done){
//create an array 2 elements shorter than received
let newRow = new Array(row.length - 2);
//write all data but the first two columns
for(let i = 0; i < newRow.length; i++){
newRow[i] = row[i+2];
}
this.push(newRow.toString() + '\n');
done();
}
}
function rename(file, originalFile, DMS){
return new Promise((resolve, reject) => {
var dealer;
var date;
var header = true;
var parser = parse({delimiter : ",", quote:'\\'});
//for each row of data
var stream = originalFile.createReadStream();
stream.pipe(parser)
.on('data', (row)=>{
//if this is the first line do nothing
if(header){
header = false;
}
//otherwise record the contents of the first two columns and then destroy the stream
else {
dealer = row[0].toString().replace('"', '').replace('"', '');
date = row[1].toString().replace('"', '').replace('"', '');
stream.end();
}
})
.on('finish', function(){
var newName = dealer + ' ' + date + '_' + DMS + 'temp.csv';
//if this was not triggered by the renaming of a file
if(!file.name.includes(dealer)&&!file.name.includes(':')){
console.log('Renamed ' + file.name);
originalFile.copy(newName);
originalFile.copy(newName.replace('temp',''));
}else{
newName = 'Not Renamed';
console.log('Oops, triggered by the rename');
}
resolve(newName);
});
});
}
function columnDelete(fileName){
return new Promise((resolve, reject) =>{
console.log('Deleting Columns...');
console.log(bucket.file(fileName));
var parser = parse({delimiter : ",", quote:'\\'});
var del = new DeleteColumns();
var temp = bucket.file(fileName);
var final = bucket.file(fileName.replace('temp', ''));
//for each row of data
temp.createReadStream()
//parse the csv
.pipe(parser)
//delete first two columns
.pipe(del)
//write to new file
.pipe(final.createWriteStream()
.on('finish', function(){
console.log('Columns Deleted');
temp.delete();
resolve();
})
);
});
}
exports.triggerRename = async(data, context) => {
var DMS = 'Triple';
var file = data;
//if not a temporary file
if(!file.name.includes('temp')){
//create a new File object from the name of the data passed
const originalFile = bucket.file(file.name);
//identify which database this data is from
DMSs.forEach(function(database){
if(file.name.includes(database)){
DMS = database;
}
});
//rename the file
var tempName = await rename(file, originalFile, DMS);
//if it was renamed, delete the extra columns
if (!tempName.includes('Not Renamed')){
await columnDelete(tempName);
}
} else if(file.name.includes('undefined')){
console.log(file.name + ' is invalid. Deleted.');
bucket.file(file.name).delete();
}
else {
console.log( file.name + ' is a temporary file. Did not rename.');
}
};
What I expect to be output is as below:
Deleting Columns...
Columns Deleted
Nice and simple, letting us know when it has started and finished.
However, I get this instead:
Deleting Columns...
ApiError: No such object: <file> at at Object.parseHttpRespMessage(......)
finished with status: 'crash'
Which is not wanted for obvious reasons. My next thought is to make sure that the file hasn't been deleted by another instance of the script midway through, but to do that I would have to check to see if the file is being used by another stream, which is, to my knowledge, not possible.
Any ideas out there?
When I was creating the file I called the asynchronous function copy() and moved on, meaning that when trying to access the file it was not finished copying. Unknown to me, the File Object is a reference variable, and did not actually contain the file itself. While the file was copying, the pointer was present but it was pointing to an unfinished file.
Thus, "No Such Object". To fix this, I simply used a callback to make sure that the copying was finished before I was accessing the file.
Thanks to Doug Stevenson for letting me know about the pointer!

Translating a while loop to a promise

I am trying to create a report which is stretching the limits of my understanding of both Node and MySQL.
I have already established that the annual date information I need will need to be called for each year I want to display. I now have a query that can return the correct data that I need, I just need to repeat that within a Node environment.
Coming from a Delphi background the following code would provide me with the data I need
LoadTurnover = function (req, reply) {
const queryDay = "-04-06";
const maxDate = new Date();
let queryYear = 2000;
let qd = new Date(queryYear + queryDay);
let dateArray = [];
while (qd < maxDate) {
// Get the data from the server
let data = getData(sql);
//
let turnoverObj = {};
turnoverObj.date = qd;
turnoverObj.Employees = data[0][0].Employees;
turnoverObj.Leavers = data[1][0].Leavers;
// Add current year data to our result set
dateArray.push(turnoverObj);
// Setup the next year condition
queryYear ++;
qd = new Date(queryYear + queryDay);
}
};
I need to be able send a Promise to the DB server (getData) and populate the turnoverObj and dateArray appropriatly. This needs to repeat until qd is greater than today's date.
You can use .map() function of Bluebird promise:
var Promise = require('bluebird);
let dates = [];
let dateArray = [];
while (qd < maxDate) {
dates.push(qd);
queryYear++;
qd = new Date(queryYear + queryDay);
}
Promise.map(dates, function(singleDate){
return getData(sql).then(function(data){
let turnoverObj = {};
turnoverObj.date = singleDate;
turnoverObj.Employees = data[0][0].Employees;
turnoverObj.Leavers = data[1][0].Leavers;
dateArray.push(turnoverObj);
return;
});
}).then(function(finalResult){
console.log(dateArray);
});
Hope this helps somehow.

Resources