I'm trying to convert large amount of json data to excel and tried couple of modules
Below are my findings, if anyone used better node module which handle more data please let me know so that i can explore
json2xls
JSON array with 100000 length took 402574ms
once i exceeded to 200000 it failed with this error FATAL ERROR: CALL_AND_RETRY_2 Allocation failed - process out of memory
node-xls
JSON array with 100000 length took 444578ms
I tried this in windows 7 system with 8GB RAM, Intel Core i7, CPU # 2.10Ghz - 2.70Ghz
First push your data into a temporary array with required column and then convert it into xls, I have done it in following manner:
// use the below package to convert json to xls
var json2xls = require('json2xls');
json.forEach(function(instance, indexx,record){
var tempArry = {
'ColoumnName1' : record[indexx].columnNameVlaue,
'ColoumnName2' : record[indexx].columnNameVlaue,
'ColoumnName3' : record[indexx].columnNameVlaue,
'ColoumnName4' : record[indexx].columnNameVlaue
}
jsonArray.push(tempArry);
});
//this code is for sorting xls with required value
jsonArray.sort(function(a, b) {
return parseFloat(b.ColoumnName4) - parseFloat(a.ColoumnName4);
});
var xls = json2xls(jsonArray);
fs.writeFileSync('yourXLName.xlsx', xls, 'binary');
Dont try to add all the data into the excel file, use the specific columns you want in the file to be saved.
If its a nodejs project then do this,
const xlsx = require("xlsx")//npm install xlsx
const fs = require("fs")//npm install fs
var rawFile = fs.readFileSync("./datas.json")//dir of your json file as param
var raw = JSON.parse(rawFile)
var files = []
for (each in raw){
files.push(raw[each])
}
var obj = files.map((e) =>{
return e
})
var newWB = xlsx.book_new()
var newWS = xlsx.utils.json_to_sheet(obj)
xlsx.utils.book_append_sheet(newWB,newWS,"name")//workbook name as param
xlsx.writeFile(newWB,"Sample-Sales-Data.xlsx")//file name as param
In Ratul Das' answer, there is a typo on the following line:
var newWB = xlsx.book_new()
The code should read:
var newWB = xslx.utils.book_new()
The snippet below is the code I use to generate an Excel spreadsheet from an array of JSON objects named imageList:
const workSheet = XLSX.utils.json_to_sheet(imageList);
const workBook = XLSX.utils.book_new();
XLSX.utils.book_append_sheet(workBook, workSheet, "Product Image Catalog");
// Generate buffer
XLSX.write(workBook, {bookType: 'xlsx', type: 'buffer'})
// Binary String
XLSX.write(workBook, {bookType: 'xlsx', type: 'binary'})
XLSX.writeFile(workBook, 'image-catalog.xlsx')
Building the buffer helps with large amounts of data.
If your JSON is already properly formatted, you juste have to do:
const json2xls = require('json2xls');
// Example JSON
const json = [{firstName: 'Bob', name: 'Lennon'}, {firstName: 'Jack', name: 'Sparrow'}]
const xls = json2xls(json);
fs.writeFileSync('exported.xlsx', xls, 'binary');
Works fine, and very simple.
Related
All,
I have two simple arrays as below and want to dump the data in to excel in Nodejs. Please suggest a good npm packages for the same.
var Headers = ['ChangeId', 'ChangeDescription', 'ChangeDate', 'Enhancement/Fix', 'ExcutorTeam'];
var Data = ['INC1234', 'Multiple Cert cleanup', '04/07/2022', 'Enhancement', 'IlevelSupport'];
You can use xlsx npm package and do something like this
This is a quick example that works, things could be improved but it will at least give you a starting point
const xlsx = require('xlsx');
let Headers = ['ChangeId', 'ChangeDescription', 'ChangeDate', 'Enhancement/Fix', 'ExcutorTeam'];
let Data = ['INC1234', 'Multiple Cert cleanup', '04/07/2022', 'Enhancement', 'IlevelSupport'];
let workbook = xlsx.utils.book_new();
let worksheet = xlsx.utils.aoa_to_sheet([]);
xlsx.utils.book_append_sheet(workbook, worksheet);
xlsx.utils.sheet_add_aoa(worksheet, [Headers], { origin: 'A1' });
xlsx.utils.sheet_add_aoa(worksheet, [Data], { origin: 'A2' });
xlsx.writeFile(workbook, "Test.xlsx");
CSV format is just comma separated string with new lines. With that in mind we want to create a string that has the format:
ChangeId,ChangeDescription,ChangeDate,Enhancement/Fix,ExcutorTeam
INC1234,Multiple Cert cleanup,04/07/2022,Enhancement,IlevelSupport
We can iterate over Data in Headers.length sized chunks:
const numColumns = Headers.length;
let file = `${Headers.join(",")}\n`;
for (let i = 0; i < Data.length; i += numColumns) {
file += `${Data.slice(i, i + numColumns).join(",")}\n`;
}
fs.writeFileSync("file.csv", file);
Hope that gets you started in the right direction.
I want to read xls file in node js using the code below.
const xlsxFile = require('read-excel-file/node');
const xlsx = require('xlsx');
const workbook = xlsx.readFile('POTemplate.xls');
const sheet_name_list = workbook.SheetNames;
var data = xlsx.utils.sheet_to_json(workbook.Sheets[sheet_name_list[0]]);
var data5 = JSON.stringify(data[6]);
console.log(data5);
by using this code how can we get each cell value row-wise?
Just iterate over array of objects
Example
data.forEach(el => console.table(el))
I need to export the data generated by this code into a CSV file. I am new to node.js/puppeteer so I am struggling on generating a CSV file.
I understand I can use the fs write function and tried adding this to the end of my code to no avail:
const fs = require('fs');
const csv = await page.$$eval('.product_desc_txt', function(products){
// Iterate over product descriptions
let csvLines = products.map(function(product){
// Inside of each product find product SKU and its price
let productId = document.querySelector(".custom-body-copy").innerText.trim();
let productPrice = document.querySelector("span[data-wishlist-linkfee]").innerText.trim();
// Fomrat them as a csv line
return `${productId};${productPrice}`;
});
// Join all lines into one file
return csvLines.join("\n");
});
fs.writeFileSync("test.csv", csv)
});
You've got csv with data from puppeteer, but don't use it. Just write the data to file:
fs.writeFileSync("test.csv", csv);
Also writing to file this
'${productId};${productPrice}'
won't work, there are no such variables at that place and even if there were, the correct way to format variables into a string is with backticks:
`${productId};${productPrice}`
I am using xlsx package. Please suggest any other package if available
I tried this but i am not getting complete data
var sheet, jsonData,
excelString = new Buffer(base64, 'base64').toString('binary'),
workbook = excelParser.read(excelString, {type: 'binary'});
if (workbook.Sheets['Sheet1']) {
sheet = workbook.Sheets['Sheet1'];
workbook.Sheets['Sheet1'] = sheet;
jsonData = excelParser.utils.sheet_to_json(workbook.Sheets['Sheet1'], {});
you can use
npm install excel
use this.
var parseXlsx = require('excel');
parseXlsx('Spreadsheet.xlsx', function(err, data) {
if(err) throw err;
// data is an array of arrays
});
and you can also visit Converting excel file to json in nodejs
Note: Make sure your Excel file is not opened by any other software (especially Excel !).
Happy coding.
var XLSX = require('xlsx');
var workbook = XLSX.readFile('Test1.xlsx');
var first_sheet_name = workbook.SheetNames[0];
// to read the value of individual cell
var address_of_cell = 'C4';
var worksheet = workbook.Sheets[first_sheet_name];
var desired_cell = worksheet[address_of_cell];
var desired_value = (desired_cell ? desired_cell.v : undefined);
console.log(desired_value);
For further clarification read through the documentation of xlsx https://www.npmjs.com/package/xlsx
var XLSX = require('xlsx');
var workbook = XLSX.readFile(path);
var sheet_name_list = workbook.SheetNames;
var xlData = XLSX.utils.sheet_to_json(workbook.Sheets[sheet_name_list[0]]);
for (let i = 0; i < xlData.length; i++) {
let employee_choice1 = xlData[i].employee_choice
var employee_choice = employee_choice1.split(',')
//implement your condition
}
I have several arrays that contain data that I would like to export, each array to a txt file, in order to be analyzed using MATLAB.
Let's say my array is:
var xPosition = [];
// some algorithm that adds content to xPosition
// TODO: export array into a txt file let's call it x_n.txt
It would be great to store each element of an array per line.
I have found a guide for the solution to my question in this post. The following code is what I ended up using:
var fs = require('fs');
var xPosition = [];
// some algorithm that adds content to xPosition
var file = fs.createWriteStream('./positions/x_n.txt');
file.on('error', function(err) { /* error handling */ });
xPosition.forEach(function(v) { file.write(v + '\n'); });
file.end();
The solution you found works, but here's how I'd have done it:
var fs = require('fs');
var xPosition = [1,2,3]; // Generate this
var fileName = './positions/x_n.txt';
fs.writeFileSync(fileName, xPosition.join('\n'));
This uses node's synchronous file writing capability, which is ideal for your purposes. You don't have to open or close file handles, etc. I'd use streams only if I had gigabytes of data to write out.