Does anyone know what the best way would be to parse a log file (Salesforce) into a structured JSON object
There are certain sections in the log file that can be identified, like EXECUTION_STARTED,EXECUTION_FINISHED, CODE_UNIT_STARTED CODE_UNIT_FINISHED and many more
There are also time information that I would like to have in the JSON object
Are there any libraries available in nodejs that could be used to accomplish this ?
I was looking into antlr4 but it seems quite complex
-Jani
Looks like a CSV -> JSON conversion
// Reading the file using default
// fs npm package
const fs = require("fs");
csv = fs.readFileSync("CSV_file.csv")
// Convert the data to String and
// split it in an array
var array = csv.toString().split("\r");
// All the rows of the CSV will be
// converted to JSON objects which
// will be added to result in an array
let result = [];
// The array[0] contains all the
// header columns so we store them
// in headers array
let headers = array[0].split(", ")
// Since headers are separated, we
// need to traverse remaining n-1 rows.
for (let i = 1; i < array.length - 1; i++) {
let obj = {}
// Create an empty object to later add
// values of the current row to it
// Declare string str as current array
// value to change the delimiter and
// store the generated string in a new
// string s
let str = array[i]
let s = ''
// By Default, we get the comma separated
// values of a cell in quotes " " so we
// use flag to keep track of quotes and
// split the string accordingly
// If we encounter opening quote (")
// then we keep commas as it is otherwise
// we replace them with pipe |
// We keep adding the characters we
// traverse to a String s
let flag = 0
for (let ch of str) {
if (ch === '"' && flag === 0) {
flag = 1
}
else if (ch === '"' && flag == 1) flag = 0
if (ch === ', ' && flag === 0) ch = '|'
if (ch !== '"') s += ch
}
// Split the string using pipe delimiter |
// and store the values in a properties array
let properties = s.split("|")
// For each header, if the value contains
// multiple comma separated data, then we
// store it in the form of array otherwise
// directly the value is stored
for (let j in headers) {
if (properties[j].includes(", ")) {
obj[headers[j]] = properties[j]
.split(", ").map(item => item.trim())
}
else obj[headers[j]] = properties[j]
}
// Add the generated object to our
// result array
result.push(obj)
}
// Convert the resultant array to json and
// generate the JSON output file.
let json = JSON.stringify(result);
fs.writeFileSync('output.json', json);
Change "CSV_file.csv" above to whatever your CSV file is.
Save the js file as app.js and run using
node app.js
Source: https://www.geeksforgeeks.org/how-to-convert-csv-to-json-file-having-comma-separated-values-in-node-js/
I have a Google Sheet with a bunch of numbers that get added up and totalled using the SUM feature.
The totals are in cells D2 and E2
I need a Google Apps Script that will export each of those cells individually to their own .txt files (labelled D2Total.txt & E2Total.txt) in my google drive.
I need them to overwrite themselves every time the script is run because I will be having them update constantly (every few minutes) using the triggers in G Suite.
This is probably pretty straight forward but I am lame when it comes to coding.
I tried some random code on the internet that was kind of close but would only export the entire spreadsheet.
Append a table of data from Spreadsheet to a Google Doc or Ascii Text File
Assuming you have CSV data in a 'A2' like this '1,2,3,4,5,6,7,8,9,10,11,12
13,14,14,16,17,18,19,20,21,22,23,24 with line delimiters '\n' the you can use this code to create a Google Document and append it as a table or simply export it to an ascii text file.
Export to Google Document:
function exportToGDoc() {
var fn=Utilities.formatDate(new Date(), Session.getScriptTimeZone(), "MM/dd/yy:HH:mm") + '_Report';
var ss=SpreadsheetApp.getActive();
var sh=ss.getSheetByName('Sheet62');//Need Sheet Name
var rg=sh.getRange(1,1);
var csv=rg.getValue();
var vA=Utilities.parseCsv(csv);
var fldr=DriveApp.getFolderById('Your Folder Id');//Need Folder Id
var dk=DocumentApp.create(fn);
var bd=dk.getBody();
bd.appendTable(vA)
dk.saveAndClose();
var fileId=dk.getId();
Drive.Files.update({"parents": [{"id": fldr.getId()}]}, dk.getId());//Need to install Drive API
}
Export to Ascii Text File:
function exportToAsciiText() {
var fn=Utilities.formatDate(new Date(), Session.getScriptTimeZone(), "MM/dd/yy:HH:mm") + '_Report.csv';
var ss=SpreadsheetApp.getActive();
var sh=ss.getSheetByName('Sheet62');//Need Sheet Name
var rg=sh.getRange(1,1);//A1
var csv=rg.getValue();
var fldr=DriveApp.getFolderById('Your Folder Id');//Need Folder Id
var file=fldr.createFile(fn, csv, MimeType.PLAIN_TEXT);
}
Reloading the same ascii text file
function exportToAsciiText() {
var fn='MyReport.csv';//Your report name
var ss=SpreadsheetApp.getActive();
var sh=ss.getSheetByName('Sheet62');//Need Sheet Name
var rg=sh.getRange(1,1);//A1
var csv=rg.getValue();
var fldr=DriveApp.getFolderById('Your Folder Id');//Need Folder Id
var files=fldr.getFilesByName(fn);
var n=0;
while(files.hasNext()) {
var file=files.next();
n++;
}
switch (n) {
case 0:
var file=fldr.createFile(fn, csv, MimeType.PLAIN_TEXT);
break;
case 1:
file.setContent(csv);
break;
default:
SpreadsheetApp.getUi().showModalDialog(HtmlService.createHtmlOutput(Utilities.formatString('You have more than one file named %s in folder named %s.',fn,fldr.getName())), 'Error: To many files with that name.');
break;
}
}
Drive API
DocumentApp Class
SpreadsheetApp Class
I need to update a table that is shared. The info for this table is first collected into an Excel table files and then uploaded to google drive every day. I found some code that converts the .xls files to a google spreadsheet file, I need to copy the data from this converted file and update the shared one each day. My problem now is that the file I will use for updating the shared spreadsheet will be different eachday, so how can I have the script to get the new file ID eachday. I need these updates to be done automatically each day.
This is the code I have found so far but can't seem to get it to work. First part converts the .xls file to google spreadsheet file that part works but i cant seem to get the function for updating the shared table to work, i cant get the ID of the created file. Would also be nice if a function an be added to the code to to delete the files after they have been converted and the shared table has been updated with them.
function importXLS(){
var files = DriveApp.searchFiles('title contains ".xls"');
var destinationFolderId = "ID of folder with .xls file that is being converted each day";
var existingFileNames = getFilesInFolder(destinationFolderId);
while(files.hasNext()){
var xFile = files.next();
var name = xFile.getName();
try {
if (existingFileNames[name] && (name.indexOf('.xls')>-1)) {
var ID = xFile.getId();
var xBlob = xFile.getBlob();
var newFile = { title : name,
key : ID,
'parents':[{"id": destinationFolderId}]
}
file = Drive.Files.insert(newFile, xBlob, {
convert: true
});
}
} catch (error) {
console.error("Error with file " + name + ": " + error);
}
}
}
/**
* Get an object of all file names in the specified folder.
* #param {string} folderId
* #returns {Object} files - {filename: true}
*/
function getFilesInFolder(folderId) {
var folder = DriveApp.getFolderById(folderId);
var filesIterator = folder.getFiles();
var files = {};
while (filesIterator.hasNext()) {
var file = filesIterator.next();
files[file.getName()] = true;
}
return files;
}
function CopyContent() {
var ID = importXLS(ID);
var source = SpreadsheetApp.openById(importXLS(ID));//the source needs to be the new file i get eachday
var sheet = source.getSheets()[0];
var destination = SpreadsheetApp.openById("ID of shared table here");
sheet.copyTo(destination);
}
Use Paste Special > Paste values only
The copypastetype to use is PASTE_VALUES
Example from https://developers.google.com/apps-script/reference/spreadsheet/range#copytodestination-copypastetype-transposed
// The code below copies only the values of the first 5 columns over to the 6th column.
var sheet = SpreadsheetApp.getActiveSheet();
sheet.getRange("A:E").copyTo(sheet.getRange("F1"), spreadsheetApp.CopyPasteType.PASTE_VALUES);
In my user interface (angularjs) I create new row. Each row have file upload button. I want to upload all files together with metadata and save each row in one call. The complex object which I post to Nodejs API is somewhat like below
var activity = {
"Id" : 1,
"Name" : "Test",
"Steps" : [
{
"StepId":1,
"FileUrl": {fileObject} // this property if bound with the file upload directive 'ng-file-upload' by Daniel Farid
"Description" : "Save this file"
},
{
"StepId":2,
"FileUrl": {fileObject} // this property if bound with the file upload directive 'ng-file-upload' by Daniel Farid
"Description" : "Save this file2"
}
]
}
This JSON will be posted to Node js API. On Nodejs side I am using multer to save the uploaded files to server. I get all the files in API using multer's .any() method, but I get the posted object without Steps[x].FileUrl property.
The file object that has the information about the field name in which this file was added. Below is the info I see in debugger.
Array[2]
length:2
[0]:Object
destination:"C:\DeleteThis\"
encoding:"7bit"
fieldname:"Steps[0][FileUrl]"
filename:"ed13d2a61cb38c43f1f46a221855a896"
mimetype:"image/png"
originalname:"deploy.png"
path:"C:\DeleteThis\ed13d2a61cb38c43f1f46a221855a896"
size:2347
[1]:Object
Now what I want to do it, since My complex object that is posted does not have Steps[0].FileUrl property, I want to iterate for each file (i.e. req.files) and use fieldname to create this property and assign the originalName as value to it.
How I am trying to do it
var deployment = req.body;
if(req.files){
var app = _config.getApplicationConfig(req.body.ApplicationId);
req.files.forEach(function(f){
//Move file to the deployment folder.
_utils.createDirIfNotExist(app.packageDir);
var newPath = _utils.DetermineFileName(f.originalname, app.packageDir);
_fs.renameSync(f.path, path.join(app.packageDir,newPath));
var newFileName = path.basename(newPath);
//set the file url to corresponding field
var evalExp = "deployment." + f.fieldname; //I get evalExpression as "deployment.Steps[0][FileUrl]"
eval(evalExp); //Here it fails saying FileUrl is not defined
evalExp = "deployment." + f.fieldname + "= \"" + newFileName.toString() + "\"";
eval(evalExp);
});
}
Does anyone know how can as assign the property to an object at run time?
I have found solution to this as below
I have wrote a function that converts the [] notation to . notation ie. myobj[myprop] to myobj.myprop
var convertToDotNotation = function (keyPath) {
var bracketSyntaxRegex = new RegExp(/\[([^0-9])\w+\]/g); //matches the javascript property defined in [] syntax but not an array
var matches = keyPath.match(bracketSyntaxRegex)
if(matches && matches.length > 0){
matches.forEach(function(p){
//replace '[' with '.' and ']' with emptyspace
var dotSyntax = p.replace("[",".").replace("]","");
keyPath = keyPath.replace(p,dotSyntax);
});
}
return keyPath;
}
This will give me the '.' notation which can dynamically create the property and set the value
var newFileName = "MyFile.pdf";
var evalExp = "deployment[0].[FileUrl]" ;
var temp = convertToDotNotation(evalExp);
eval(temp + "= \"" + newFileName + "\"");
Hope it helps someone.
I am a nodejs programmer . Now I have a table of data that I want to save in Excel File format . How do I go about doing this ?
I found a few Node libraries . But most of them are Excel Parsers rather than Excel Writers .I am using a Linux Server . Hence need something that can run on Linux . Please let me know if there are any helpful libraries that you know of .
Or is there a way I can convert a CSV file to an xls file ( programmatically ) ?
excel4node is a maintained, native Excel file creator built from the official specification. It's similar to, but more maintained than msexcel-builder mentioned in the other answer.
// Require library
var excel = require('excel4node');
// Create a new instance of a Workbook class
var workbook = new excel.Workbook();
// Add Worksheets to the workbook
var worksheet = workbook.addWorksheet('Sheet 1');
var worksheet2 = workbook.addWorksheet('Sheet 2');
// Create a reusable style
var style = workbook.createStyle({
font: {
color: '#FF0800',
size: 12
},
numberFormat: '$#,##0.00; ($#,##0.00); -'
});
// Set value of cell A1 to 100 as a number type styled with paramaters of style
worksheet.cell(1,1).number(100).style(style);
// Set value of cell B1 to 300 as a number type styled with paramaters of style
worksheet.cell(1,2).number(200).style(style);
// Set value of cell C1 to a formula styled with paramaters of style
worksheet.cell(1,3).formula('A1 + B1').style(style);
// Set value of cell A2 to 'string' styled with paramaters of style
worksheet.cell(2,1).string('string').style(style);
// Set value of cell A3 to true as a boolean type styled with paramaters of style but with an adjustment to the font size.
worksheet.cell(3,1).bool(true).style(style).style({font: {size: 14}});
workbook.write('Excel.xlsx');
I just figured a simple way out . This works -
Just create a file with Tabs as delimiters ( similar to CSV but replace comma with Tab ). Save it with extension .XLS . The file can be opened in Excel .
Some code to help --
var fs = require('fs');
var writeStream = fs.createWriteStream("file.xls");
var header="Sl No"+"\t"+" Age"+"\t"+"Name"+"\n";
var row1 = "0"+"\t"+" 21"+"\t"+"Rob"+"\n";
var row2 = "1"+"\t"+" 22"+"\t"+"bob"+"\n";
writeStream.write(header);
writeStream.write(row1);
writeStream.write(row2);
writeStream.close();
This creates the file in XLS file format . It doesnt work if you try XLSX instead of XLS .
Use msexcel-builder. Install it with:
npm install msexcel-builder
Then:
// Create a new workbook file in current working-path
var workbook = excelbuilder.createWorkbook('./', 'sample.xlsx')
// Create a new worksheet with 10 columns and 12 rows
var sheet1 = workbook.createSheet('sheet1', 10, 12);
// Fill some data
sheet1.set(1, 1, 'I am title');
for (var i = 2; i < 5; i++)
sheet1.set(i, 1, 'test'+i);
// Save it
workbook.save(function(ok){
if (!ok)
workbook.cancel();
else
console.log('congratulations, your workbook created');
});
Although this question has several answers, they may now be a little dated.
New readers may prefer to consider the xlsx or "sheetsJS" package, which now seems to now be by far the most popular node package for this use case.
The current top answer recommends excel4node , which does look quite good - but the latter package seems less maintained (and far less popular) than the former.
Answering the question directly, using xlsx:
const XLSX = require('xlsx');
/* create a new blank workbook */
const wb = XLSX.utils.book_new();
// Do stuff, write data
//
//
// write the workbook object to a file
XLSX.writeFile(workbook, 'out.xlsx');
You should check ExcelJS
Works with CSV and XLSX formats.
Great for reading/writing XLSX streams. I've used it to stream an XLSX download to an Express response object, basically like this:
app.get('/some/route', function(req, res) {
res.writeHead(200, {
'Content-Disposition': 'attachment; filename="file.xlsx"',
'Transfer-Encoding': 'chunked',
'Content-Type': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
})
var workbook = new Excel.stream.xlsx.WorkbookWriter({ stream: res })
var worksheet = workbook.addWorksheet('some-worksheet')
worksheet.addRow(['foo', 'bar']).commit()
worksheet.commit()
workbook.commit()
}
Works great for large files, performs much better than excel4node (got huge memory usage & Node process "out of memory" crash after nearly 5 minutes for a file containing 4 million cells in 20 sheets) since its streaming capabilities are much more limited (does not allows to "commit()" data to retrieve chunks as soon as they can be generated)
See also this SO answer.
Using fs package we can create excel/CSV file from JSON data.
Step 1: Store JSON data in a variable (here it is in jsn variable).
Step 2: Create empty string variable(here it is data).
Step 3: Append every property of jsn to string variable data, while appending put '\t' in-between 2 cells and '\n' after completing the row.
Code:
var fs = require('fs');
var jsn = [{
"name": "Nilesh",
"school": "RDTC",
"marks": "77"
},{
"name": "Sagar",
"school": "RC",
"marks": "99.99"
},{
"name": "Prashant",
"school": "Solapur",
"marks": "100"
}];
var data='';
for (var i = 0; i < jsn.length; i++) {
data=data+jsn[i].name+'\t'+jsn[i].school+'\t'+jsn[i].marks+'\n';
}
fs.appendFile('Filename.xls', data, (err) => {
if (err) throw err;
console.log('File created');
});
XLSx in the new Office is just a zipped collection of XML and other files. So you could generate that and zip it accordingly.
Bonus: you can create a very nice template with styles and so on:
Create a template in 'your favorite spreadsheet program'
Save it as ODS or XLSx
Unzip the contents
Use it as base and fill content.xml (or xl/worksheets/sheet1.xml) with your data
Zip it all before serving
However I found ODS (openoffice) much more approachable (excel can still open it), here is what I found in content.xml
<table:table-row table:style-name="ro1">
<table:table-cell office:value-type="string" table:style-name="ce1">
<text:p>here be a1</text:p>
</table:table-cell>
<table:table-cell office:value-type="string" table:style-name="ce1">
<text:p>here is b1</text:p>
</table:table-cell>
<table:table-cell table:number-columns-repeated="16382"/>
</table:table-row>
Or - build on #Jamaica Geek's answer, using Express - to avoid saving and reading a file:
res.attachment('file.xls');
var header="Sl No"+"\t"+" Age"+"\t"+"Name"+"\n";
var row1 = [0,21,'BOB'].join('\t')
var row2 = [0,22,'bob'].join('\t');
var c = header + row1 + row2;
return res.send(c);
install exceljs
npm i exceljs --save
import exceljs
var Excel = require('exceljs');
var workbook = new Excel.Workbook();
create workbook
var options = {
filename: __dirname+'/Reports/'+reportName,
useStyles: true,
useSharedStrings: true
};
var workbook = new Excel.stream.xlsx.WorkbookWriter(options);
after create worksheet
var worksheet = workbook.addWorksheet('Rate Sheet',{properties:{tabColor:{argb:'FFC0000'}}});
in worksheet.column array you pass column name in header and array key
pass in key
worksheet.columns = [
{ header: 'column name', key: 'array key', width: 35},
{ header: 'column name', key: 'array key', width: 35},
{ header: 'column name', key: 'array key', width: 20},
];
after using forEach loop append row one by one in exel file
array.forEach(function(row){ worksheet.addRow(row); })
you can also perfome loop on each exel row and cell
worksheet.eachRow(function(row, rowNumber) {
console.log('Row ' + rowNumber + ' = ' + JSON.stringify(row.values));
});
row.eachCell(function(cell, colNumber) {
console.log('Cell ' + colNumber + ' = ' + cell.value);
});
Use exceljs library for creating and writing into existing excel sheets.
You can check this tutorial for detailed explanation.
link
First parameter is the source file
Second parameter is the separator
Third parameter is the resulting file (*.xlsx)
Attention: to increase node heap use: node --max-old-space-size=4096 index.js
const fs = require('fs');
var xl = require('excel4node');
const data = fs.readFileSync(process.argv[2], 'utf-8');
const lines = data.split(/\r?\n/);
const linesFromOne = [null].concat(lines);
var wb = new xl.Workbook();
var ws = wb.addWorksheet('Planilha 1');
for (let j=1;j<=linesFromOne.length-1;j++){
// Create a reusable style
var style = wb.createStyle({
font: {
color: '#050000',
size: 12,
},
});
pieces = linesFromOne[j].split(process.argv[3])
pieces.forEach((element, index) =>{
ws.cell(j, index+1).string(element)
.style(style);
});
}
wb.write(process.argv[4]);