I'd like to ask.
How to get the total number of rows in the CSV file using nodejs.
After that, I'd like to append a new column based on line number (into the same file or creating another file) by using nodejs.
Eg of CSV file
Justin,36 years
Jason, 37 years
Now it's 2 line, So i'd like to get
1,Justin,36 years
2,Jason, 37 years
in the same CSV file or creating another CSV file.
Please suggest to me how could I.
Thanks.
This should be relatively straightforward, I'll use the csv-parse module, though you could use any parser really.
We'll map each line, prepending the line number to each one, join and write to the output file.
const { promisify } = require("util");
const fs = require("fs");
const readFile = promisify(fs.readFile);
const writeFile = promisify(fs.writeFile);
const parse = promisify(require("csv-parse"));
async function prependLineNumbers(inputFile, outputFile, delimiter = ",", eol = "\n") {
const input = await readFile(inputFile, { encoding: "utf8" });
const lines = await parse(input, { delimiter });
// Prepend line number to each line...
const output = lines.map((line, lineIndex) => [lineIndex + 1, ...line].join(delimiter)).join(eol)
writeFile(outputFile, output);
}
prependLineNumbers("example.csv", "example_with_line_numbers.csv");
Related
I need to export the data generated by this code into a CSV file. I am new to node.js/puppeteer so I am struggling on generating a CSV file.
I understand I can use the fs write function and tried adding this to the end of my code to no avail:
const fs = require('fs');
const csv = await page.$$eval('.product_desc_txt', function(products){
// Iterate over product descriptions
let csvLines = products.map(function(product){
// Inside of each product find product SKU and its price
let productId = document.querySelector(".custom-body-copy").innerText.trim();
let productPrice = document.querySelector("span[data-wishlist-linkfee]").innerText.trim();
// Fomrat them as a csv line
return `${productId};${productPrice}`;
});
// Join all lines into one file
return csvLines.join("\n");
});
fs.writeFileSync("test.csv", csv)
});
You've got csv with data from puppeteer, but don't use it. Just write the data to file:
fs.writeFileSync("test.csv", csv);
Also writing to file this
'${productId};${productPrice}'
won't work, there are no such variables at that place and even if there were, the correct way to format variables into a string is with backticks:
`${productId};${productPrice}`
I would like to truncate a file by newline \n so that it only grows to some max number of lines. How do I do that with something like fs.appendFileSync?
You can address this problem by investigating readline API from node:
const fs = require('fs');
const readline = require('readline');
async function processLineByLine() {
const fileStream = fs.createReadStream('input.txt');
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity
});
for await (const line of rl) {
// count your lines in the file
// you can copy into output stream the content
// of every line till it did not pass the max line number
}
// if the counter is not yet finished using
// rl.write() you can continue appending to the file
}
processLineByLine();
A second idea very similar to this one was answered here:
Parsing huge logfiles in Node.js - read in line-by-line
I know you can read line-by-line with require('readline'), is there a good way to read a file character by character? Perhaps just use readline and then split the line into characters?
I am trying to convert this code:
const fs = require('fs');
const lines = String(fs.readFileSync(x));
for(const c of lines){
// do what I wanna do with the c
}
looking to make this into something like:
fs.createReadStream().pipe(readCharByChar).on('char', c => {
// do what I wanna do with the c
});
Simple for loop
let data = fs.readFileSync('filepath', 'utf-8');
for (const ch of data){
console.log(ch
}
Using forEach
let data = fs.readFileSync('filepath', 'utf-8');
data.split('').forEach(ch => console.log(ch)
I am reading a file line by line with nodejs 'readline'. I am reading a .csv file line by line and converting it to JSON. I start with writing '{' then each line is parsed, formatted, and terminated with ','. I want to do something different with the last line than the preceding lines ie. terminate with a '}' instead of a ','. How do I detect that the current line is the last line.
var readline = require("readline");
var fs = require("fs");
var read_in = readline.createInterface({
input: fs.createReadStream(file),
crlfDelay: Infinity
});
var write_out = fs.createWriteStream("intermediate.json")
write_out.write("{");
read_in.on("line", function (line) {
var a = line.split(",");
var b = "\"" + a[0].trim() + "\" : \"" + a[1] + "\",\r\n"
write_out.write(b);
})
read_in.on("close", function () {
write_out.write("}"); // leaves an incorrectly formatted JSON file
})
A fun way to do this using a csv, but it should work with any type of doc.
Document Contents
one,apple
two,banana
three,orange
Code
// Load document
const originalStream = fs.readFileSync(`my_document.csv`);
// Convert document to hex
const hexStream = new Buffer.from(streambase).toString('hex')
// Output
// 6f6e652c6170706c650a74776f2c62616e616e610a74687265652c6f72616e6765
// New lines are separated by 0a so we can assume thats the start of a new line
// Retrieve last line in hex format
const lastLineHex = hexStream.split("0a").splice(-1)[0]
// Output
// 74687265652c6f72616e6765
const convertLastLineBackToUtf = new Buffer.from(lastLineHex, 'hex').toString();
// Output
// three,orange
To check if they are on the last line you could compare it with this final output.
I have a 1.4GB csv file that I want to go through row-by-row and parse each row. Once each row has been parsed, add that row to the stream and write the output as a tsv file. I thought the below code worked, but it simply adds each row to the end of the previous row without adding in line breaks as I expected. I also tried adding .pipe(split2()) to the line before the .pipe(writeStream) to split the data before writing but that simply froze the application.
Has anybody been successfully reading and writing with this process in node?
var fs = require('fs'),
_ = require('lodash'),
split2 = require('split2'),
through2 = require('through2');
fs.createReadStream('input_file_name.csv')
.pipe(split2())
.pipe(through2.obj(function (chunk, enc, callback) {
// Process the CSV row
var row = _.zipObject(['header1', 'header2', 'header3'], chunk.toString().split(','));
this.push(processRow(row).join('\t')); // does an action to each row
callback()
}))
.pipe(fs.createWriteStream('output_file_name.tsv'));
Realized I was missing a good CSV parser, in lieu of simply splitting on ,, as well as adding a \n to the end of each data string.
var fs = require('fs'),
_ = require('lodash'),
parse = require('csv-parse'),
transform = require('stream-transform');
var parser = parse();
var transformer = transform(function (record, callback) {
var row = _.zipObject(['header1', 'header2', 'header3'], record);
callback(null, processRow(row).join('\t') + '\n');
}, {parallel: 10}
);
fs.createReadStream('input_file_name.csv')
.pipe(parser)
.pipe(transformer)
.pipe(fs.createWriteStream('output_file_name.tsv'));