Node 'readline' how todetect that current line is last line in a file - node.js

I am reading a file line by line with nodejs 'readline'. I am reading a .csv file line by line and converting it to JSON. I start with writing '{' then each line is parsed, formatted, and terminated with ','. I want to do something different with the last line than the preceding lines ie. terminate with a '}' instead of a ','. How do I detect that the current line is the last line.
var readline = require("readline");
var fs = require("fs");
var read_in = readline.createInterface({
input: fs.createReadStream(file),
crlfDelay: Infinity
});
var write_out = fs.createWriteStream("intermediate.json")
write_out.write("{");
read_in.on("line", function (line) {
var a = line.split(",");
var b = "\"" + a[0].trim() + "\" : \"" + a[1] + "\",\r\n"
write_out.write(b);
})
read_in.on("close", function () {
write_out.write("}"); // leaves an incorrectly formatted JSON file
})

A fun way to do this using a csv, but it should work with any type of doc.
Document Contents
one,apple
two,banana
three,orange
Code
// Load document
const originalStream = fs.readFileSync(`my_document.csv`);
// Convert document to hex
const hexStream = new Buffer.from(streambase).toString('hex')
// Output
// 6f6e652c6170706c650a74776f2c62616e616e610a74687265652c6f72616e6765
// New lines are separated by 0a so we can assume thats the start of a new line
// Retrieve last line in hex format
const lastLineHex = hexStream.split("0a").splice(-1)[0]
// Output
// 74687265652c6f72616e6765
const convertLastLineBackToUtf = new Buffer.from(lastLineHex, 'hex').toString();
// Output
// three,orange
To check if they are on the last line you could compare it with this final output.

Related

how to get total number of lines in csv file and append as new column based on line number by using nodejs

I'd like to ask.
How to get the total number of rows in the CSV file using nodejs.
After that, I'd like to append a new column based on line number (into the same file or creating another file) by using nodejs.
Eg of CSV file
Justin,36 years
Jason, 37 years
Now it's 2 line, So i'd like to get
1,Justin,36 years
2,Jason, 37 years
in the same CSV file or creating another CSV file.
Please suggest to me how could I.
Thanks.
This should be relatively straightforward, I'll use the csv-parse module, though you could use any parser really.
We'll map each line, prepending the line number to each one, join and write to the output file.
const { promisify } = require("util");
const fs = require("fs");
const readFile = promisify(fs.readFile);
const writeFile = promisify(fs.writeFile);
const parse = promisify(require("csv-parse"));
async function prependLineNumbers(inputFile, outputFile, delimiter = ",", eol = "\n") {
const input = await readFile(inputFile, { encoding: "utf8" });
const lines = await parse(input, { delimiter });
// Prepend line number to each line...
const output = lines.map((line, lineIndex) => [lineIndex + 1, ...line].join(delimiter)).join(eol)
writeFile(outputFile, output);
}
prependLineNumbers("example.csv", "example_with_line_numbers.csv");

How to append to a file in Node.js but limit the file to a certain size

I would like to truncate a file by newline \n so that it only grows to some max number of lines. How do I do that with something like fs.appendFileSync?
You can address this problem by investigating readline API from node:
const fs = require('fs');
const readline = require('readline');
async function processLineByLine() {
const fileStream = fs.createReadStream('input.txt');
const rl = readline.createInterface({
input: fileStream,
crlfDelay: Infinity
});
for await (const line of rl) {
// count your lines in the file
// you can copy into output stream the content
// of every line till it did not pass the max line number
}
// if the counter is not yet finished using
// rl.write() you can continue appending to the file
}
processLineByLine();
A second idea very similar to this one was answered here:
Parsing huge logfiles in Node.js - read in line-by-line

How to get bytes of string in node.js?

I want to read the size of bytes of a file as I read it. I have this
var path = 'training_data/dat1.txt';
var fs = require("fs"); //Load the filesystem module
var stats = fs.statSync(path);
var fileSizeInBytes = stats["size"];
var accSize = 0;
var lineReader = require('readline').createInterface({
input: fs.createReadStream(path)
});
lineReader.on('line', function (line) {
accSize += Buffer.byteLength(line, 'utf8');
console.log(accSize + "/" + fileSizeInBytes);
});
lineReader.on('close', function() {
console.log('completed!');
});
But it doesn't print out the correct filesize.
7/166
16/166
23/166
32/166
39/166
48/166
55/166
64/166
71/166
80/166
87/166
96/166
103/166
112/166
it prints this for example.
Does anyone know whats wrong?
The lineReader doesn't include the newline \n character in the buffer as each line is read, which is where your bytes are missing.
Try this:
accSize += Buffer.byteLength(line + '\n', 'utf8');
EDIT
If the file being read uses Windows line endings, you'll need to add two characters, since those will also have a carriage return in addition to line feed, denoted as '\r\n'. (see this for more details)

Write a line into a .txt file with Node.js

I want to use Node.js to create a simple logging system which prints a line before the past line into a .txt file. However, I don't know how the file system functionality from Node.js works.
Can someone explain it?
Inserting data into the middle of a text file is not a simple task. If possible, you should append it to the end of your file.
The easiest way to append data some text file is to use build-in fs.appendFile(filename, data[, options], callback) function from fs module:
var fs = require('fs')
fs.appendFile('log.txt', 'new data', function (err) {
if (err) {
// append failed
} else {
// done
}
})
But if you want to write data to log file several times, then it'll be best to use fs.createWriteStream(path[, options]) function instead:
var fs = require('fs')
var logger = fs.createWriteStream('log.txt', {
flags: 'a' // 'a' means appending (old data will be preserved)
})
logger.write('some data') // append string to your file
logger.write('more data') // again
logger.write('and more') // again
Node will keep appending new data to your file every time you'll call .write, until your application will be closed, or until you'll manually close the stream calling .end:
logger.end() // close string
Note that logger.write in the above example does not write to a new line. To write data to a new line:
var writeLine = (line) => logger.write(`\n${line}`);
writeLine('Data written to a new line');
Simply use fs module and something like this:
fs.appendFile('server.log', 'string to append', function (err) {
if (err) return console.log(err);
console.log('Appended!');
});
Step 1
If you have a small file
Read all the file data in to memory
Step 2
Convert file data string into Array
Step 3
Search the array to find a location where you want to insert the text
Step 4
Once you have the location insert your text
yourArray.splice(index,0,"new added test");
Step 5
convert your array to string
yourArray.join("");
Step 6
write your file like so
fs.createWriteStream(yourArray);
This is not advised if your file is too big
I created a log file which prints data into text file using "Winston" logger. The source code is here below,
const { createLogger, format, transports } = require('winston');
var fs = require('fs')
var logger = fs.createWriteStream('Data Log.txt', {
flags: 'a'
})
const os = require('os');
var sleep = require('system-sleep');
var endOfLine = require('os').EOL;
var t = ' ';
var s = ' ';
var q = ' ';
var array1=[];
var array2=[];
var array3=[];
var array4=[];
array1[0] = 78;
array1[1] = 56;
array1[2] = 24;
array1[3] = 34;
for (var n=0;n<4;n++)
{
array2[n]=array1[n].toString();
}
for (var k=0;k<4;k++)
{
array3[k]=Buffer.from(' ');
}
for (var a=0;a<4;a++)
{
array4[a]=Buffer.from(array2[a]);
}
for (m=0;m<4;m++)
{
array4[m].copy(array3[m],0);
}
logger.write('Date'+q);
logger.write('Time'+(q+' '))
logger.write('Data 01'+t);
logger.write('Data 02'+t);
logger.write('Data 03'+t);
logger.write('Data 04'+t)
logger.write(endOfLine);
logger.write(endOfLine);
function mydata() //user defined function
{
logger.write(datechar+s);
logger.write(timechar+s);
for ( n = 0; n < 4; n++)
{
logger.write(array3[n]);
}
logger.write(endOfLine);
}
var now = new Date();
var dateFormat = require('dateformat');
var date = dateFormat(now,"isoDate");
var time = dateFormat(now, "h:MM:ss TT ");
var datechar = date.toString();
var timechar = time.toString();
mydata();
sleep(5*1000);

In Node.js, how to read a file, append a string at a specified line or delete a string from a certain line?

I need to open an existing JavaScript file, check if this string exists:
var LocalStrategy = require('passport-local').Strategy;
If it doesn't, then append it at the top with the rest of require() lines.
In another case, I need to check if that string exists, and if it does, I would like to remove just that line.
I have looked at fs.readFile, fs.writeFile, fs.open but I don't think it is capable of doing what I need. Any suggestions?
This is a simplified script:
var fs = require('fs');
var search = "var LocalStrategy = require('passport-local').Strategy;";
function append (line) {
line = line || 0;
var body = fs.readFileSync('example.js').toString();
if (body.indexOf(search) < 0 ) {
body = body.split('\n');
body.splice(line + 1,0,search);
body = body.filter(function(str){ return str; }); // remove empty lines
var output = body.join('\n');
fs.writeFileSync('example.js', output);
}
}
function remove () {
var body = fs.readFileSync('example.js').toString();
var idx = body.indexOf(search);
if (idx >= 0 ) {
var output = body.substr(0, idx) + body.substr(idx + search.length);
fs.writeFileSync('example.js', output);
}
}

Resources