Read the last line of a CSV file and extract one value - node.js

New to Node.js and trying to pull a value from the very last line of a CSV file. Here is the CSV:
Unit ID,Date,Time,Audio File
Log File Created,3/6/2013,11:18:25 AM,file:\\\C:\Users\Ben\Documents\1_03-06-2013_1114-50.mp3
1,3/6/2013,11:20:24 AM,file:\\\C:\AlertLog\1_03-06-2013_1120-24.mp3
1,3/6/2013,11:20:39 AM,file:\\\C:\AlertLog\1_03-06-2013_1120-24.mp3
The part I am trying to grab is file:\\\C:\AlertLog\1_03-06-2013_1120-24.mp3 - preferably getting rid of the file:\\\ part.
Sorry that I do not have any code to show, just have a few hours of experience with Node.js and cannot seem to find any docs on how to accomplish something like this. Any help would be appreciated. Thanks!

Regular file
Read the file like a regular file, split the file contents into lines, take the last line, split by a comma and take the last part.
var fs = require('fs'); // file system module
fs.readFile('/path/to/file.csv', 'utf-8', function(err, data) {
if (err) throw err;
var lines = data.trim().split('\n');
var lastLine = lines.slice(-1)[0];
var fields = lastLine.split(',');
var audioFile = fields.slice(-1)[0].replace('file:\\\\', '');
console.log(audioFile);
});
File System module documentation
CSV parser
You can also use the node-csv-parser module.
var fs = require('fs');
var csv = require('csv');
csv()
.from.stream(fs.createReadStream('/path/to/file.csv'))
.to.array(function(data, count) {
var lastLine = data.slice(-1)[0];
var audioFile = lastLine.slice(-1)[0].replace('file:\\\\', '');
console.log(audioFile);
});

I did it by reading the file backwards until the last line was read:
Update 2022 (use modern javascript and read buffer only once)
import { open } from 'node:fs/promises';
import path from 'node:path';
import { fileURLToPath } from 'url';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const fileName = path.join(__dirname, 'test.txt');
async function readLastLine(name) {
var line = '';
var chunkSize = 200 // how many characters to read from the end of file
const fd = await open(name);
const st = await fd.stat();
const buf = Buffer.alloc(chunkSize);
const len = buf.length;
const { bytesRead, buffer } = await fd.read(buf, 0, len, st.size - len)
for (let i = len - 1; i > -1; i--) {
const isEol = buffer[i] === 0x0a // 0x0a == '\n'
const isCtrl = buffer[i] < 0x20 // 0-31 are ASCII control characters
if (isEol && line.length > 0) {
break;
} else if (!isCtrl && !isEol) {
line = String.fromCharCode(buffer[i]) + line;
}
}
fd.close();
return line;
}
try {
const line = await readLastLine(fileName)
console.log(line);
} catch (err) {
console.error(err);
}
Old 2014 answer
var fs = require('fs');
var path = require('path');
var fileName = path.join(__dirname, 'test.txt');
var readLastLine = function(name, callback) {
fs.stat(name, function(err, stat) {
fs.open(name, 'r', function(err, fd) {
if(err) throw err;
var i = 0;
var line = '';
var readPrevious = function(buf) {
fs.read(fd, buf, 0, buf.length, stat.size-buf.length-i, function(err, bytesRead, buffer) {
if(err) throw err;
line = String.fromCharCode(buffer[0]) + line;
if (buffer[0] === 0x0a) { //0x0a == '\n'
callback(line);
} else {
i++;
readPrevious(new Buffer(1));
}
});
}
readPrevious(new Buffer(1));
});
});
}
readLastLine(fileName, function(line) {
console.log(line);
});

Related

NodeJS Appending File differently to source file

So I have some code that takes an input file, Replaces strings in that file and appends those replaced strings to another file.
The problem is that the append function outputs a slightly different file (not where the replaces are). The strings are replaced fine its just some of the lines are swapped and some lines do not have a break.
Code:
const settings = require('../settings.json')
const lineReader = require('line-reader');
const sleep = require('system-sleep')
const Promise = require('bluebird');
var eachLine = Promise.promisify(lineReader.eachLine);
const fs = require('fs')
function makeid(length) {
var result = '';
var characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
var charactersLength = characters.length;
for ( var i = 0; i < length; i++ ) {
result += characters.charAt(Math.floor(Math.random() *
charactersLength));
}
return result;
}
fs.stat('./unblacklisted.rbxlx', function(err, stat) {
if(err == null) {
fs.unlink('./unblacklisted.rbxlx', function(err) {
if (err) throw err
})
} else if(err.code === 'ENOENT') {
// file does not exist
} else {
console.log('Some other error: ', err.code);
}
})
eachLine(`./${settings['CorrectMapName/fileName']}`, function(line) {
if (typeof line.split('referent=')[1] !== 'undefined') {
let treatedline = line.replace(/(\r\n|\n|\r)/gm, "");
let test = treatedline.split('referent=')[1]
let hello = (treatedline.replace(test, `"RBX${Buffer.from(makeid(17), 'ascii').toString('hex').toUpperCase()}">`))
fs.appendFile('./unblacklisted.rbxlx',`${hello}\n`, function (err) {
if (err) throw err;
});
} else {
let treatedline = line.replace(/(\r\n|\n|\r)/gm, "");
fs.appendFile('./unblacklisted.rbxlx', `${treatedline}\n`, function (err) {
if (err) throw err;
return;
});
}
}).then(function() {
__callback()
})
The input and output files are XML and I will provide two pastebin URL's for the input file and output file
Input: https://pastebin.com/cHbzL1W6
Output: https://pastebin.com/C53YBwMy
These look very similar but if you run them through a file comparer some lines are switched and vise versa
Would love to fix this, Any help would be GREATLY appreciated
Using jifriend00's first comment I saw that fs.appendFile() was not appending Synchronously
I fixed this by using fs.appendFileSync()
If anyone reading has this problem just use fs.appendFileSync() :D

Increment the filename if it already exists when saving files in node.js?

Is there a simple way to make node.js increment the filename of a file (i.e., append a number, so that it doesn't overwrite previous files) when saving it?
Below is my attempt:
// can i make this a one-liner?:)
async function incrementIfExists(dirPath, fileName, data, increment=1) {
const fs = require('fs'),
path = require('path'),
errorFunc = (err) => console.error(err);
// Get the last file saved with same fileName (i.e., the one that has the greatest increment number), if there is one
let lastFile = await fs.promises.readdir(dirPath)
.then(files => {
let result = '';
for (const name of files) {
if (!name.startsWith(fileName)) continue;
if ((name.length < result.length) || (name.length === result.length && name < result)) continue;
result = name;
}
return result;
})
.catch(errorFunc);
if (lastFile) {
const lastIncrementNr = Number(lastFile.slice((fileName + '_').length));
if (increment <= lastIncrementNr) increment = lastIncrementNr + 1;
}
fileName = path.join(dirPath, fileName);
while (true) {
let breakLoop = await fs.promises.writeFile(lastFile ? fileName + '_' + increment : fileName, data, {encoding: 'utf8', flag: 'wx'})
.then(fd => true)
.catch(err => {
if (err.code === 'EEXIST') {console.log(err);
return false;
}
throw err;
});
if (breakLoop) break;
increment++;
}
}
incrementIfExists('.', fileName, data);
Related:
How to not overwrite file in node.js
Creating a file only if it doesn't exist in Node.js
I use something similar to version uploaded image files on disk. I decided to use the "EEXIST" error to increment the number rather than explicitly iterating the files in the directory.
const writeFile = async(filename, data, increment = 0) => {
const name = `${path.basename(filename, path.extname(filename))}${increment || ""}${path.extname(filename)}`
return await fs.writeFile(name, data, { encoding: 'utf8', flag: 'wx' }).catch(async ex => {
if (ex.code === "EEXIST") return await writeFile(filename, data, increment += 1)
throw ex
}) || name
}
const unversionedFile = await writeFile("./file.txt", "hello world")
const version1File = await writeFile("./file.txt", "hello world")
const version2File = await writeFile("./file.txt", "hello world")

Node.js reading textfiles in current directory and validate

This is actually the answer from my previous question.... the supplied code works for me. All I needed to do was retain a file counter (global) and in the read after validating add to array (global) which gets passed back to rendering process.
// Called from Render Process.
ipcMain.on('search-text-files', (event, arg) => {
const fs = require('fs');
const path = require('path');
var txtArr = [];
var fileName = '';
var fCtr = 0;
fs.readdir(__dirname+'/', function (err, items) {
if (err) {
throw err;
}
// loop through directory items
for (var i=0; i<items.length; i++) {
if (path.extname(items[i].toString() == '.txt') {
fctr+=1;
fileName = items[i].toString();
// read the file & validate
fs.readfile(__dirname+'/'+fileName, (err, data) {
if (err) {
throw err;
}
var checkArr[];
var curFile = '';
checkArr = data.toString().split('\r');
// access contents line by line
for (var line=0; line<checkArr.length; line++) {
... perform some sort of validation
... assign curFile from contents
}
if (file is valid) {
txtArr.push(curfile);
}
fCtr-=1;
if (fCtr == 0) {
event.sender.send('text-files-found', txtArr);
}
});
}
}
});
});

how to parse data from two csv in node.js

I am very new to node.js. I have one script that will parse the csv and generate the required output file. Now I want to fetch some of column data from another csv at the same time and add that value to the output file.
Script :
var csv = require('csv');
var fs = require('fs');
var progress = require('progress-stream');
var date = require('date-and-time');
var indexStat = 0;
var header = [];
var headerLine = '$lang=en\n\nINSERT_UPDATE Customer;uid;name;address;phoneno'
var delimeter = ',';
var semicolon = ';';
var inputFile = __dirname+'/project/customer.csv';
var outputFile = __dirname+'/project/customer.impex';
var inputFileName = 'customer.csv';
var outputFileName = 'customer.impex';
function generateRecord(json) {
var record = semicolon + json.uid + semicolon + json.name + semicolon + json.address;
return record;
}
var writeStream = fs.createWriteStream(outputFile);
var parser = csv.parse({
delimiter: delimeter
}, function (err, data) {
if (err) {
console.log(err);
}
});
var transformer = csv.transform(function (data) {
var line = '';
if (indexStat == 0) {
header = data;
var line = headerLine;
} else {
var line = generateRecord(generateRecordObject(data));
}
indexStat++;
writeStream.write(line + '\r\n');
});
function stringSplitter(dataRow) {
var str = dataRow.toString();
return str.split(delimeter);
}
function generateRecordObject(dataRow) {
var record = {};
dataRow.forEach(function (value, index) {
if (header[index] != '') {
record[header[index].toLowerCase()] = value;
}
});
return record;
}
var stat = fs.statSync(inputFile);
var str = progress({
length: stat.size,
time: 100
});
str.on('progress', function (progress) {
writeCompletedPercentageForRead(progress.percentage, progress.eta, progress.runtime, progress.speed);
});
function removeLineBreaks(obj) {
obj = obj.replace(/\\N/g, '');
obj = obj.replace(/&/g, '&');
return obj;
}
function writeCompletedPercentageForRead(p, e, r, s) {
process.stdout.clearLine();
process.stdout.cursorTo(0);
process.stdout.write(`${inputFileName} read in progress to write ${outputFileName} ... Completed:${parseInt(p, 10)} %, ETA:${e} seconds, Elapsed:${r} seconds, Rate:${parseInt(s/1024, 10)} KBps`);
};
fs.createReadStream(inputFile).pipe(str).pipe(parser).pipe(transformer);
customer.csv ->
uid,name,address
1234,manish,bangalore
The above script is working fine and generating customer.impex file as below
INSERT_UPDATE Customer;uid;name;address;phoneno
;1234;manish;bangalore
Now i want to populate phoneno as well but phoneno field is define in another csv file lets say 'customer_phone.csv'.
customer_phone.csv -
uid,phoneno
1234,98777767467
I want to match uid column of customer.csv with customer_phone.csv and get the phoneno from that csv. Finally i want to add phoneno in the customer.impex outfile file.
I have no idea how to parse two csv here and generate the file. Any help ?
var Converter = require("csvtojson").Converter;
var fs = require("fs");
var pathToMainCSV = "./customer.csv";
var pathToContactsCSV = "./contact.csv";
var customerConverter = new Converter({});
var contactConverter = new Converter({});
var contanierObj = {};
function processContacts() {
fs.createReadStream(pathToContactsCSV).pipe(contactConverter);
}
function createImpexFile() {
var headerLine = '$lang=en\n\nINSERT_UPDATE Customer;uid;name;address;phoneno;\n';
Object.keys(contanierObj).forEach(obj => {
Object.keys(contanierObj[obj]).forEach(data => {
headerLine += contanierObj[obj][data] + ';';
});
headerLine += '\n';
});
fs.writeFile("./new.impex", headerLine, function(err) {
if (err) {
return console.log(err);
}
console.log("The file was saved!");
});
}
customerConverter.on("end_parsed", function(jsonArray) {
jsonArray.forEach(v => {
contanierObj[v.uid] = v;
});
processContacts();
});
contactConverter.on("end_parsed", function(jsonArray) {
jsonArray.forEach(v => {
contanierObj[v.uid].contact = v.phoneno;
});
createImpexFile();
});
fs.createReadStream(pathToMainCSV).pipe(customerConverter);
Kindly use something like i have done above, format the string according to your needs

Nodejs streams writablestream drain event not firing

I am trying to read in a large file, do some computation and then write to a much bigger file. To prevent excessive memory consumption, I am using streams. The problem that I am facing is that the writestream is not firing the "drain" event, which signals that the writes have been flushed to disk. In order to prevent "back-pressure", I am waiting for the drain event to be fired before I start writing to the buffer again. While debugging I found that after a .write() call returns false and the line fvfileStream.once('drain', test) is executed, the program just stops and does not do anything.
Here is the code:
var fs = require('fs');
//a test function I created to see if the callback is called after drain.
var test = function(){
console.log("Done Draining");
}
fs.readFile('/another/file/to/be/read', {
encoding: "utf8"
}, function(err, data) {
if (err) throw err;
//Make an array containing tags.
var tags = data.split('\n');
//create a write stream.
var fvfileStream = fs.createWriteStream('/path/TagFeatureVectors.csv');
//read in the question posts
var qfileStream = fs.createReadStream('/Big/file/QuestionsWithTags.csv', {
encoding: "utf8"
});
var partialRow = null;
var writable = true;
var count = 0;
var doRead = function() {
var qData = qfileStream.read();
var questions = qData.split('\n');
if (partialRow != null) {
questions[0] = partialRow + questions[0];
partialRow = null;
}
var lastRow = questions[questions.length - 1];
if (lastRow.charAt(lastRow.length - 1) != '\n') {
partialRow = lastRow;
}
questions.forEach(function(row, index, array) {
count++;
var fields = row.split(',');
console.log("Processing question number: " + count + " id: " + fields[0]);
var tagString = fields[1];
var regex = new RegExp(/<([^>]+)>/g);
tags.forEach(function(tag, index, array) {
var found = false;
var questionTags;
while ((questionTags = regex.exec(tagString)) != null) {
var currentTag = questionTags[1]
if (currentTag === tag) {
found = true;
break;
}
};
//This is where the writestream is written to
if (found) {
writable = fvfileStream.write("1,", "utf8");
}else {
writable = fvfileStream.write("0,","utf8");
}
});
});
fvfileStream.write("\n");
}
qfileStream.on('readable', function() {
if (writable) {
doRead();
} else {
//Waiting for drain event.
fvfileStream.once('drain', test);
}
});
qfileStream.on('end', function() {
fvfileStream.end();
});
});
Updated
Based on advise provided by #loganfsmyth, I implemented transform streams, but still ran into the same issue. Here is my updated code:
var fs = require('fs');
var stream = require('stream');
var util = require('util');
var Transform = stream.Transform;
function FVCreator(options) {
// allow use without new
if (!(this instanceof FVCreator)) {
return new FVCreator(options);
}
// init Transform
Transform.call(this, options);
}
util.inherits(FVCreator, Transform);
var partialRow = null;
var count = 0;
var tags;
FVCreator.prototype._transform = function(chunk, enc, cb) {
var that = this;
var questions = chunk.toString().split('\n');
if (partialRow != null) {
questions[0] = partialRow + questions[0];
partialRow = null;
}
var lastRow = questions[questions.length - 1];
if (lastRow.charAt(lastRow.length - 1) != '\n') {
partialRow = lastRow;
questions.splice(questions.length - 1, 1);
}
questions.forEach(function(row, index, array) {
count++;
var fields = row.split(',');
console.log("Processing question number: " + count + " id: " + fields[0]);
var tagString = fields[1];
var regex = new RegExp(/<([^>]+)>/g);
tags.forEach(function(tag, index, array) {
var found = false;
var questionTags;
while ((questionTags = regex.exec(tagString)) != null) {
var currentTag = questionTags[1]
if (currentTag === tag) {
found = true;
break;
}
};
if (found) {
that.push("1,", "utf8");
} else {
that.push("0,", "utf8");
}
});
});
this.push("\n", "utf8");
cb();
};
fs.readFile('/another/file/to/be/read', {
encoding: "utf8"
}, function(err, data) {
if (err) throw err;
//Make an array containing tags.
tags = data.split('\n');
//write to a file.
var fvfileStream = fs.createWriteStream('/path/TagFeatureVectors.csv');
//read in the question posts
var qfileStream = fs.createReadStream('/large/file/to/be/read', {
encoding: "utf8"
});
var fvc = new FVCreator();
qfileStream.pipe(fvc).pipe(fvfileStream);
});
I am running this on OSX Yosemite.

Resources