I have a function e.g.
function writeToStream(writeStream) {
writeStream.write("test\n");
}
How can I make this apply a transform before writing to the final destination? Note: I don't have control over the writeToStream function, I just want to apply a transformation to what it is writing out
const fs = require("fs");
const { Transform } = require("stream");
const upperCaseTr = new Transform({
transform(chunk, encoding, callback) {
//this is never called???
callback(null, chunk.toString().toUpperCase());
},
});
function writeToStream(writeStream) {
writeStream.write("test\n");
}
const r = fs.createWriteStream("test.txt");
writeToStream(upperCaseTr.pipe(r));
With the above code, my custom Transform upperCaseTr is never called
This answer was very helpful in solving this https://stackoverflow.com/a/50537771/2129219
Specifically, upperCaseTr.pipe(r) just returns r, so upperCaseTr is never called.
Here is an answer that uses through2, similar to the linked answer
const fs = require("fs");
const through2 = require("through2");
function writeToStream(writeStream) {
writeStream.write("test test test test test\n");
}
const stream = through2.obj((chunk, enc, callback) => {
callback(null, chunk.toString().toUpperCase());
});
const out = fs.createWriteStream("test.txt");
stream.pipe(out);
writeToStream(stream);
Here is one using a Transform more similar to my original question
const fs = require("fs");
const { Transform } = require("stream");
function writeToStream(writeStream) {
writeStream.write("test test test test test\n");
}
const upperCaseTr = new Transform({
transform(chunk, encoding, callback) {
callback(null, chunk.toString().toUpperCase());
},
});
const out = fs.createWriteStream("test.txt");
upperCaseTr.pipe(out);
writeToStream(upperCaseTr);
Related
I have these excel file contains columns : No, itemCode, startAt, endAt
I loop the data using createReadStream
And I want to detect the invalid datetime from excel file and write the error into test.txt file
Here's my current code
const csv = require('csv-parser');
const fs = require('fs');
const moment = require('moment');
const yesql = require('yesql');
const _ = require('lodash');
const { exit } = require('process');
async function run() {
fs.createReadStream('test.csv')
.pipe(csv())
.on('data', (data) => csvResults.push(data))
.on('end', async () => {
const groups = _.groupBy(csvResults, item => item['itemCode']);
for (const itemCode in groups) {
for (const item of groups[itemCode]) {
const startDateTime = moment(item['startAt'], 'YYYY-MM-DD HH:mm:ss');
const endDateTime = moment(item['endAt'], 'YYYY-MM-DD HH:mm:ss');
if (startDateTime.isAfter(endDateTime)) {
console.error(`Datetime invalid ${item['No']}`);
// Want to write into file if got an error
var stream = fs.createWriteStream("test.txt");
stream.once('open', function(fd) {
stream.write(`${item['No']} : ${startDateTime} ${endDateTime} \n`);
stream.end();
});
continue;
}
}
}
exit();
});
}
run();
Your for loop does NOT wait for the fs.createWriteStream() stuff to complete so you end up trying to write to the same file multiple times at the same time which creates a mess in that file. Likewise, you call exit() before your writes are even done also because the for loop doesn't wait for the stream to finish.
If what you want to do is to append each error condition to your file, then you can do it like this:
const csv = require('csv-parser');
const fs = require('fs');
const fsp = fs.promises;
const moment = require('moment');
const yesql = require('yesql');
const _ = require('lodash');
const { exit } = require('process');
async function run() {
fs.createReadStream('test.csv')
.pipe(csv())
.on('data', (data) => csvResults.push(data))
.on('end', async () => {
const groups = _.groupBy(csvResults, item => item['itemCode']);
for (const itemCode in groups) {
for (const item of groups[itemCode]) {
try {
const startDateTime = moment(item['startAt'], 'YYYY-MM-DD HH:mm:ss');
const endDateTime = moment(item['endAt'], 'YYYY-MM-DD HH:mm:ss');
if (startDateTime.isAfter(endDateTime)) {
console.error(`Datetime invalid ${item['No']}`);
// append into file if got an error
await fsp.appendFile("test.txt", `${item['No']} : ${startDateTime} ${endDateTime} \n`);
}
} catch(e) {
console.log(e);
// not sure what you want to do here if you got an error while appending to the file
}
}
}
exit();
});
}
run();
If you want to just record the first error in the file and then stop further processing, you can use fsp.writeFile() instead of fsp.appendFile() and put your call to exit() right after that call.
So I have a json file that changes continously and I need to read it AFTER a value called auth-token is written to the file, here what I get now:
const json = fs.readFileSync("some-json.json")
const headers = JSON.parse(json);
return headers
But it reads the file before anything can be written to it, is there anyway that I can use fs.watchFile() and watch the file UNTIL the value is written?
Thanks
You can use fs.watch although its behavior is a bit unreliable with multiple events triggered upon file change (but I don't think it would be a problem here).
Here is a small sample:
const { watch } = require('fs');
const { readFile } = require('fs/promises');
(async () => {
const result = await new Promise((resolve) => {
const watcher = watch('some-json.json', async (eventType, filename) => {
try {
const fileContent = await readFile(filename);
const headers = JSON.parse(fileContent.toString());
if (headers['auth-token']) { // or whatever test you need here
watcher.close();
resolve(headers);
}
} catch (e) {}
});
});
console.log(result);
})();
Note that if your file gets modified many times before it contains the desired header, it might be preferable to replace the usage of fs.watch by a setInterval to read the file at regular intervals until it contains the value you expect.
Here is what it would look like:
const { readFile } = require('fs/promises');
(async () => {
const waitingTime = 1000;
const result = await new Promise((resolve) => {
const interval = setInterval(async (eventType, filename) => {
const fileContent = await readFile('some-json.json');
try {
const headers = JSON.parse(fileContent.toString());
if (headers['auth-token']) { // or whatever test you need here
clearInterval(interval);
resolve(headers);
}
} catch (e) {}
}, waitingTime);
});
console.log(result);
})();
module.exports = {
name: "help",
execute(msg, args){
const fs = require("fs");
const commandFiles = fs.readdirSync("./commands/").filter(file => file.endsWith(".js"));
for (const file of commandFiles){
const name = file.slice(0, -3);
const descriptionFileName = name.concat(".desc");
const descriptionFile = `./commands/${descriptionFileName}`;
var output = "Help:";
fs.readFile(descriptionFile, function(err, data){
const helpLine = name.concat(" - ",data.toString());
output = output + "\n" + helpLine
});
msg.channel.send(output);
}
}
}
Expected output:
help - description
ping - description
Output:
Help:
Help:
Any idea why that happens?
Im new at coding and very new at js.
you didn't get the expected result because readFile(file, cb) reads a file asynchronously. This means that it just schedule a callback cb to be executed once the I/O operation has been completed. However the following code:
msg.channel.send(output)
will be executed synchronously so the output will remain with the initial value.
One way to handle this could be with promises, here a partial example based on your code:
module.exports = {
name: 'help',
async execute(msg, args) {
const { readFile, readdir } = require('fs').promises;
const fs = require('fs');
const commandFiles = await readdir('./commands/').filter((file) => file.endsWith('.js'));
const promises = [];
for (const file of commandFiles) {
promises.push(
fs.readFile(file)
)
}
const results = await Promise.all(promises);
// manipulate results as you want
msg.channel.send(results);
},
};
Note that because of the async prefix the exported execute function you need to handle a promise in the consumer of this module
Another approach could be to use a fully parallel control flow pattern
Some references:
promises
async/await
control flow
I want to read from file to stream, pipe the output to a function that will upperCase the content and then write to file. This is my attempt. What am I doing wrong?
const fs = require('fs')
const fred = q => {
return q.toUpperCase()
}
fs.createReadStream('input.txt')
.pipe(fred)
.pipe(fs.createWriteStream('output.txt'))
Currently the error is:
dest.on is not a function
Based on answer from Marco but tidied up:
const fs = require('fs')
const {Transform} = require('stream')
const upperCaseTransform = new Transform({
transform: (chunk, encoding, done) => {
const result = chunk.toString().toUpperCase()
done(null, result)
}
})
fs.createReadStream('input.txt')
.pipe(upperCaseTransform)
.pipe(fs.createWriteStream('output.txt'))
You have to use Transform if you want to "transform" streams. I recommend you to read: https://community.risingstack.com/the-definitive-guide-to-object-streams-in-node-js/
const fs = require('fs')
const Transform = require('stream').Transform;
/// Create the transform stream:
var uppercase = new Transform({
decodeStrings: false
});
uppercase._transform = function(chunk, encoding, done) {
done(null, chunk.toString().toUpperCase());
};
fs.createReadStream('input.txt')
.pipe(uppercase)
.pipe(fs.createWriteStream('output.txt'))
EDIT: You need to call .toString() in chunk because it's a buffer! :)
Using async iteration is the cleaner new way to transform streams:
const fs = require('fs');
(async () => {
const out = fs.createWriteStream('output.txt');
for await (const chunk of fs.createReadStream('input.txt', 'utf8')) {
out.write(chunk.toUpperCase());
}
})();
As you can see, this way is a lot more terse and readable if you already are working in an async function context.
I'm learning reactive programming using RxJS and encounter a case when I need to read a file line-by-line. Actually I solved it using a solution likes:
https://gist.github.com/yvele/447555b1c5060952a279
It works, but I need to use some normal JS code to transform the stream of Buffers to stream of lines. (use "readline" module in example above)
I wonder if there are other ways to transform an Observable of Buffer to Observable of line, using RxJS operators, likes example below.
var Rx = require('rx');
var fs = require('fs');
var lines = Rx.Observable
.fromEvent(rl, 'data') // emits buffers overtime
// some transforms ...
.subscribe(
(line) => console.log(line), // emit string line by line
err => console.log("Error: %s", err),
() => console.log("Completed")
);
You can probably achieve something pretty close to what you want with scan and concatMap.
Something like:
bufferSource
.concat(Rx.Observable.of("\n")) // parens was missing // to make sure we don't miss the last line!
.scan(({ buffer }, b) => {
const splitted = buffer.concat(b).split("\n");
const rest = splitted.pop();
return { buffer: rest, items: splitted };
}, { buffer: "", items: [] })
// Each item here is a pair { buffer: string, items: string[] }
// such that buffer contains the remaining input text that has no newline
// and items contains the lines that have been produced by the last buffer
.concatMap(({ items }) => items)
// we flatten this into a sequence of items (strings)
.subscribe(
item => console.log(item),
err => console.log(err),
() => console.log("Done with this buffer source"),
);
You can use following class
'use strict'
const lineReader = require('line-reader');
const Rx = require('rxjs');
const RxOp = require('rxjs/operators');
class CSVReader {
constructor(filepath {
this.filepath = filepath;
}
readByLines()
{
const source = new Rx.Subject();
lineReader.open(this.filepath, (err, reader)=> {
Rx.of(0).pipe(
RxOp.expand(val => {
reader.nextLine((err2, line) => source.next(line));
return Rx.of(1 + val);
}),
RxOp.takeWhile(_=> {
let has = reader.hasNextLine();
if(!has) source.complete();
return has;
})
).subscribe(_=>_);
})
return source;
}
}
module.exports = CSVReader
and use it as follows
const { bufferCount } = require('rxjs/operators');
let reader = new CSVReader('path/to/file');
reader.readByLines()
.pipe(bufferCount(2)) // chunk size
.subscribe(chunk=> {
console.log({chunk});
});
I would say like this:
const readline = require('readline');
const fs = require('fs');
const path = require('path');
const {fromEvent, race, Observable} = require('rxjs');
const {tap, takeUntil, take, map} = require('rxjs/operators');
const rl = readline.createInterface({
input: fs.createReadStream(path.resolve('./', 'myfile'))
});
let obs = new Observable(observer=>{
rl.on('line', val => observer.next(val)),
rl.on('error', err => observer.error(err)),
rl.on('close', complete => observer.complete(complete))
})
.pipe(tap(line=>console.log(`line: ${line}`)))
obs.subscribe(()=>{},
(e)=>console.log(`Error reading file: ${e}`),
()=>console.log("Read complete"))
An alternative for creating the observable could be:
let obs = fromEvent(rl, 'line')
.pipe(
takeUntil(race(
fromEvent(rl, 'close').pipe(take(1)) ,
fromEvent(rl, 'error').pipe(map((err)=>{throw err}))
)))
Ideally, rxjs could have provided an operator like: fromEvent(emitter, nextEvent, errorEvent, completeEvent ) to help keep the above code even simpler.
I tried a bunch of the above answers and built my own ugly version. Then, I poked around the code on GitHub and found that RxJS handles stream like objects - there's no point in mucking around with events. Just pass a ReadStream to from and it tests it for ReadableStreamLike and then turns it into an AsyncGenerator.
import * as readline from 'node:readline';
import { from } from 'rxjs';
const file = fs.createReadStream(fileName);
const line = readline.createInterface({ input: file });
const line$ = from(line).subscribe({
next: (dat) => { ... },
error: (err) => { ... },
complete: () => { ... }
});