So I have a json file that changes continously and I need to read it AFTER a value called auth-token is written to the file, here what I get now:
const json = fs.readFileSync("some-json.json")
const headers = JSON.parse(json);
return headers
But it reads the file before anything can be written to it, is there anyway that I can use fs.watchFile() and watch the file UNTIL the value is written?
Thanks
You can use fs.watch although its behavior is a bit unreliable with multiple events triggered upon file change (but I don't think it would be a problem here).
Here is a small sample:
const { watch } = require('fs');
const { readFile } = require('fs/promises');
(async () => {
const result = await new Promise((resolve) => {
const watcher = watch('some-json.json', async (eventType, filename) => {
try {
const fileContent = await readFile(filename);
const headers = JSON.parse(fileContent.toString());
if (headers['auth-token']) { // or whatever test you need here
watcher.close();
resolve(headers);
}
} catch (e) {}
});
});
console.log(result);
})();
Note that if your file gets modified many times before it contains the desired header, it might be preferable to replace the usage of fs.watch by a setInterval to read the file at regular intervals until it contains the value you expect.
Here is what it would look like:
const { readFile } = require('fs/promises');
(async () => {
const waitingTime = 1000;
const result = await new Promise((resolve) => {
const interval = setInterval(async (eventType, filename) => {
const fileContent = await readFile('some-json.json');
try {
const headers = JSON.parse(fileContent.toString());
if (headers['auth-token']) { // or whatever test you need here
clearInterval(interval);
resolve(headers);
}
} catch (e) {}
}, waitingTime);
});
console.log(result);
})();
Related
I'm using Node.js v18.12.1 and Discord.js v14. for developing the Discord bot. I need to read some data from Firebase. I'm confused because I'm used to how Java with Hibernate fetches data differently. Here, I need to use onValue() listener.
My onValue() acts strange. Instead of just reading the data from Firebase, it skips entirely, then it triggers multiple times, each time skipping the body block of its code, and then it actually does the code after.
I've read somewhere on this forum that this can happen because there are more onValue() listeners that are subscribed and they are all fired up. Someone mentioned I need to use the off() function somewhere "before" the onValue(). This confuses me because I'm using this listener in many locations. I need it in each command file, in execute(interaction) functions. You know, when you need to execute slash commands in Discord. I have it something like this:
async execute(interaction) {
const infographicRef = ref(db, '/infographics/arena/' + interaction.options.getString("arena-team"));
var imageUrl = null;
var postUrl = null;
onValue(infographicRef, (snapshot) => {
imageUrl = snapshot.child("image-url").val();
interaction.reply(imageUrl);
})
},
And I planned for each command, in each command.js file to have onValue(). I'm not sure exactly what to do.
Also, I tried to work around this with once() method, I see it in Firebase documentation, but I got the error: ref.once() is not a function.
It seems that after first triggering of onValue method when the body is not executed, my code in interactionCreate.js is triggered as well, it points for a command to be executed again:
const { Events } = require('discord.js');
module.exports = {
name: Events.InteractionCreate,
async execute(interaction) {
if (!interaction.isChatInputCommand()) return;
const command = interaction.client.commands.get(interaction.commandName);
if (!command) {
console.error(`No command matching ${interaction.commandName} was found.`);
return;
}
try {
await command.execute(interaction);
} catch (error) {
console.error(`Error executing ${interaction.commandName}`);
console.error(error);
}
},
};
my bot.js (which is in my case an index file)
const fs = require('node:fs');
const path = require('node:path');
const { Client, Collection, Events, GatewayIntentBits } = require('discord.js');
const { token } = require('./config.json');
const client = new Client({ intents: [GatewayIntentBits.Guilds] });
const eventsPath = path.join(__dirname, 'events');
const eventFiles = fs.readdirSync(eventsPath).filter(file => file.endsWith('.js'));
for (const file of eventFiles) {
const filePath = path.join(eventsPath, file);
const event = require(filePath);
if (event.once) {
client.once(event.name, (...args) => event.execute(...args));
} else {
client.on(event.name, (...args) => event.execute(...args));
}
}
client.commands = new Collection();
const commandsPath = path.join(__dirname, 'commands');
const commandFiles = fs.readdirSync(commandsPath).filter(file => file.endsWith('.js'));
for (const file of commandFiles) {
const filePath = path.join(commandsPath, file);
const command = require(filePath);
client.commands.set(command.data.name, command);
}
client.once(Events.ClientReady, () => {
console.log('Ready!');
});
client.on(Events.InteractionCreate, async interaction => {
if (!interaction.isChatInputCommand()) return;
const command = client.commands.get(interaction.commandName);
if (!command) return;
try {
await command.execute(interaction);
} catch (error) {
console.error(error);
await interaction.reply({ content: 'There was an error while executing this command!', ephemeral: true });
}
});
client.login(token);
The onValue function registers a realtime listener, that continues to monitor the value on the database.
If you want to read a value once, that'd be done with get() function in v9 (which is the equivalent of the once method in earlier SDK versions). Have a look at the code sample in the documentation on reading data once.
I have these excel file contains columns : No, itemCode, startAt, endAt
I loop the data using createReadStream
And I want to detect the invalid datetime from excel file and write the error into test.txt file
Here's my current code
const csv = require('csv-parser');
const fs = require('fs');
const moment = require('moment');
const yesql = require('yesql');
const _ = require('lodash');
const { exit } = require('process');
async function run() {
fs.createReadStream('test.csv')
.pipe(csv())
.on('data', (data) => csvResults.push(data))
.on('end', async () => {
const groups = _.groupBy(csvResults, item => item['itemCode']);
for (const itemCode in groups) {
for (const item of groups[itemCode]) {
const startDateTime = moment(item['startAt'], 'YYYY-MM-DD HH:mm:ss');
const endDateTime = moment(item['endAt'], 'YYYY-MM-DD HH:mm:ss');
if (startDateTime.isAfter(endDateTime)) {
console.error(`Datetime invalid ${item['No']}`);
// Want to write into file if got an error
var stream = fs.createWriteStream("test.txt");
stream.once('open', function(fd) {
stream.write(`${item['No']} : ${startDateTime} ${endDateTime} \n`);
stream.end();
});
continue;
}
}
}
exit();
});
}
run();
Your for loop does NOT wait for the fs.createWriteStream() stuff to complete so you end up trying to write to the same file multiple times at the same time which creates a mess in that file. Likewise, you call exit() before your writes are even done also because the for loop doesn't wait for the stream to finish.
If what you want to do is to append each error condition to your file, then you can do it like this:
const csv = require('csv-parser');
const fs = require('fs');
const fsp = fs.promises;
const moment = require('moment');
const yesql = require('yesql');
const _ = require('lodash');
const { exit } = require('process');
async function run() {
fs.createReadStream('test.csv')
.pipe(csv())
.on('data', (data) => csvResults.push(data))
.on('end', async () => {
const groups = _.groupBy(csvResults, item => item['itemCode']);
for (const itemCode in groups) {
for (const item of groups[itemCode]) {
try {
const startDateTime = moment(item['startAt'], 'YYYY-MM-DD HH:mm:ss');
const endDateTime = moment(item['endAt'], 'YYYY-MM-DD HH:mm:ss');
if (startDateTime.isAfter(endDateTime)) {
console.error(`Datetime invalid ${item['No']}`);
// append into file if got an error
await fsp.appendFile("test.txt", `${item['No']} : ${startDateTime} ${endDateTime} \n`);
}
} catch(e) {
console.log(e);
// not sure what you want to do here if you got an error while appending to the file
}
}
}
exit();
});
}
run();
If you want to just record the first error in the file and then stop further processing, you can use fsp.writeFile() instead of fsp.appendFile() and put your call to exit() right after that call.
I'm trying to crawl several web pages to check broken links and writing the results of the links to a json files, however, after the first file is completed the app crashes with no error popping up...
I'm using Puppeteer to crawl, Bluebird to run each link concurrently and fs to write the files.
WHAT IVE TRIED:
switching file type to '.txt' or '.php', this works but I need to create another loop outside the current workflow to convert the files from '.txt' to '.json'. Renaming the file right after writing to it also causes the app to crash.
using try catch statements for fs.writeFile but it never throws an error
the entire app outside of express, this worked at some point but i trying to use it within the framework
const express = require('express');
const router = express.Router();
const puppeteer = require('puppeteer');
const bluebird = require("bluebird");
const fs = require('fs');
router.get('/', function(req, res, next) {
(async () => {
// Our (multiple) URLs.
const urls = ['https://www.testing.com/allergy-test/', 'https://www.testing.com/genetic-testing/'];
const withBrowser = async (fn) => {
const browser = await puppeteer.launch();
try {
return await fn(browser);
} finally {
await browser.close();
}
}
const withPage = (browser) => async (fn) => {
const page = await browser.newPage();
// Turns request interceptor on.
await page.setRequestInterception(true);
// Ignore all the asset requests, just get the document.
page.on('request', request => {
if (request.resourceType() === 'document' ) {
request.continue();
} else {
request.abort();
}
});
try {
return await fn(page);
} finally {
await page.close();
}
}
const results = await withBrowser(async (browser) => {
return bluebird.map(urls, async (url) => {
return withPage(browser)(async (page) => {
await page.goto(url, {
waitUntil: 'domcontentloaded',
timeout: 0 // Removes timeout.
});
// Search for urls we want to "crawl".
const hrefs = await page.$$eval('a[href^="https://www.testing.com/"]', as => as.map(a => a.href));
// Predefine our arrays.
let links = [];
let redirect = [];
// Loops through each /goto/ url on page
for (const href of Object.entries(hrefs)) {
response = await page.goto(href[1], {
waitUntil: 'domcontentloaded',
timeout: 0 // Remove timeout.
});
const chain = response.request().redirectChain();
const link = {
'source_url': href[1],
'status': response.status(),
'final_url': response.url(),
'redirect_count': chain.length,
};
// Loops through the redirect chain for each href.
for ( const ch of chain) {
redirect = {
status: ch.response().status(),
url: ch.url(),
};
}
// Push all info of target link into links
links.push(link);
}
// JSONify the data.
const linksJson = JSON.stringify(links);
fileName = url.replace('https://www.testing.com/', '');
fileName = fileName.replace(/[^a-zA-Z0-9\-]/g, '');
// Write data to file in /tmp directory.
fs.writeFile(`./tmp/${fileName}.json`, linksJson, (err) => {
if (err) {
return console.log(err);
}
});
});
}, {concurrency: 4}); // How many pages to run at a time.
});
})();
});
module.exports = router;
UPDATE:
So there is nothing wrong with my code... I realized nodemon was stopping the process after each file was saved. Since nodemon would detect a "file change" it kept restarting my server after the first item
I'm creating a small tool for internal user with puppeteer.
Basically I got a csv file with some data i "read" and fill form with.
As I try to cleanup my project to be reusable i'm struggle a little bit:
I create a file name parsecsv.js
const config = require('../config.json');
const parse = require('csv-parse');
const fs = require('fs');
const processFile = async () => {
records = []
const parser = fs
.createReadStream(config.sourceFile)
.pipe(parse({
// CSV options
from_line: 1,
delimiter: ";",
}));
let i =1;
for await (const record of parser) {
records.push(record)
i++;
}
return records
}
const processFileData = async () => {
const records = await processFile()
console.info(records);
return records
}
module.exports ={
processFile, processFileData
}
in an other Js file i made
const parseCSV = require('./src/ParseCsv');
const records = parseCSV.processFileData();
const data = parseCSV.processFile();
console.log(typeof records);
console.table(records);
console.log(typeof data);
console.table(data);
But I never get my data only an empty oject.
How I can get my data to be able to "share" it with other function ?
thanks
as your functions are async ones and they return a promises, you can do something like
const parseCSV = require('./src/ParseCsv');
(async () => {
const records = await parseCSV.processFileData();
const data = await parseCSV.processFile();
console.log(typeof records);
console.table(records);
console.log(typeof data);
console.table(data);
})()
I'm creating a bot that when it receives a message begins to read a text file and responds to the message with the contents of the file.
Unfortunately, I can't get out of this asynchronous hell and I only get errors, undefined or promise
The last experiment was this:
const fs = require('fs');
const readline = require('readline');
// bot.listen("message").reply(responseText())
function readFile(file) {
var text = '';
var readInterface = readline.createInterface({
input: fs.createReadStream(file),
terminal: false
});
readInterface.on('line', function(line) {
linea = line.trim();
console.log(linea);
text += linea;
}).on('close', function() {
return text;
});
});
}
async function responseText() {
var content = await readFile("file.txt");
content.then(function(data) {
return data;
})
}
What I would like to get then is delay the response until I get the contents of the file.
I know that node is based on async but I can't figure out how to handle it!
Thanks all
If you want to use async-await need to create a promise and return it.
function readFile(file) {
return new Promise((res, rej) => {
try {
var text = '';
var readInterface = readline.createInterface({
input: fs.createReadStream(file),
terminal: false
});
readInterface
.on('line', function (line) {
linea = line.trim();
text += linea;
})
.on('close', function () {
res(text);
});
} catch(err){
rej(err)
}
});
}
If your using express.js or any framework built on top of it, you can simply pipe the readstream to the response since express's responses are streams to begin with:
const es = require('event-stream')
...
let getFileStream = path => (
fs.createReadStream(path)
.pipe(es.split())
.pipe(es.map(function (data, cb) {
cb(null
, inspect(JSON.parse(data)))
}))
);
router.get('/message', function (req, res, next) {
let file$ = getFileStream(yourFilePath)
file$.on('error', next).pipe(res)
})
If you need to transform the file content, you can use a transform stream or as shown in the example above, a synchronous event-stream mapping. The idea is to always play around with the file content at stream level to avoid having to load the entire file content in memory.
You don't really want to buffer the whole file content in memory. It can quickly become a problem with huge files on a busy day. what you need is to pipe the file stream directly to the browser. Same principle applies for any kind of consumer.
Of course, if the mechanism is all internal, you should only pass the file path along or the actual stream until you need to actually open the file and do something with the content. In this case, you go back to your stream toolbox, whether it be the native node.js stream API implementation, the event-stream package or some kind of observable library like rxjs.
I had a similar issue in an app that watches a directory for new files, reads the file(s) and returns derived data based on the file content. My Reader function is based on this async example from the nodejs docs. I return options, which contains the context, only after the file is read completely.
const { createReadStream } = require('fs')
const { createInterface } = require('readline')
const { once } = require('events')
// Reader.js
async function Reader (options) {
let { file, cb } = options
let fileStream = createReadStream(file)
const readInterface = createInterface({
input: fileStream,
crlfDelay: Infinity
})
readInterface.on('line', (line) => {
cb(line)
})
await once(readInterface, 'close')
return options
}
module.exports = Reader
I then have a file which imports my Reader and defines how to use it. I define a callback function to pass to the line event listener. I bind the callback to the options object that I pass to my Reader function. In the
readFile function I make sure to return the call to Reader, which is a Promise.
/**
* #desc callback to instruct what to do with each line read
*
* #param {*} line
*/
const readFileLine = function (line) {
linea = line.trim();
console.log(linea);
text += linea;
this.context += linea
}
/**
* #desc once the added file is ready to be processed read file line by line
* #listens {Event} listens for `process` event
*/
const readFile = (options) => {
return Reader(options)
}
/**
* #desc Call the file reader and do what you need with the reponse
*
*/
const getResponseFromFiles = (file) => {
const opts = {}
opts.cb = readFileLine.bind(opts)
opts.context = ''
opts.file = file
readFile(opts)
.then(data => {
process.exitCode = 0
console.log(data)
return data
})
.catch(err => {
process.exitCode = 1
console.log(err.message)
})
}