I want to read a .csv. The consensus seems to be to do something like this:
const myCsv = "./my.csv";
// Throws exception if file doesn't exist
// File permissions are also 777 (just to figure this out)
fs.statSync(myCsv);
fs.createReadStream(myCsv)
.pipe(csv())
.on("data", () => console.log('got data')
.on("end", () => console.log('the end')
.on("error", () => console.log('error')
);
When I run this, none of the console.log() statements execute, and no errors/warnings.
Indeed, to try to understand what's going on, I run the following test case:
(() => {
const readStream = fs.createReadStream(myCsv);
readStream.on("open", () => {
console.log("open");
});
readStream.on("data", (chunk) => {
console.log("data");
});
readStream.on("error", (err) => {
console.log("error");
console.log(err);
});
readStream.on("end", () => {
console.log("end");
});
readStream.on("finish", () => {
console.log("finish");
});
})();
After fs.createReadStream(), readStream.fd = null (that can't be good), and none of the console statements executes.
NodeJs version = v16.19.0, and I'm running it on Ubuntu 22.04.
What am I missing? Thanks!
Related
I want to compare the data of two files and for that, I'm reading that file using the fs module but since I want to compare the values so I thought to store the value in an external variable but when I do console.log(budget_details) I get nothing in console. Please someone help. Please point me out if my approach is wrong and if we don't need to do that in nodejs. I'm new to nodejs.
import csv from 'csv-parser'
import fs from 'fs';
let budget_details
const budgetProcessing = (budget_file_path) => {
try{
fs.createReadStream(budget_file_path)
.pipe(csv())
.on('data', (row) => {
budget_details = row
})
.on('end', () => {
console.log('CSV file successfully processed');
});
}
catch(error){
console.log(error)
}
}
budgetProcessing('budget.csv')
console.log(budget_details)
Let's first explain why you don't get the expected result, it doesnt have to do with scope actually:
import csv from 'csv-parser'
import fs from 'fs';
let budget_details
const budgetProcessing = (budget_file_path) => {
try{
fs.createReadStream(budget_file_path)
.pipe(csv())
.on('data', (row) => {
budget_details = row
})
.on('end', () => {
console.log('CSV file successfully processed');
});
}
catch(error){
console.log(error)
}
}
budgetProcessing('budget.csv')
console.log(budget_details)
fs.createReadStream is not itslef exactly asynchronous but then we pipe the returned stream to csv-parser which does event based parsing, so even if you call budgetProcessing before the console.log(budget_details) the stream reading has most likely not runned yet and budget_details is still undefined.
To fix this, you could move this console.log(budget_details) where it is set like so:
let budget_details
const budgetProcessing = (budget_file_path) => {
try{
fs.createReadStream(budget_file_path)
.pipe(csv())
.on('data', (row) => {
budget_details = row
console.log(budget_details)
})
.on('end', () => {
console.log('CSV file successfully processed');
});
}
catch(error){
console.log(error)
}
}
budgetProcessing('budget.csv')
but then the variable itself wouldnt serve any real purpose so instead you could do this:
const budgetProcessing = (budget_file_path, callback) => {
try{
fs.createReadStream(budget_file_path)
.pipe(csv())
.on('data', (row) => {
callback(row)
})
.on('end', () => {
console.log('CSV file successfully processed');
});
}
catch(error){
console.log(error)
}
}
budgetProcessing('budget.csv', (budget_details) => {
console.log(budget_details) // or anything with budget_details
})
Lastly, I want to make clear that the callback will be called for each row of the csv as specified in csv-parser's documentation
your code is not asynchronous. Anything with 'on', which takes a function, would indicate that it is event driven. You need something like:
import csv from 'csv-parser'
import fs from 'fs';
let budget_details
const budgetProcessing = (budget_file_path) => new Promise((resolve, reject) => {
try {
fs.createReadStream(budget_file_path)
.pipe(csv())
.on('data', (row) => {
budget_details = row
})
.on('end', () => {
console.log('CSV file successfully processed');
resolve()
});
} catch (error) {
console.log(error)
reject(error)
}
})
budgetProcessing('budget.csv')
.then(() => console.log(budget_details))
I know for sure that my pullData module is getting the data back from the file read but the function calling it, though it has an await, is not getting the data.
This is the module (./initialise.js) that reads the data:
const fs = require('fs');
const getData = () => {
return new Promise((resolve, reject) => {
fs.readFile('./Sybernika.txt',
{ encoding: 'utf8', flag: 'r' },
function (err, data) {
if (err)
reject(err);
else
resolve(data);
});
});
};
module.exports = {getData};
And this is where it gets called (app.js):
const init = require('./initialise');
const pullData = async () => {
init.getData().then((data) => {
return data;
}).catch((err) => {
console.log(err);
});
};
const start = async() => {
let data = await pullData();
console.log(data);
}
start();
putting 'console.log(data)' just before return(data) in the resolve part of the call shows the data so I know it's being read OK. However, that final console.log shows my data variabkle as being undefined.
Any suggestions?
It's either
const pullData = async () => {
return init.getData().then((data) => {
return data;
}).catch((err) => {
console.log(err);
});
};
or
const pullData = async () =>
init.getData().then((data) => {
return data;
}).catch((err) => {
console.log(err);
});
Both versions make sure a promise returned by then/catch is passed down to the caller.
Hi guys I'm facing problem with my Node.js api with Express when I'm trying to get files from FTP and then send then over my API as base64.
I'm using -> promise-ftp (https://www.npmjs.com/package/promise-ftp).
This is how endpoint looks like:
getData = (req, res, next) => {
const ftp = new PromiseFtp();
let data = [];
ftp.connect({host: 'xxxl',user: 'xxx',password: 'xxx'})
.then(() => {
return ftp.get('xxx.pdf');
}).then((stream) => {
return new Promise((resolve, reject) => {
stream.once('close', resolve);
stream.once('error', reject);
stream.pipe(fs.createReadStream('test.pdf'));
stream
.on('error', (err) => {
return res.send({errorMessage: err});
})
.on('data', (chunk) => data.push(chunk))
.on('end', () => {
const buffer = Buffer.concat(data);
label = buffer.toString('base64');
return res.send(label);
});
});
}).then(() => {
return ftp.end();
});
}
The problem is that I don't want to save this file localy next to api files and when I remove line stream.pipe(fs.createReadStream('test.pdf')); it doesn't work.
I'm not sure what pipe is doing here.
May you please help me?
readable.pipe(writable) is part of Node's Stream API, which transparently writes the data that is read from the readable into the writable stream, handling backpressure for you. Piping the data to the filesystem is unnecessary, and Express Response object implements the writable stream interface so you could just pipe the stream returned from the FTP promise directly to the res object.
getData = async (req, res) => {
const ftp = new PromiseFtp();
try {
await ftp.connect({host: 'xxxl',user: 'xxx',password: 'xxx'});
const stream = await ftp.get('xxx.pdf');
res.type('pdf');
await new Promise((resolve, reject) => {
res.on('finish', resolve);
stream.once('error', reject);
stream.pipe(res);
});
} catch(e) {
console.error(e);
} finally {
await ftp.end();
}
}
If you don't have a Node version that supports async/await, here's a Promise-only version:
getData = (req, res) => {
const ftp = new PromiseFtp();
ftp
.connect({host: 'xxxl',user: 'xxx',password: 'xxx'})
.then(() => ftp.get('xxx.pdf'))
.then(stream => {
res.type('pdf');
return new Promise((resolve, reject) => {
res.on('finish', resolve);
stream.once('error', reject);
stream.pipe(res);
});
})
.catch(e => {
console.error(e);
})
.finally(() => ftp.end());
}
Here you have a good use-case for using a Promise's finally()-method or a try/catch/finally block, which will ensure that ftp.end() is called even if an error occurs or not.
Note that I've deliberately left out sending the error back to clients as doing such things could possibly leak sensitive information. A better solution is to setup proper server-side logging with request context.
I hesitate on what is the best solution between these snippets:
simple_client({
method: 'get',
path: key,
id: 'download'
})
.then(req => new Promise((resolve, reject) => {
const write_stream = fs.createWriteStream(path);
req.on('result', (err, res) => {
if (err) {
reject(err);
}
res.pipe(write_stream);
res.on('end', () => {
console.log('Done!');
resolve();
});
});
req.end();
}))
.then(() => {
console.log(`Is it safe to touch ${path} file here?`);
});
or
simple_client({
method: 'get',
path: key,
id: 'download'
})
.then(req => new Promise((resolve, reject) => {
const write_stream = fs.createWriteStream(path);
req.on('result', (err, res) => {
if (err) {
reject(err);
}
res.pipe(write_stream);
write_stream.on('finish', () => {
console.log('Done!');
resolve();
});
});
req.end();
}))
.then(() => {
console.log(`Is it safe to touch ${path} file here?`);
});
Readable pipe documentation says:
By default, stream.end() is called on the destination Writable stream when the source Readable stream emits 'end'
So am I right to assume the file is fully written and closed when 'end' event has been emitted from the readable stream? Therefore first snippet would be good enough. Thanks for your help!
client.hydrated().then(function (client) {
client.query({ query: x})
.then(function logData(data) {
console.log('results of query: ', data);
fs.writeFileSync('notesdata.json', JSON.stringify(data))
})
.catch(console.error);`enter code here`
});
var xy=fs.readFileSync('notesdata.json');
This is a part of AWS sdk. The last line which reads the data from teh file is executed first and this function is executed afterwards. I know about call back functions but don't know how to implement it here, can i get some help.
Thank you.
The promises are incorrectly used. They should always be chained to provide consistent control flow and error handling:
client.hydrated().then(function (client) {
return client.query({ query: x})
})
.then(function logData(data) {
console.log('results of query: ', data);
fs.writeFileSync('notesdata.json', JSON.stringify(data))
// no real need to read it from file because it's already available as `data`
var xy=fs.readFileSync('notesdata.json');
})
.catch(console.error);
you have two choices, first is use promise chain:
let initClient = () => {
return new Promise((resolve, reject) => {
client.hydrated().then(function (client) {
resolve(client);
}).catch(err => reject(err))
})
}
let query = (client) => {
return new Promise((resolve, reject) => {
client.query({query: x}).then((data) => {
fs.writeFileSync('notesdata.json', JSON.stringify(data))
resolve()
}).catch(err => reject(err))
});
};
initClient().then(query).then(() => {
var xy=fs.readFileSync('notesdata.json');
}).catch(err => {
console.log('err: ', err);
})
if you use async, await the code is much easier:
(async () => {
let client = await client.hydrated();
let data = await client.query({query: x});
console.log('data: ', data);
})().then().catch(err => {
console.log('err: ', err);
})