I have used the following async method to check if file exists:
var fs = require('fs');
module.exports.exists = function(path, cb) {
fs.stat(require('path').join(__dirname, '../public', path), function(err, stat) {
if (err == null) {
console.log('File exists :' + path);
cb(true);
} else if (err.code == 'ENOENT') {
//file does not exist
console.log('No file: ' + path);
cb(false);
} else {
console.log('Some other error: ', err.code);
cb(false);
}
});
}
If file exists, i redirect to the file in pathStr. If not, display that resource does not exist. Although this is asynchronous, i notice that the web server is much slower and accepts fewer requests with this fs.stat method(if this is commented out, the performance is significantly better). Is there a way in node.js to speed up file exists or not check asynchronously ?
Related
I have a simple trigger set on file upload to Firebase. It reads uploaded file, process it and saves results to database. It works for a while, after that it crashes and stops working. Usually uploading function helps. Does anybody have an idea what might be the reason? Am I getting out of memory or ... ?
Here is the code :
const {Storage} = require('#google-cloud/storage');
const path = require('path');
const storage = new Storage();
exports.processLogs = functions
.region('europe-west1')
.storage
.object()
.onFinalize(async (object) => {
const filename = path.basename(object.name);
const bucket = storage.bucket(object.bucket);
try {
await bucket.file(filename).download(async (err, contents) => {
if (err) {
console.log('error', err);
return null
}
//Proces file and store into db
// (...)
bucket.file(filename).delete();
});
} catch(e){
console.log('error',e)
}
});
Error I am getting is :
Anonymous caller does not have storage.objects.get access to project-name.appspot.com/CrTwBuyNR2-log-2020-1-16-12-18.csv.'
thanks
As you are using Firebase, I recommend initialising the bucket from firebase-admin instead of #google-cloud/storage directly. This will sort out permissions so that security rules are skipped.
In your code you have also incorrectly mixed the callback and async/await APIs. Because this code is running in a Cloud Function, I recommend exclusively using Promises and async/await.
The code below is a rewrite with the following changes:
Code has been split into logical blocks
No callback API usage (see File#download)
Each block will log and throw errors separately for easier debugging
One-line log messages (i.e. no stack trace)
Leaves full error logging to Cloud Functions (makes finding erroneous runs easier)
const admin = require('firebase-admin');
const functions = require('firebase-functions');
admin.initializeApp();
exports.processLogs = functions.region('europe-west1').storage.object()
.onFinalize(async (object) => {
const bucketRef = admin.storage().bucket(object.bucket);
const fileRef = bucketRef.file(object.name);
console.log('Processing "' + object.id + '"...');
// 1) DOWNLOAD
let [contents] = await fileRef.download()
.catch((err) => {
console.log('DOWNLOAD FAILED: ', (err.code ? err.code + ': ' : '') + err.message);
throw err;
});
// 2) PARSE
let dataToUpload = {};
try {
// Transform file contents
dataToUpload = JSON.parse(contents);
} catch (err) {
console.log('PARSE FAILED: ', (err.code ? err.code + ': ' : '') + err.message);
throw err;
}
// 3) DATABASE SET
const dbRef = admin.database().ref('path/to/data');
await dbRef.set(dataToUpload)
.catch((err) => {
console.log('DATABASE SET FAILED: ', (err.code ? err.code + ': ' : '') + err.message);
throw err;
});
// 4) CLEANUP
await fileRef.delete()
.catch((err) => {
console.log('CLEANUP FAILED: ', (err.code ? err.code + ': ' : '') + err.message);
throw err;
});
// 5) LOG SUCCESS
console.log('SUCCEEDED');
});
The log messages above can also be bundled into a helper function if desired:
function logAndRethrowError(err, name) {
console.log((name || 'ERROR') + ': ', ((err.code ? err.code + ': ' : '') + err.message) || err);
throw err;
}
// Usage:
let [contents] = await fileRef.download()
.catch(err => logAndRethrowError(err, 'DOWNLOAD FAILED'));
try {
// ...
} catch (err) { logAndRethrowError(err, 'PARSE FAILED') }
This looks like a problem with Security Rules not being set properly? Please try this document or video.
For starters try this:
service firebase.storage {
match /b/{bucket}/o {
match /{allPaths=**} {
allow read, write: if true;
}
}
}
And if it proves successful set Storage Rules properly.
Looking through the fs docs, I am looking for a flag that I can use with fs.appendFile, where an error will be raised if the path does not exist.
I am seeing flags that pertain to raising errors if the path does already exist, but I am not seeing flags that will raise errors if the path does not exist -
https://nodejs.org/api/fs.html
First off, I assume you mean fs.appendFile(), since the fs.append() you refer to is not in the fs module.
There does not appear to be a flag that opens the file for appending that returns an error if the file does not exist. You could write one yourself. Here's a general idea for how to do so:
fs.appendToFileIfExist = function(file, data, encoding, callback) {
// check for optional encoding argument
if (typeof encoding === "function") {
callback = encoding;
encoding = 'utf8';
}
// r+ opens file for reading and writing. Error occurs if the file does
fs.open(file, 'r+', function(err, fd) {
if (err) return callback(err);
function done(err) {
fs.close(fd, function(close_err) {
fd = null;
if (!err && close_err) {
// if no error passed in and there was a close error, return that
return callback(close_err);
} else {
// otherwise return error passed in
callback(err);
}
});
}
// file is open here, call done(err) when we're done to clean up open file
// get length of file so we know how to append
fs.fstat(fd, function(err, stats) {
if (err) return done(err);
// write data to the end of the file
fs.write(fd, data, stats.size, encoding, function(err) {
done(err);
});
});
});
}
You could, of course, just test to see if the file exists before calling fs.appendFile(), but that is not recommended because of race conditions. Instead, it is recommended that you set the right flags on fs.open() and let that trigger an error if the file does not exist.
I need to find a way return a string to digest in my primary code block as well as a callback or something to start working on the rest of the code in my primary code block once the value for digest is returned.
Please Help!
Here is my current code that does not work.
var digest = checkIntegrity(filePath, res[3]);
//digest always come back undefined and never matches res[2] so file always deletes
if (digest === 0){
console.log('File Inaccessible');
} else {
if (digest === res[2]){
createNewFile();
} else {
console.log('File hash doesn't match');
delBadFile();
}
}
function checkIntegrity(filePath, algorithm, cb){
console.log('in checkIntegrity');
var hash = crypto.createHash(algorithm);
var digest;
//see if file is there
fs.stat(filePath, function(fileErr, fileStats){
if(fileErr){
//error accessing file, most likely file does not exist
return 0;
} else {
//file exists
var fileIn = fs.createReadStream(filePath);
fileIn.on('data', function(chunk){
if (chunk) {
hash.update(chunk);
}
});
fileIn.on('end', function(){
return hash.digest('hex');
});
}
});
}
You're checkIntegrity function is asynchronous, i.e. it accepts a callback. Any value that you wish to pass as a result of that function should be passed as an argument to that callback function. What is happening in your example is that checkIntegrity is calling out to fs.stat (which is also asynchronous) and then returns undefined straightaway - before fs.stat has chance to complete.
You have a choice:
Change the call from fs.stat to fs.statSync. That is a synchronous version of the stat function.
Change your code to use callbacks properly:
checkIntegrity(filePath, res[3], function (err, digest) {
if (err) return console.error(err);
if (digest === 0) {
console.log('File Inaccessible');
} else {
if (digest === res[2]){
createNewFile();
} else {
console.log('File hash doesn\'t match');
delBadFile();
}
}
});
function checkIntegrity(filePath, algorithm, cb){
console.log('in checkIntegrity');
var hash = crypto.createHash(algorithm);
var digest;
//see if file is there
fs.stat(filePath, function(fileErr, fileStats) {
if(fileErr){
//error accessing file, most likely file does not exist
return cb(fileErr);
} else {
//file exists
var fileIn = fs.createReadStream(filePath);
fileIn.on('data', function(chunk){
if (chunk) {
hash.update(chunk);
}
});
fileIn.on('end', function() {
cb(null, hash.digest('hes'));
});
}
});
}
In my opinion, asynchronous code and callbacks are such a fundamental part of Node.js I would encourage you to go for option 2. It is definitely worthwhile learning. There are hundreds of sites out there like callbackhell.com that will do a much better job at explaining callbacks.
fs.exists is now deprecated for a decent reason that I should try to open a file and catch error to be sure nothing is possible to delete the file in between checking and opening. But if I need to create a new file instead of opening an existing file, how do I guarantee that there is no file before I try to create it?
You can't. You can however, create a new file or open an existing one if it exists:
fs.open("/path", "a+", function(err, data){ // open for reading and appending
if(err) return handleError(err);
// work with file here, if file does not exist it will be created
});
Alternatively, open it with "ax+" which will error if it already exists, letting you handle the error.
module.exports = fs.existsSync || function existsSync(filePath){
try{
fs.statSync(filePath);
}catch(err){
if(err.code == 'ENOENT') return false;
}
return true;
};
https://gist.github.com/FGRibreau/3323836
https://stackoverflow.com/a/31545073/2435443
fs = require('fs') ;
var path = 'sth' ;
fs.stat(path, function(err, stat) {
if (err) {
if ('ENOENT' == err.code) {
//file did'nt exist so for example send 404 to client
} else {
//it is a server error so for example send 500 to client
}
} else {
//every thing was ok so for example you can read it and send it to client
}
} );
Sorry, just starting with node. This might be a very novice question.
Let's say I have some code which reads some files from a directory in the file system:
var fs = require('fs');
fs.readdir(__dirname + '/myfiles', function (err, files) {
if (err) throw err;
files.forEach(function (fileName) {
fs.readFile(__dirname + '/myfiles/' + fileName, function (err, data) {
if (err) throw err;
console.log('finished reading file ' + fileName + ': ' + data);
module.exports.files.push(data);
});
});
});
Note that all of this occurs asynchronously. Let's also say I have a Mocha test which executes this code:
describe('fileProvider', function () {
describe('#files', function () {
it.only('files array not empty', function () {
assert(fileProvider.files.length > 0, 'files.length is zero');
});
});
});
The mocha test runs before the files are finished being read. I know this because I see the console.log statement after I see the little dot that indicates a mocha test being run (at least I think that is what is being indicated). Also, if I surround the assert with a setTimeout, the assert passes.
How should I structure my code so that I can ensure the async file operations are completed? Note that this is not just a problem with testing - I need the files to be loaded fully before I can do real work in my app as well.
I don't think the right answer is to read files synchronously, because that will block the Node request / response loop, right?
Bonus question:
Even if I put the assert in a setTimeout with a 0 timeout value, the test still passes. Is this because just putting it in a setTimeout kicks it to the end of the processing chain or something so the filesystem work finishes first?
You can implement a complete callback after all files have been read.
exports.files = [];
exports.initialize = initialize;
function initialize(callback) {
var fs = require('fs');
fs.readdir(__dirname + '/myfiles', function (err, files) {
if (err) throw err;
files.forEach(function (fileName) {
fs.readFile(__dirname + '/myfiles/' + fileName, function (err, data) {
if (err) throw err;
console.log('finished reading file ' + fileName + ': ' + data);
exports.files.push(data);
if (exports.files.length == files.length) {
callback();
}
});
});
}
You can call the file operation method by doing something like:
var f = require('./files.js');
if (f.files.length < 1) {
console.log('initializing');
f.initialize(function () {
console.log('After: ' + f.files.length);
var another = require('./files.js');
console.log('Another module: ' + another.files.length);
});
}
EDIT: Since you want to only have to call this once, you could initialize it once when the application loads. According to Node.js documentation, modules are cached after the first time they are loaded. The two above examples have been edited as well.
To avoid being caught up in nested callbacks. You might want to use async's each that will allow you to do the tasks asynchronously in a non-blocking manner:
https://github.com/caolan/async#each
I think that's a good test, the same thing would happen in any app that used your module, i.e. it's code could be run before files is set. What you need to do is create a callback like #making3 suggests, or use promises. I haven't used mocha, but there's a section on ascynchronous calls. You could export the promise itself:
module.exports.getFiles = new Promise((resolve, reject) => {
datas = [];
fs.readdir(__dirname + '/myfiles', function (err, files) {
if (err) {
reject(err);
return;
}
files.forEach(function (fileName) {
fs.readFile(__dirname + '/myfiles/' + fileName, function (err, data) {
if (err) {
reject(err);
return;
}
console.log('finished reading file ' + fileName + ': ' + data);
datas.push(data);
if (datas.length == files.length) {
resolve(datas);
}
});
});
});
}
chai-as-promissed lets you work directly with promises using eventually, or you can use the callback passed to your test I think:
describe('fileProvider', function () {
describe('#files', function () {
it.only('files array not empty', function (done) {
fileProvider.getFiles.then(function(value) {
assert(value.length > 0, 'files.length is zero');
done();
}, function(err) {
done(err);
})
});
});
});