Bit of context: I am learning nodejs/express and have got a small application that in the end should function as an api. I have got a routes directory with a few subdirectories containing files such as Post.js or Users.js, each file defining a few routes for Posts, Users etc.
I have the following bit of code in my index.js placed in routes directly:
public readDir(path, app) {
let dir = path != null ? path : __dirname;
fs.readdir(dir, (err, elements) => {
if(err) throw err;
if(!elements) return;
elements.forEach(element => {
if(element === "init.js") return;
let new_path = x.join(dir, "/", element);
fs.lstat(new_path , (err, stat) => {
if(err) throw err;
if(stat.isDirectory()) {
this.readDir(new_path , app);
} else if(stat.isFile()) {
require(PATH)(app);
}
});
});
});
}
What it does is the following: It reads the Routes directory with each subdirectory by calling itself in a loop and requiring any file that is found (path module is imported as x, I should probably change that sometime). This works fortunately, every route is mapped properly and can be accessed by making a call with postman / insomnia.
My question would be how this could be done better, primarily performance wise whilst still keeping the structure of multiple files and/or directories?
I have already seen this answer and this one and though both seem like great and functional answers I was wondering which would be the better option?
Any pointers would be great!
Related
O/S is ubuntu 16, node version is 4.2.6.
I have source / development code and run / distribution code, the source.js files are minified and mangled to create equivalent source.min.js files, and I would like for node js require to automatically search for .min.js files as well as .js files.
But as I have a lot of files, I would prefer not to have to go through every require in every file and instead modify the built-in require() function.
This is a very simple implementation of a stand alone function, but how can I modify the built-in function to behave the same way ?
function require(file){
try{return require(file)}
catch(e){return require(file+='.min.js')}
}
You can achieve this by modifying prototype function require of Module class and apply it globally
Here is how you can do it :
var pathModule = require('path');
var assert = require('assert').ok;
module.constructor.prototype.require = function (path) {
var self = this;
assert(typeof path === 'string', 'path must be a string');
assert(path, 'missing path');
try {
return self.constructor._load(path, self);
} catch (err) {
// if module not found, we have nothing to do, simply throw it back.
if (err.code === 'MODULE_NOT_FOUND') {
throw err;
}
// resolve the path to get absolute path
path = pathModule.resolve(__dirname, path+".min.js")
// Write to log or whatever
console.log('Error in file: ' + path);
}
}
I have a file(data.file an image), I would like to save this image. Now an image with the same name could exist before it. I would like to overwrite if so or create it if it does not exist since before. I read that the flag "w" should do this.
Code:
fs.writeFile('/avatar/myFile.png', data.file, {
flag: "w"
}, function(err) {
if (err) {
return console.log(err);
}
console.log("The file was saved!");
});
Error:
[Error: ENOENT: no such file or directory, open '/avatar/myFile.png']
errno: -2,
code: 'ENOENT',
syscall: 'open',
path: '/avatar/myFile.png'
This is probably because you are trying to write to root of file system instead of your app directory '/avatar/myFile.png' -> __dirname + '/avatar/myFile.png' should do the trick, also check if folder exists. node.js won't create parent folder for you.
Many of us are getting this error because parent path does not exist. E.g. you have /tmp directory available but there is no folder "foo" and you are writing to /tmp/foo/bar.txt.
To solve this, you can use mkdirp - adapted from How to write file if parent folder doesn't exist?
Option A) Using Callbacks
const mkdirp = require('mkdirp');
const fs = require('fs');
const getDirName = require('path').dirname;
function writeFile(path, contents, cb) {
mkdirp(getDirName(path), function (err) {
if (err) return cb(err);
fs.writeFile(path, contents, cb);
});
}
Option B) Using Async/Await
Or if you have an environment where you can use async/await:
const mkdirp = require('mkdirp');
const fs = require('fs');
const writeFile = async (path, content) => {
await mkdirp(path);
fs.writeFileSync(path, content);
}
I solved a similar problem where I was trying to create a file with a name that contained characters that are not allowed. Watch out for that as well because it gives the same error message.
I ran into this error when creating some nested folders asynchronously right before creating the files. The destination folders wouldn't always be created before promises to write the files started. I solved this by using mkdirSync instead of 'mkdir' in order to create the folders synchronously.
try {
fs.mkdirSync(DestinationFolder, { recursive: true } );
} catch (e) {
console.log('Cannot create folder ', e);
}
fs.writeFile(path.join(DestinationFolder, fileName), 'File Content Here', (err) => {
if (err) throw err;
});
Actually, the error message for the file names that are not allowed in Linux/ Unix system comes up with the same error which is extremely confusing. Please check the file name if it has any of the reserved characters. These are the reserved /, >, <, |, :, & characters for Linux / Unix system. For a good read follow this link.
It tells you that the avatar folder does not exist.
Before writing a file into this folder, you need to check that a directory called "avatar" exists and if it doesn't, create it:
if (!fs.existsSync('/avatar')) {
fs.mkdirSync('/avatar', { recursive: true});
}
you can use './' as a prefix for your path.
in your example, you will write:
fs.writeFile('./avatar/myFile.png', data.file, (err) => {
if (err) {
return console.log(err);
}
console.log("The file was saved!");
});
I had this error because I tried to run:
fs.writeFile(file)
fs.unlink(file)
...lots of code... probably not async issue...
fs.writeFile(file)
in the same script. The exception occurred on the second writeFile call. Removing the first two calls solved the problem.
In my case, I use async fs.mkdir() and then, without waiting for this task to complete, I tried to create a file fs.writeFile()...
As SergeS mentioned, using / attempts to write in your system root folder, but instead of using __dirname, which points to the path of the file where writeFile is invoked, you can use process.cwd() to point to the project's directory. Example:
writeFile(`${process.cwd()}/pictures/myFile.png`, data, (err) => {...});
If you want to avoid string concatenations/interpolations, you may also use path.join(process.cwd(), 'pictures', 'myFile.png') (more details, including directory creation, in this digitalocean article).
I've got the following code:
fs.open("uploads/test.txt", "a", "0755", function(err, fd){
if(err) { console.log(err); }
else {
file.handler = fd; //We store the file handler so we can write to it later
...
}
});
The file is created and written to perfectly when I simply have "uploads/test", but when I try to do "uploads/test.txt" it breaks. Any ideas?
I think you should try using
var path = './uploads/test.txt'.
Or
var path = __dirname + 'your_path';
fs.open(path, "a", "0755", function(err, fd){
if(err) { console.log(err); }
else {
file.handler = fd; //We store the file handler so we can write to it later
...
}
});
This is really silly, but I found what was causing my code to break:
fs.open works as intended. The bug was with my file detection setup using nodemon.
The reason is everytime my app would load it would run the above mentioned code. The code would then write to a new file in my apps /uploads directory. Nodemon would then detect the new file and restart the app thus creating a vicious circle.
I'm working on a file uploader script which creates a new folder (based on the timestamp) and moves the uploaded file to the created folder.
Sometimes it works and sometimes I'm getting a ENOENT rename Error (file/folder not exist).
The following code is in my post route:
var form = new multiparty.Form({
uploadDir: "C:"+ path.sep + "files"
});
form.parse(req, function(err, fields, files) {
var dirPath = "C:"+ path.sep + "files" + path.sep + +new Date;
fs.mkdir(dirPath, 0777, function(err, dirPath){
if (err) console.error(err);
console.log("Created new folder");
fs.rename(files.file[i].path, dirPath + path.sep + files.file[i].originalFilename, function(err){
if (err) console.error(err);
console.log("Moved file");
});
}(err, dirPath));
next();
});
I'm using express(4) and the multiparty module.
As you can see I'm using async functions.
So the question is: What is wrong with my code?
Edit
The error I'm talking about: "Error: ENOENT, rename 'C:\files\7384-1r41cry.png'"
It has something to do with a race condition. With fs.mkdirSync everything works fine.
My guess would be some sort of race condition happening here.
This kind of stuff is easy to get wrong and hard to get it right.
I normally use gulp for this kind of stuff and maybe should you :)
To copy a whole directory tree into some other directory wouldn't be easier.
gulp.src('./inputDir/**/*').pipe(gulp.dest('./outputDir')
And all files from inputDir would be copied into outputDir
But maybe coping is not a option. The files could too large, right?
Lets hack it a bit to make it work the way we want.
var fs = require('fs')
, gulp = require('gulp')
, thr = require('through2').obj
, SRC = './test/**/*.{css,html,js}' // it could be 'my/file/located/here'
, OUT = './output' // it could be 'my/newer/file/located/there'
;
gulp.src(SRC)
.pipe(thr(function(chunk, enc, next){
chunk.contents = new Buffer('') // cleaning the contents of the file
chunk._originalPath = chunk.path
next(null, chunk)
}))
.pipe(gulp.dest(OUT))
.pipe(thr(function(chunk, enc, next){
// now a place holder file exists at our destination.
// so we can write to it and being convident it exists
console.log('moving file from', chunk._originalPath, 'to', chunk.path)
fs.rename(chunk._originalPath, chunk.path, function(err){
if (err) return next(err)
next()
})
}))
This moves all css, html and js files from input to output, regardless of how many nested directories there are
gulp is awesome :)
Ok a few things...
you really should start using promises, it makes the code easier to read and the error handling is way superior. I usually use when.js, but there are other alternatives.
you should throw the errors or return on errors or you would try to continue running the function even when the previous operations failed.
you do
if (err) console.error(err);
should be
if (err) throw err;
I'm a beginner in Node.js, and was having trouble with this piece of code.
var fs = require('fs');
Framework.Router = function() {
this.run = function(req, res) {
fs.exists(global.info.controller_file, function(exists) {
if (exists) {
// Here's the problem
res.writeHead(200, {'Content-Type':'text/html'});
var cname = App.ucfirst(global.info.controller)+'Controller';
var c = require(global.info.controller_file);
var c = new App[cname]();
var action = global.info.action;
c[action].apply(global.info.action, global.info.params);
res.end();
} else {
App.notFound();
return false;
}
});
}
};
The problem lies in the part after checking if the 'global.info.controller_file' exists, I can't seem to get the code to work properly inside the: if (exists) { ... NOT WORKING }
I tried logging out the values for all the variables in that section, and they have their expected values, however the line: c[action].apply(global.info.action, global.info.params);
is not running as expected. It is supposed to call a function in the controller_file and is supposed to do a simple res.write('hello world');. I wasn't having this problem before I started checking for the file using fs.exists. Everything inside the if statement, worked perfectly fine before this check.
Why is the code not running as expected? Why does the request just time out?
Does it have something to do with the whole synchronous vs asynchronous thing? (Sorry, I'm a complete beginner)
Thank you
Like others have commented, I would suggest you rewrite your code to bring it more in-line with the Node.js design patterns, then see if your problem still exists. In the meantime, here's something which may help:
The advice about not using require dynamically at "run time" should be heeded, and calling fs.exists() on every request is tremendously wasteful. However, say you want to load all *.js files in a directory (perhaps a "controllers" directory). This is best accomplished using an index.js file.
For example, save the following as app/controllers/index.js
var fs = require('fs');
var files = fs.readdirSync(__dirname);
var dotJs = /\.js$/;
for (var i in files) {
if (files[i] !== 'index.js' && dotJs.test(files[i]))
exports[files[i].replace(dotJs, '')] = require('./' + files[i]);
}
Then, at the start of app/router.js, add:
var controllers = require('./controllers');
Now you can access the app/controllers/test.js module by using controllers.test. So, instead of:
fs.exists(controllerFile, function (exists) {
if (exists) {
...
}
});
simply:
if (controllers[controllerName]) {
...
}
This way you can retain the dynamic functionality you desire without unnecessary disk IO.