I am trying to monitor a file that is (soft) symlink'ed with node.js' watchFile() with the following code:
var fs=require('fs')
, file= './somesymlink'
, config= {persist:true, interval:1};
fs.watchFile(file, config, function(curr, prev) {
if((curr.mtime+'')!=(prev.mtime+'')) {
console.log( file+' changed');
}
});
In the above code, ./somesymlink is a (soft) symlink to /path/to/the/actual/file.
When changes are made to the /path/to/the/actual/file, no event is fired. I have to replace the symlink with /path/to/the/actual/file to make it work. It seems to me that watchFile is not able to watch symlink'ed files. Of course I could make this work by using spawn+tail method but I prefer not to use that path as it would introduce more overhead.
So my question is how can I watch symlink'ed files in node.js using watchFile(). Thanks folks in advance.
You could use fs.readlink:
fs.readlink(file, function(err, realFile) {
if(!err) {
fs.watch(realFile, ... );
}
});
Of course, you could get fancier and write a little wrapper that can watch either the file or it's link, so you don't have to think about it.
UPDATE: Here's such a wrapper, for the future:
/** Helper for watchFile, also handling symlinks */
function watchFile(path, callback) {
// Check if it's a link
fs.lstat(path, function(err, stats) {
if(err) {
// Handle errors
return callback(err);
} else if(stats.isSymbolicLink()) {
// Read symlink
fs.readlink(path, function(err, realPath) {
// Handle errors
if(err) return callback(err);
// Watch the real file
fs.watch(realPath, callback);
});
} else {
// It's not a symlink, just watch it
fs.watch(path, callback);
}
});
}
Related
I am trying to make a file upload using Node.js and the Formidable module.
npm install formidable
And then I made this, please read the notes - where I can explain what each function does and describes the algorithm:
// get access to the files that were sent;
// at this time I don't want the files to be uploaded yet;
// in the next function I will validate those files.
function form_parse() {
form.parse(req, (err, fields, files) => {
if (err) return req.Cast.error(err);
if (Object.keys(files).length==0) return req.Cast.badRequest();
req.files = files;
return validate_files();
});
}
// I made an object with options to validate against the
// files. it works and continues to the process_files()
// function only whether files are verified.
function validate_files() {
let limitations = require('../uploads-limitations');
try {
limitation = limitations[req.params.resource];
} catch(err) {
return req.Cast.error(err);
}
let validateFiles = require('../services/validate-files');
validateFiles(req, limitation, err => {
if (err) return req.Cast.badRequest(err);
return process_files();
});
}
// here is the problem - form.on doesn't get fired.
// This is the time I want to save those files - after
// fully verified
function process_files() {
form.on('file', function(name, file) {
console.log(`file name: ${file.name}`);
file.path = path.join(__dirname, '../tmp_uploads/' + file.name);
});
form.on('error', err => {
return req.Cast.error(err);
});
form.on('end', () => {
console.log(`successfully saved`);
return req.Cast.ok();
});
}
form_parse();
As you can see and as I have described - the validation works but when I want to actually save those files the form.on (events) are not getting fired.
Yes because at the end of your process, after parsing and validating, you attach events listeners. This should be done first, before starting parsing. Because these events (on file, on error, on end) happen during the parsing, not after.
form.on('file',...) // First, attach your listeners
.on('error', ...)
.on('end', ...);
form.parse(req) // then start the parsing
I am using following code to create directories in sync way. It checks the existence of the directory, deletes it if exists and creates it. All operations are in sync way. I am looping this operation for 5 times. I am getting different results each time. Sometimes it creates only 4 directories, sometimes it creates all 5. What is the reason for this unstability in the code?
fs.readdir(dir, function(err, filenames) {
if (err) {
onError(err);
return;
}
filenames.forEach(function(filename) {
fs.readFile(dir + filename, 'utf-8', function(err, content) {
if (err) {
onError(err);
return;
}
AsyncFunc(content, ....)
.then(newContent => {
filenames.forEach(function(filename) {
if (fs.existsSync(currentDirName)) {
fs.rmdirSync(currentDirName);
}
fs.mkdirSync(currentDirName, '0766');
});
});
});
If you are using sync functions you can not use callbacks. Also if you want to remove a folder you need to use rmdirSync(filename);
var fs = require('fs');
var filenames = ['1','2','3','4'];
filenames.forEach(function(filename) {
if (fs.existsSync(filename)) {
fs.rmdirSync(filename);
}
fs.mkdirSync(filename, '0766');
});
Goal is to create distinct functions which separate out the work of loading multiple (xml) files and parsing them. I could do this all in one function, but the nested callbacks begin to get ugly. In other words, I don't want to do this:
// Explore directory
fs.readdir(path, function (err, files) {
if(err) throw err;
// touch each file
files.forEach(function(file) {
fs.readFile(path+file, function(err, data) {
if (err) throw err;
someAsyncFunction ( function (someAsyncFunctionResult) {
// Do some work, then call another async function...
nestedAsynchFunction ( function (nestedAsyncFunctionResult) {
// Do Final Work here, X levels deep. Ouch!
});
});
});
});
});
Instead, I want one function which reads my files and puts each file's XML payload into an array of objects which is returned to the caller (each object represents the name of the file and the XML in the file). Here's the function that might load up reports into an array:
function loadReports (callback) {
var path = "./downloaded-reports/";
var reports = [];
// There are TWO files in this path....
fs.readdir(path, function (err, files) {
if(err) throw err;
files.forEach(function(file) {
fs.readFile(path+file, function(err, data) {
if (err) throw err;
reports.push({ report: file, XML: data.toString()});
//gets called twice, which makes for strangeness in the calling function
callback(null, reports);
});
});
// callback won't work here, returns NULL reports b/c they haven't been processed yet
//callback(null, reports);
});
}
...and here's the function which will call the one above:
function parseReports() {
loadReports( function(err, data) {
console.log ("loadReports callback");
var reportXML = new jsxml.XML(data[0].XML);
var datasources = reportXML.child('datasources').child('datasource').child('connection').attribute("dbname").toString();
console.log(JSON.stringify(datasources,null, 2));
// More async about to be done below
} );
}
As you can see in the loadReports() comments, I can't get the callback to work right. It either calls back BEFORE the array is has been populated at all, or it calls back twice - once for each fs.readFile operation.
SO...what is the best way to deal with this sort of situation? In brief - What's the best design pattern for a function which processes multiple things asynchronously, so that it ONLY calls back when all "things" have been completely processed? The simpler the better. Do I need to use some sort of queuing module like Q or node-queue?
Thanks much!
Edit: Something like this works inside the deepest loop in terms of not hitting the callback twice, but it seems like a kludge:
fs.readdir(path, function (err, files) {
if(err) throw err;
files.forEach(function(file) {
fs.readFile(path+file, function(err, data) {
if (err) throw err;
reports.push({ report: file, XML: data.toString()});
// WORKS, but seems hacky.
if (reports.length = files.length) callback(null, reports);
});
});
});
I want to remove some files in a directory after reaching some limits.(for example remove files if number of files more than 20)
It would be great if any automation can be done to remove those files.
In details:
In my case there is a uploads directory, where I'm uploading the images. For each new image, a directory is created and the image resides in the directory. So I want to keep some of the newly created or recently used directories and remove others after a certain limit(for example after reaching 20 numbers of directories). While creating new images, it'll check the limit and if exceeds the max dir limits, it'll remove the unused directories.
Note: The directories are not empty.
How can i do that using Node.js
Any help would be appreciable.
The most widely used technique would be to have an API that can delete files in your folder. Take a look at
fs.unlink
You can get more details here
Once you have this API, it is preferable to have a cron call this API every month or so. Take a look at
crontab -e
If you're running Node on a Linux server, you can use the exec module to execute Linux commands. For example, here is a function I use to move old log files:
var exec = require('child_process').exec;
exec('mv ' + __dirname + '/log/*.log ' + __dirname + '/log/archive',
function(err, stdout, stderr) {
if (err) {
console.log('Error archiving log files: ' + stderr);
} else {
console.log('Log files archived to ' + __dirname + '/log/archive');
}
});
You can use any Linux command - so you could use this approach to remove files as well.
I create a "cronjob" function in node.js to remove files in a folder (note child folders will be ignore)
USAGE:
// keep only 5 newest files in `logs` folder
watchAndRemoveOldFiles('logs', 5, function (err, removeFiles) {
console.log('These files has been remove:', removeFiles);
});
Full code (you need npm install async to run the code):
var fs = require('fs');
var path = require('path');
var async = require('async');
function findAndRemoveOldFiles(inputDir, keepCount, callback) {
if(!callback) {
callback = function (err, removeFiles) {
// default callback: doing nothing
};
};
fs.readdir(inputDir, function (err, files) {
if(err) {
return callback(err);
}
fileNames = files.map(function (fileName) {
return path.join(inputDir, fileName);
});
async.map(fileNames, function (fileName, cb) {
fs.stat(fileName, function (err, stat) {
if(err) {
return cb(err);
};
cb(null, {
name: fileName,
isFile: stat.isFile(),
time: stat.mtime,
});
});
}, function (err, files) {
if(err) {
return callback(err);
};
files = files.filter(function (file) {
return file.isFile;
})
files.sort(function (filea, fileb) {
return filea.time < fileb.time;
});
files = files.slice(keepCount);
async.map(files, function (file, cb) {
fs.unlink(file.name, function (err) {
if(err) {
return cb(err);
};
cb(null, file.name);
});
}, function (err, removedFiles) {
if(err) {
return callback(err);
}
callback(null, removedFiles);
});
});
});
}
function watchAndRemoveOldFiles(inputDir, keepCount, callback) {
findAndRemoveOldFiles(inputDir, keepCount, callback);
fs.watch(inputDir, function () {
findAndRemoveOldFiles(inputDir, keepCount, callback);
});
}
// USAGE: watch and remove old files, keep only 5 newest files
watchAndRemoveOldFiles('log', 5, function (err, removeFiles) {
console.log('These files has been remove:', removeFiles);
});
you might consider setting up a kue task:
https://github.com/learnboost/kue
Kue (or a slight wrapper/mod on top of it) is likely to be what makes it into core for our scheduled jobs down the road.
I need to write file to the following path:
fs.writeFile('/folder1/folder2/file.txt', 'content', function () {…});
But '/folder1/folder2' path may not exists. So I get the following error:
message=ENOENT, open /folder1/folder2/file.txt
How can I write content to that path?
As of Node v10, this is built into the fs.mkdir function, which we can use in combination with path.dirname:
var fs = require('fs');
var getDirName = require('path').dirname;
function writeFile(path, contents, cb) {
fs.mkdir(getDirName(path), { recursive: true}, function (err) {
if (err) return cb(err);
fs.writeFile(path, contents, cb);
});
}
For older versions, you can use mkdirp:
var mkdirp = require('mkdirp');
var fs = require('fs');
var getDirName = require('path').dirname;
function writeFile(path, contents, cb) {
mkdirp(getDirName(path), function (err) {
if (err) return cb(err);
fs.writeFile(path, contents, cb);
});
}
If the whole path already exists, mkdirp is a noop. Otherwise it creates all missing directories for you.
This module does what you want: https://npmjs.org/package/writefile . Got it when googling for "writefile mkdirp". This module returns a promise instead of taking a callback, so be sure to read some introduction to promises first. It might actually complicate things for you.
The function I gave works in any case.
I find that the easiest way to do this is to use the outputFile() method from the fs-extra module.
Almost the same as writeFile (i.e. it overwrites), except that if the parent directory does not exist, it's created. options are what you'd pass to fs.writeFile().
Example:
var fs = require('fs-extra');
var file = '/tmp/this/path/does/not/exist/file.txt'
fs.outputFile(file, 'hello!', function (err) {
console.log(err); // => null
fs.readFile(file, 'utf8', function (err, data) {
console.log(data); // => hello!
});
});
It also has promise support out of the box these days!.
Edit
NodeJS version 10.12.0 has added a native support for both mkdir and mkdirSync to create the parent director recursively with recursive: true option as the following:
fs.mkdirSync(targetDir, { recursive: true });
And if you prefer fs Promises API, you can write
fs.promises.mkdir(targetDir, { recursive: true });
Original Answer
Create the parent directories recursively if they do not exist! (Zero dependencies)
const fs = require('fs');
const path = require('path');
function mkDirByPathSync(targetDir, { isRelativeToScript = false } = {}) {
const sep = path.sep;
const initDir = path.isAbsolute(targetDir) ? sep : '';
const baseDir = isRelativeToScript ? __dirname : '.';
return targetDir.split(sep).reduce((parentDir, childDir) => {
const curDir = path.resolve(baseDir, parentDir, childDir);
try {
fs.mkdirSync(curDir);
} catch (err) {
if (err.code === 'EEXIST') { // curDir already exists!
return curDir;
}
// To avoid `EISDIR` error on Mac and `EACCES`-->`ENOENT` and `EPERM` on Windows.
if (err.code === 'ENOENT') { // Throw the original parentDir error on curDir `ENOENT` failure.
throw new Error(`EACCES: permission denied, mkdir '${parentDir}'`);
}
const caughtErr = ['EACCES', 'EPERM', 'EISDIR'].indexOf(err.code) > -1;
if (!caughtErr || caughtErr && curDir === path.resolve(targetDir)) {
throw err; // Throw if it's just the last created dir.
}
}
return curDir;
}, initDir);
}
Usage
// Default, make directories relative to current working directory.
mkDirByPathSync('path/to/dir');
// Make directories relative to the current script.
mkDirByPathSync('path/to/dir', {isRelativeToScript: true});
// Make directories with an absolute path.
mkDirByPathSync('/path/to/dir');
Demo
Try It!
Explanations
[UPDATE] This solution handles platform-specific errors like EISDIR for Mac and EPERM and EACCES for Windows.
This solution handles both relative and absolute paths.
In the case of relative paths, target directories will be created (resolved) in the current working directory. To Resolve them relative to the current script dir, pass {isRelativeToScript: true}.
Using path.sep and path.resolve(), not just / concatenation, to avoid cross-platform issues.
Using fs.mkdirSync and handling the error with try/catch if thrown to handle race conditions: another process may add the file between the calls to fs.existsSync() and fs.mkdirSync() and causes an exception.
The other way to achieve that could be checking if a file exists then creating it, I.e, if (!fs.existsSync(curDir) fs.mkdirSync(curDir);. But this is an anti-pattern that leaves the code vulnerable to race conditions.
Requires Node v6 and newer to support destructuring. (If you have problems implementing this solution with old Node versions, just leave me a comment)
Perhaps most simply, you can just use the fs-path npm module.
Your code would then look like:
var fsPath = require('fs-path');
fsPath.writeFile('/folder1/folder2/file.txt', 'content', function(err){
if(err) {
throw err;
} else {
console.log('wrote a file like DaVinci drew machines');
}
});
With node-fs-extra you can do it easily.
Install it
npm install --save fs-extra
Then use the outputFile method instead of writeFileSync
const fs = require('fs-extra');
fs.outputFile('tmp/test.txt', 'Hey there!', err => {
if(err) {
console.log(err);
} else {
console.log('The file was saved!');
}
})
You can use
fs.stat('/folder1/folder2', function(err, stats){ ... });
stats is a fs.Stats type of object, you may check stats.isDirectory(). Depending on the examination of err and stats you can do something, fs.mkdir( ... ) or throw an error.
Reference
Update: Fixed the commas in the code.
Here's my custom function to recursively create directories (with no external dependencies):
var fs = require('fs');
var path = require('path');
var myMkdirSync = function(dir){
if (fs.existsSync(dir)){
return
}
try{
fs.mkdirSync(dir)
}catch(err){
if(err.code == 'ENOENT'){
myMkdirSync(path.dirname(dir)) //create parent dir
myMkdirSync(dir) //create dir
}
}
}
myMkdirSync(path.dirname(filePath));
var file = fs.createWriteStream(filePath);
Here is my function which works in Node 10.12.0. Hope this will help.
const fs = require('fs');
function(dir,filename,content){
fs.promises.mkdir(dir, { recursive: true }).catch(error => { console.error('caught exception : ', error.message); });
fs.writeFile(dir+filename, content, function (err) {
if (err) throw err;
console.info('file saved!');
});
}
Here's part of Myrne Stol's answer broken out as a separate answer:
This module does what you want: https://npmjs.org/package/writefile .
Got it when googling for "writefile mkdirp". This module returns a
promise instead of taking a callback, so be sure to read some
introduction to promises first. It might actually complicate things
for you.
let name = "./new_folder/" + file_name + ".png";
await driver.takeScreenshot().then(
function(image, err) {
require('mkdirp')(require('path').dirname(name), (err) => {
require('fs').writeFile(name, image, 'base64', function(err) {
console.log(err);
});
});
}
);
In Windows you can use this code:
try {
fs.writeFileSync( './/..//..//filename.txt' , 'the text to write in the file', 'utf-8' );
}
catch(e){
console.log(" catch XXXXXXXXX ");
}
This code in windows create file in 2 folder above the current folder.
but I Can't create file in C:\ Directly