I am trying to delete array of files but one or two of them gets deleted .Pls check the code I am sure I am doing something wrong.
I am searching for a file in directory and if found I am First Removing it from db..and after response I am removing file from directory
if (files.length > 0) {
files.forEach(function(filename) {
fileDir = path.join(__dirname, '/uploads/' + decodeURI(filename));
fs.stat(fileDir, function(err, stats) {
if (stats.isFile()) {
// fs.unlink(fileDir, function(err) {
form_op.deleteImg(url.resolve('http://localhost/uploads/', filename), query._id, function(err, result) {
if (err) {
throw console.log(err)
}
fs.unlink(fileDir);
});
// });
}
});
done++;
});
through debugging I found that filename and fileDir variable are getting next file in the array without waiting for fs.stat(fileDir, function(err, stats) { to complete.. is there any other way I can do it? the check and remove??
my files variable is array only file's name.
fileDir = path.join(__dirname, '/uploads/' + decodeURI(filename));
You are missing a var declaration here, therefore the fileDir variable is global and shared by all of your callbacks.
Use var fileDir = ... to have the variable scoped to your function.
Related
So I have the following code
var processed;
fs.readFile(path, 'utf-8', function(err, data) {
processed = false;
//checking if text is in file and setting flag
processed = true;
});
if (processed == true) {
try {
var fname = path.substring(path.lastIndexOf("\\") + 1);
fs.moveSync(path, './processedxml/' + fname, {
overwrite: true
})
} catch (err) {
console.log("Error while moving file to processed folder " + err);
}
}
But I don't get the desired output. Because looks like the readfile is executed by a separate thread and so the value of "processed" is not reliable.
I am not very familiar with nodejs so any help will be greatly appreciated.
Yes, you are right, your executions are performed by different threads.
In this scenario, you'll need to use promises.
You can solve your need easily by using "Promise FS" (you can use any other promise solution anyway).
Your code would be something like the following:
fs = require('promise-fs');
var fname = 'test.txt' ;
var toMove = false ;
fs.readFile('test.txt','utf8')
.then (function (content) {
if(content.indexOf('is VALID') !== -1) {
console.log('pattern found!');
toMove = true ;
}
else { toMove = false
}
return toMove ;
}).
then (function (toMove) {
if(toMove) {
var oldPath = 'test.txt'
var newPath = '/tmp/moved/file.txt'
fs.rename(oldPath, newPath, function (err) {
if (err) throw err
console.log('Successfully renamed - moved!')
}) ;
}
})
.catch (function (err) {
console.log(err);
})
Create a file "test.txt" and add the following contents:
this is text.file contents
token is VALID
The code above will evaluate if "is VALID" is present as content and if it does then it will move the file "test.txt" from your current folder to a new one called "moved" in "/tmp" directory. It will also rename the file as "file.txt" file name.
Hope it helps you.
Regards
It looks like you're shadowing path, trying to use it as a variable and as a node module. The easiest way to make this work is to choose a different variable name for the file and move the processing logic into the callback of fs.readFile.
var path = require('path');
var fs = require('fs-extra');
var file = 'some/file/path/foo.xml';
var text = 'search text';
fs.readFile(file, 'utf-8', function (err, data) {
if (err) {
console.error(err);
} else {
//checking if text is in file and setting flag
if (data.indexOf(text) > -1) {
try {
var fname = path.basename(file);
fs.moveSync(file, './processedxml/' + fname, {
overwrite: true
})
} catch (err) {
console.log("Error while moving file to processed folder " + err);
}
}
}
});
I have created a function to populate a list of objects but they are not being written.
if (stats.isDirectory()) {
objlist=[];
//do something here.. the path was correct
fs.readdir(path, (err, files) => {
objlist=[];
for(file in files){
//console.log(files[file])
objlist[file]={};
fs.stat(path + '\\' + files[file], function(err,stats){
if(err)
throw err;
if(stats.isDirectory()){
objlist[file]['file'] = 'folder'
}
if(stats.isFile()){
objlist[file]['file'] = 'file'
}
objlist[file]['name'] = files[file]
objlist[file]['size'] = stats.size
//console.log(objlist)
//console.log(stats)
});
}
});
console.log(objlist);
return objlist;
}
However the function returns an empty objlist; Can you please suggest what I am doing wrong
this code will be helphul. use let file in files
objlist=[];
//do something here.. the path was correct
fs.readdir(path, (err, files) => {
objlist=[];
for(let file in files){
objlist[file]={};
fs.stat(path + '\\' + files[file], function(err,stats){
if(err)
throw err;
if(stats.isDirectory()){
objlist[file]['file'] = 'folder'
}
if(stats.isFile()){
objlist[file]['file'] = 'file'
}
objlist[file]['name'] = files[file]
objlist[file]['size'] = stats.size
if(file == files.length-1) {
console.log(objlist);
}
});
}
});
You must add the objects to the array with the method push, like this:
objlist=[];
for(file in files){
obj = {};
obj.file = file;
//... do all stuff with obj
objlist.push(obj);
}
Also your function is asynchronous inside the fs.readdir, so if you have to build and populate in the function or do an await to receive the result.
Otherwise you will call the function, it will be executed asynchronously, you will continue at the main function without the wait of the response of the fs.readdir and the array will continue empty because you async function may or may not have been executed.
There are two mistakes in your code:-
1) You are using an array as an object
2) Since for loop is synchronous loop and you are trying asynchronous task inside it and hence either use synchronous version of fs or change your for loop as follows:-
for(file in files) {
obj = {};
obj[file]={};
(function(f){
fs.stat(path + '\\' + files[f], function(err,stats){
if(err)
throw err;
if(stats.isDirectory()){
obj[file]['file'] = 'folder'
}
if(stats.isFile()){
obj[file]['file'] = 'file'
}
obj[file]['name'] = files[file]
obj[file]['size'] = stats.size
objlist.push(obj)
//console.log(objlist)
//console.log(stats)
});
}
}(file))
}
Code below:
I'm using the findit walker, documentation here -> https://github.com/substack/node-findit
With this package i'm listing all the directories and files of my application, and i'm trying to send to my bucket on Amazon S3 (with my own code).
I'm not sure if the code is right, and i don't know what i need to put in the Body, inside the params object.
This part it's listening all the Directories of my app:
finder.on('directory', function (dir, stat, stop) {
var base = path.basename(dir);
if (base === '.git' || base === 'node_modules' || base === 'bower_components') {
stop();
}
else {
console.log(dir + '/');
}
});
And this one it's listening all the files of my app:
finder.on('file', function (file, stat) {
console.log(file);
});
I updated it to send data to my bucket, like this:
finder.on('file', function (file, stat) {
console.log(file);
var params = {
Bucket: BUCKET_NAME,
Key: file,
//Body:
};
//console.log(params.body);
s3.putObject(params, function(err) {
if(err) {
console.log(err);
}
else {
console.log("Success!");
}
});
});
I really don't know what i need to put inside the Body, and i don't know if the code is right. Anyone could help me?
Thanks.
to help, all code, all the code:
var fs = require('fs');
var finder = require('findit')(process.argv[2] || '.');
var path = require('path');
var aws = require('aws-sdk');
var s3 = new aws.S3();
aws.config.loadFromPath('./AwsConfig.json');
var BUCKET_NAME = 'test-dev-2';
finder.on('directory', function (dir, stat, stop) {
var base = path.basename(dir);
if (base === '.git' || base === 'node_modules' || base === 'bower_components') {
stop();
}
else {
console.log(dir + '/');
}
});
finder.on('file', function (file, stat) {
console.log(file);
var params = {
Bucket: BUCKET_NAME,
Key: file,
//Body:
};
//console.log(params.body);
s3.putObject(params, function(err) {
if(err) {
console.log(err);
}
else {
console.log("Success");
}
});
});
finder.on('error', function (err) {
console.log(err);
});
finder.on('end', function () {
console.log('Done!');
});
Based on the documentation, the Body parameter of s3.putObject can take a Buffer, Typed Array, Blob, String, or ReadableStream. The best one of those to use in most cases would be a ReadableString. You can create a ReadableString from any file using the createReadStream() function in the fs module.
So, that part your code would look something like:
finder.on('file', function (file, stat) {
console.log(file);
var params = {
Bucket: BUCKET_NAME,
Key: file,
Body: fs.createReadStream(file) // NOTE: You might need to adjust "file" so that it's either an absolute path, or relative to your code's directory.
};
s3.putObject(params, function(err) {
if(err) {
console.log(err);
}
else {
console.log("Success!");
}
});
});
I also want to point out that you might run in to a problem with this code if you pass it a directory with a lot of files. putObject is an asynchronous function, which means it'll be called and then the code will move on to something else while it's doing its thing (ok, that's a gross simplification, but you can think of it that way). What that means in terms of this code is that you'll essentially be uploading all the files it finds at the same time; that's not good.
What I'd suggest is to use something like the async module to queue your file uploads so that only a few of them happen at a time.
Essentially you'd move the code you have in your file event handler to the queue's worker method, like so:
var async = require('async');
var uploadQueue = async.queue(function(file, callback) {
var params = {
Bucket: BUCKET_NAME,
Key: file,
Body: fs.createReadStream(file) // NOTE: You might need to adjust "file" so that it's either an absolute path, or relative to your code's directory.
};
s3.putObject(params, function(err) {
if(err) {
console.log(err);
}
else {
console.log("Success!");
}
callback(err); // <-- Don't forget the callback call here so that the queue knows this item is done
});
}, 2); // <-- This "2" is the maximum number of files to upload at once
Note the 2 at the end there, that specifies your concurrency which, in this case, is how many files to upload at once.
Then, your file event handler simply becomes:
finder.on('file', function (file, stat) {
uploadQueue.push(file);
});
That will queue up all the files it finds and upload them 2 at a time until it goes through all of them.
An easier and arguably more efficient solution may be to just tar up the directory and upload that single tar file (also gzipped if you want). There are tar modules on npm, but you could also just spawn a child process for it too.
I want to remove some files in a directory after reaching some limits.(for example remove files if number of files more than 20)
It would be great if any automation can be done to remove those files.
In details:
In my case there is a uploads directory, where I'm uploading the images. For each new image, a directory is created and the image resides in the directory. So I want to keep some of the newly created or recently used directories and remove others after a certain limit(for example after reaching 20 numbers of directories). While creating new images, it'll check the limit and if exceeds the max dir limits, it'll remove the unused directories.
Note: The directories are not empty.
How can i do that using Node.js
Any help would be appreciable.
The most widely used technique would be to have an API that can delete files in your folder. Take a look at
fs.unlink
You can get more details here
Once you have this API, it is preferable to have a cron call this API every month or so. Take a look at
crontab -e
If you're running Node on a Linux server, you can use the exec module to execute Linux commands. For example, here is a function I use to move old log files:
var exec = require('child_process').exec;
exec('mv ' + __dirname + '/log/*.log ' + __dirname + '/log/archive',
function(err, stdout, stderr) {
if (err) {
console.log('Error archiving log files: ' + stderr);
} else {
console.log('Log files archived to ' + __dirname + '/log/archive');
}
});
You can use any Linux command - so you could use this approach to remove files as well.
I create a "cronjob" function in node.js to remove files in a folder (note child folders will be ignore)
USAGE:
// keep only 5 newest files in `logs` folder
watchAndRemoveOldFiles('logs', 5, function (err, removeFiles) {
console.log('These files has been remove:', removeFiles);
});
Full code (you need npm install async to run the code):
var fs = require('fs');
var path = require('path');
var async = require('async');
function findAndRemoveOldFiles(inputDir, keepCount, callback) {
if(!callback) {
callback = function (err, removeFiles) {
// default callback: doing nothing
};
};
fs.readdir(inputDir, function (err, files) {
if(err) {
return callback(err);
}
fileNames = files.map(function (fileName) {
return path.join(inputDir, fileName);
});
async.map(fileNames, function (fileName, cb) {
fs.stat(fileName, function (err, stat) {
if(err) {
return cb(err);
};
cb(null, {
name: fileName,
isFile: stat.isFile(),
time: stat.mtime,
});
});
}, function (err, files) {
if(err) {
return callback(err);
};
files = files.filter(function (file) {
return file.isFile;
})
files.sort(function (filea, fileb) {
return filea.time < fileb.time;
});
files = files.slice(keepCount);
async.map(files, function (file, cb) {
fs.unlink(file.name, function (err) {
if(err) {
return cb(err);
};
cb(null, file.name);
});
}, function (err, removedFiles) {
if(err) {
return callback(err);
}
callback(null, removedFiles);
});
});
});
}
function watchAndRemoveOldFiles(inputDir, keepCount, callback) {
findAndRemoveOldFiles(inputDir, keepCount, callback);
fs.watch(inputDir, function () {
findAndRemoveOldFiles(inputDir, keepCount, callback);
});
}
// USAGE: watch and remove old files, keep only 5 newest files
watchAndRemoveOldFiles('log', 5, function (err, removeFiles) {
console.log('These files has been remove:', removeFiles);
});
you might consider setting up a kue task:
https://github.com/learnboost/kue
Kue (or a slight wrapper/mod on top of it) is likely to be what makes it into core for our scheduled jobs down the road.
I need to write file to the following path:
fs.writeFile('/folder1/folder2/file.txt', 'content', function () {…});
But '/folder1/folder2' path may not exists. So I get the following error:
message=ENOENT, open /folder1/folder2/file.txt
How can I write content to that path?
As of Node v10, this is built into the fs.mkdir function, which we can use in combination with path.dirname:
var fs = require('fs');
var getDirName = require('path').dirname;
function writeFile(path, contents, cb) {
fs.mkdir(getDirName(path), { recursive: true}, function (err) {
if (err) return cb(err);
fs.writeFile(path, contents, cb);
});
}
For older versions, you can use mkdirp:
var mkdirp = require('mkdirp');
var fs = require('fs');
var getDirName = require('path').dirname;
function writeFile(path, contents, cb) {
mkdirp(getDirName(path), function (err) {
if (err) return cb(err);
fs.writeFile(path, contents, cb);
});
}
If the whole path already exists, mkdirp is a noop. Otherwise it creates all missing directories for you.
This module does what you want: https://npmjs.org/package/writefile . Got it when googling for "writefile mkdirp". This module returns a promise instead of taking a callback, so be sure to read some introduction to promises first. It might actually complicate things for you.
The function I gave works in any case.
I find that the easiest way to do this is to use the outputFile() method from the fs-extra module.
Almost the same as writeFile (i.e. it overwrites), except that if the parent directory does not exist, it's created. options are what you'd pass to fs.writeFile().
Example:
var fs = require('fs-extra');
var file = '/tmp/this/path/does/not/exist/file.txt'
fs.outputFile(file, 'hello!', function (err) {
console.log(err); // => null
fs.readFile(file, 'utf8', function (err, data) {
console.log(data); // => hello!
});
});
It also has promise support out of the box these days!.
Edit
NodeJS version 10.12.0 has added a native support for both mkdir and mkdirSync to create the parent director recursively with recursive: true option as the following:
fs.mkdirSync(targetDir, { recursive: true });
And if you prefer fs Promises API, you can write
fs.promises.mkdir(targetDir, { recursive: true });
Original Answer
Create the parent directories recursively if they do not exist! (Zero dependencies)
const fs = require('fs');
const path = require('path');
function mkDirByPathSync(targetDir, { isRelativeToScript = false } = {}) {
const sep = path.sep;
const initDir = path.isAbsolute(targetDir) ? sep : '';
const baseDir = isRelativeToScript ? __dirname : '.';
return targetDir.split(sep).reduce((parentDir, childDir) => {
const curDir = path.resolve(baseDir, parentDir, childDir);
try {
fs.mkdirSync(curDir);
} catch (err) {
if (err.code === 'EEXIST') { // curDir already exists!
return curDir;
}
// To avoid `EISDIR` error on Mac and `EACCES`-->`ENOENT` and `EPERM` on Windows.
if (err.code === 'ENOENT') { // Throw the original parentDir error on curDir `ENOENT` failure.
throw new Error(`EACCES: permission denied, mkdir '${parentDir}'`);
}
const caughtErr = ['EACCES', 'EPERM', 'EISDIR'].indexOf(err.code) > -1;
if (!caughtErr || caughtErr && curDir === path.resolve(targetDir)) {
throw err; // Throw if it's just the last created dir.
}
}
return curDir;
}, initDir);
}
Usage
// Default, make directories relative to current working directory.
mkDirByPathSync('path/to/dir');
// Make directories relative to the current script.
mkDirByPathSync('path/to/dir', {isRelativeToScript: true});
// Make directories with an absolute path.
mkDirByPathSync('/path/to/dir');
Demo
Try It!
Explanations
[UPDATE] This solution handles platform-specific errors like EISDIR for Mac and EPERM and EACCES for Windows.
This solution handles both relative and absolute paths.
In the case of relative paths, target directories will be created (resolved) in the current working directory. To Resolve them relative to the current script dir, pass {isRelativeToScript: true}.
Using path.sep and path.resolve(), not just / concatenation, to avoid cross-platform issues.
Using fs.mkdirSync and handling the error with try/catch if thrown to handle race conditions: another process may add the file between the calls to fs.existsSync() and fs.mkdirSync() and causes an exception.
The other way to achieve that could be checking if a file exists then creating it, I.e, if (!fs.existsSync(curDir) fs.mkdirSync(curDir);. But this is an anti-pattern that leaves the code vulnerable to race conditions.
Requires Node v6 and newer to support destructuring. (If you have problems implementing this solution with old Node versions, just leave me a comment)
Perhaps most simply, you can just use the fs-path npm module.
Your code would then look like:
var fsPath = require('fs-path');
fsPath.writeFile('/folder1/folder2/file.txt', 'content', function(err){
if(err) {
throw err;
} else {
console.log('wrote a file like DaVinci drew machines');
}
});
With node-fs-extra you can do it easily.
Install it
npm install --save fs-extra
Then use the outputFile method instead of writeFileSync
const fs = require('fs-extra');
fs.outputFile('tmp/test.txt', 'Hey there!', err => {
if(err) {
console.log(err);
} else {
console.log('The file was saved!');
}
})
You can use
fs.stat('/folder1/folder2', function(err, stats){ ... });
stats is a fs.Stats type of object, you may check stats.isDirectory(). Depending on the examination of err and stats you can do something, fs.mkdir( ... ) or throw an error.
Reference
Update: Fixed the commas in the code.
Here's my custom function to recursively create directories (with no external dependencies):
var fs = require('fs');
var path = require('path');
var myMkdirSync = function(dir){
if (fs.existsSync(dir)){
return
}
try{
fs.mkdirSync(dir)
}catch(err){
if(err.code == 'ENOENT'){
myMkdirSync(path.dirname(dir)) //create parent dir
myMkdirSync(dir) //create dir
}
}
}
myMkdirSync(path.dirname(filePath));
var file = fs.createWriteStream(filePath);
Here is my function which works in Node 10.12.0. Hope this will help.
const fs = require('fs');
function(dir,filename,content){
fs.promises.mkdir(dir, { recursive: true }).catch(error => { console.error('caught exception : ', error.message); });
fs.writeFile(dir+filename, content, function (err) {
if (err) throw err;
console.info('file saved!');
});
}
Here's part of Myrne Stol's answer broken out as a separate answer:
This module does what you want: https://npmjs.org/package/writefile .
Got it when googling for "writefile mkdirp". This module returns a
promise instead of taking a callback, so be sure to read some
introduction to promises first. It might actually complicate things
for you.
let name = "./new_folder/" + file_name + ".png";
await driver.takeScreenshot().then(
function(image, err) {
require('mkdirp')(require('path').dirname(name), (err) => {
require('fs').writeFile(name, image, 'base64', function(err) {
console.log(err);
});
});
}
);
In Windows you can use this code:
try {
fs.writeFileSync( './/..//..//filename.txt' , 'the text to write in the file', 'utf-8' );
}
catch(e){
console.log(" catch XXXXXXXXX ");
}
This code in windows create file in 2 folder above the current folder.
but I Can't create file in C:\ Directly