NodeJS copyFile then unlink in callback - node.js

Question

I have a large amount of files (millions) that I want to copy from a folder to an other one, but then based on some parsing options, I will need to delete the file if it doesn't respect some criteria.
I'm doing the parsing on my local copy of the file because it would be slower on the network to parse (read file + parse) than doing it locally.
My code looks like this :
for (let file of files) {
fs.copyFile(from, to, err => {
if (err) return;
parse(file);
});
}
The parse function is something like :
parse = file => {
fs.readFile(file, (err, data) => {
//do some parsing
if (notOk) {
fs.unlink(file);
};
}
}
The problem is that it's doing all the copyFile and doesn't seem to execute the callback with the unlink, and I really need to unlink as soon as the file is finished copying since I can't afford the disk space to copy all files first.
Do I need to use the Sync version of those methods or something else?
Thanks
I would have expected an output like this :
copyFile a
copyFile b
copyFile c
parsing a
copyFile d
unlink a
parsing b
copyFile e
...
but instead I have all copyFile and none of the parsing/unlink happening.

Answer 1

If files contains millions of files, you're starting millions of concurrent copy actions, which is probably overwhelming your system.
Instead, consider using something like async.eachLimit to limit the number of concurrent copy actions.
Untested example:
const async = require('async');
async.eachLimit(files, 10, (file, callback) => {
fs.copyFile(from, to, err => {
if (err) return callback(); // or pass `err`
parse(file, callback);
});
});
function parse(file, callback) {
fs.readFile(file, (err, data) => {
//do some parsing
if (notOk) {
fs.unlink(file, callback);
} else {
return callback();
}
})
}

Answer 2

I think this is happening, the callback in the coping is finishing so the thread is finished, but the functions can not finish, you can make this sync and is going to work fine and be more clear, but other way is to transform the functions to promises so you can return the promise and wait for them.
You could write something like this.
const util = require('util');
const fs = require('fs');
const files = ['1.txt', '2.txt', '3.txt', '4.txt'];
//promisify all the fs functions to use.
const copyFilePromise = util.promisify(fs.copyFile);
const readFilePromise = util.promisify(fs.readFile);
const unlinkPromise = util.promisify(fs.unlink);
//make a parser function that return a promise.
const parse = (file) => {
return readFilePromise(file)
.then(() => {
//do some parsing
if(notOk)
return unlinkPromise(file)
});
};
//store all the promises in an array
let promises = []
//iterate for all the files using the promisify aproach of the functions
for (let file of files){
//add all the promises to an array of promises.
promises.push(copyFilePromise(from, to).then(parse(file)).catch(console.error));
}
//wait for all the promises to resolve.
Promise.all(promises);