Recursively upload files to S3, how to detect if finished? - node.js

Question

with a little help I've built an S3 uploader using Node.JS
It all works great and the files get there, they're set correctly and have the right permissions, but i'm stumped on how to detect whether the process has finished.
const async = require('async');
const AWS = require('aws-sdk');
const mime = require('mime');
const fs = require('fs');
const path = require("path");
require('dotenv').config();
const uploadDirToS3 = function(uploadPath) {
// instantiate aws object for s3
var s3 = new AWS.S3();
// async version
function walk(currentDirPath, callback) {
fs.readdir(currentDirPath, function (err, files) {
if (err) {
throw new Error(err);
}
files.forEach(function (name) {
var filePath = path.join(currentDirPath, name);
var stat = fs.statSync(filePath);
if (stat.isFile()) {
callback(filePath, stat);
} else if (stat.isDirectory()) {
walk(filePath, callback);
}
});
});
}
walk(uploadPath, function(filePath) {
fs.readFile(filePath, function (err, data) {
if (err) { throw err; }
// get content-type (html,jpeg,gif,etc...)
var metaData = mime.getType(filePath)
// set bucket, key (filename), body (file),
// public read-only and content-type
var params = {
Bucket: process.env.AWS_BUCKET,
Key: filePath,
Body: data,
ACL: 'public-read',
ContentType: metaData
};
// upload file to s3
s3.putObject(params, function(err, data) {
if (err) {
console.log(err)
} else {
console.log("Successfully uploaded "+filePath);
}
});
});
})
}
uploadDirToS3("./media/media-1517245218111")
Could it literally be a case of checking wether a callback exists and 'break;' ...ing out of the loop?
Any ideas?

Answer 1

You need to use IterateOver Pattern.
When you find a file to copy, increment a variable and when S3 copy is done, track with another variable that it is copied.
When the totalfind == totalcopied, then initiate the callback from the calling function.
function WaterfallOver(list, iterator, callback) {
var nextItemIndex = 0; //keep track of the index of the next item to be processed
function report() {
nextItemIndex++;
// if nextItemIndex equals the number of items in list, then we're done
if(nextItemIndex === list.length)
callback();
else
// otherwise, call the iterator on the next item
iterator(list[nextItemIndex], report);
}
// instead of starting all the iterations, we only start the 1st one
iterator(list[0], report);
}
Hope it helps.