NodeJS download remote to local FTP - node.js

Question

I'm trying to download a remote location on a FTP server to my local directory. I've been using the jsftp module (https://github.com/sergi/jsftp) for this, but I'm running into a problem.
Basically I'm trying to recursively download the whole remote location, including all the sub-folders etc. I've been trying to figure it out myself, but so far no luck.
so far I've been trying this:
var worker = {
ftp: null,
init: function() {
this.ftp = new JSFtp({
host: "localhost",
port: 21,
user: "username",
pass: "password"
});
this.ftp.auth("username", "password", function (result) {
if (typeof result !== "undefined") {
console.log("Something went wrong");
}
this.handleData();
}.bind(this));
},
handleData: function () {
recursive_get_files(this, "/");
}
};
function recursive_get_files(worker, dir) {
console.log("Getting directory: " + dir);
worker.ftp.ls(dir, function (err, res) {
res.forEach(function (file) {
if (file.type === 1) {
recursive_get_files(worker, dir + "/" + file.name);
} else {
worker.ftp.get(dir + "/" + file.name, "/downloads" + dir + "/" + file.name, function(err) {
if (err) {
console.log("Couldn't download file: " + file.name);
console.log(err);
}
}.bind(file.name));
}
});
});
}
The biggest problem, I think, is that all these get functions are getting called almost instantly after each other, and since the client is probably not allowed to call so many things, it will break.
I've seen a module named ftpsync (https://www.npmjs.com/package/ftpsync) do some remote syncing, from local to remote, but I need this the otherway around.
is anyone able to help me here? I've been stuck on this all day =/.

Answer 1

I was able to fix this with a friend of mine by doing the following:
function recursive_get_files(workerf, dir) {
total_pending++;
var worker = workerf;
var done = function() {
//probably callback here if needed
console.log("finished downloading ftp");
wrench.chmodSyncRecursive('downloads', 0777);
worker.respond(JSON.stringify({type: "success", message: "Finished"}));
};
var func_download = function() {
//Is the download queue empty?
if(download_queue.length === 0)
{
total_running--;
if(total_running === 0 && total_pending === 0)
{
done();
}
return;
}
//Get the next download in the queue
var download = download_queue[0];
download_queue.splice(0, 1);
//Get a free FTP connection
var curftp;
for(curftp = 0; curftp < total; curftp++) {
if (worker.ftp[curftp].used === false) {
worker.ftp[curftp].used = true;
break;
}
}
//Get the file
worker.ftp[curftp].ftp.get(download.dir + "/" + download.file.name, "downloads/" + worker.project + download.dir + "/" + download.file.name, function(file, err) {
worker.ftp[curftp].used = false;
if (err)
{
console.log("Couldn't download file with FTP(" + curftp + "): " + file);
console.log(err);
setTimeout(func_download, 0);
}
else
{
console.log("Downloaded file with FTP(" + curftp + "): " + file);
setTimeout(func_download, 0);
}
}.bind(null, download.file.name));
};
//Get a list of the current directory (Using the main connection)
worker.mainftp.ls(dir, function (err, res) {
res.forEach(function (file) {
if (file.type === 1)
{
mkdirp("downloads/" + worker.project + dir + "/" + file.name, function(err) {
if (err) console.log(err);
recursive_get_files(worker, dir + "/" + file.name);
});
}
else
{
download_queue.push({file: file, dir: dir});
if(total_running < total)
{
total_running++;
setTimeout(func_download, 0);
}
}
});
});
total_pending--;
if(total_running === 0 && total_pending === 0 && download_queue.length === 0)
{
done();
}
}
it uses wrench.js (for recursive chmod) and jsftp. Thanks for reading ^^