Node js request huge amount of pictures - node.js

Question

I am trying to request a huge amount of images on node js (n > 3000)
and then save them into a folder.
I have tried using request library and request-promise.
The problem is that if the number of pictures is too big some pictures do not complete downloading, leaving incomplete data or get an error(an empty .jpeg file). Is there a better way of downloading huge amounts of pictures?
Also i need when the pictures all download to request a function compile()
to make them into a video. I am using ffmpeg for that.
Below are 2 ways i tried doing this.
const req = require('request');
const request = require('request-promise');
const fs = require('fs');
const ffmpegPath = require('#ffmpeg-installer/ffmpeg').path;
const ffmpeg = require('fluent-ffmpeg');
ffmpeg.setFfmpegPath(ffmpegPath);
function downloadImgs(imgUrls) {
let promises = [];
let prom;
for (let i = 0; i < imgUrls.length; i++) {
imgPath = `assets/pics/st_${pad(i + 1, 3)}.jpg`
prom = request(imgUrls[i]);
prom.on('error', () => console.log('err'));
prom.pipe(fs.createWriteStream(imgPath));
promises.push(prom);
}
Promise.all(promises).then(() => {
console.log('I Run');
compilee();
});
//SECOND TRY-----------------------------------------
for (let i = 0; i < imgUrls.lengh; i++) {
imgUrl = imgUrls[i];
req(imgUrl)
.on('error', () => console.log("img error", imgUrl))
.pipe(fs.createWriteStream(`assets/pics/st_${pad(i + 1, 3)}.jpg`)).on('close', () => {
console.log('downloaded', x)
})
}
compilee();
//---------------------------------------------------------
function compilee() {
command
.on('end', onEnd)
// .on('progress', onProgress)
.on('error', onError)
.input('assets/pics/st_%03d.jpg')
.inputFPS(5)
.output('assets/output/pepe.mp4')
.outputFps(30)
.run();
}
The errors i am getting for the first: UnhandledPromiseRejectionWarning: RequestError: Error: socket hang up
and the second :
Error: socket hang up
at createHangUpError (_http_client.js:330:15)
at TLSSocket.socketOnEnd (_http_client.js:433:23)
at TLSSocket.emit (events.js:187:15)
at endReadableNT (_stream_readable.js:1098:12)
at process.internalTickCallback
(internal/process/next_tick.js:72:19) code: 'ECONNRESET' }
Also, should i download the pictures on the server or on the client if i want to sent the video to the client immediately.

Answer 1

I may suggest you to use async library. Que might be a good solution
var q = async.queue(function({url,index}, callback) {
// Do ur job here
}, 5);
// assign a callback
q.drain = function() {
console.log('All images downloaded');
};
// add items to the queue
for (let i = 0; i < imgUrls.length; i++) {
q.push({url : imgUrls[i], index : i});
}
Ofc it is not a working all code block but it is just to give you the idea. Hope it helps