How does child_process behave in Nodejs - node.js

Question

I have a working Nodejs code, however, the child_process library behaves strange, I am just wondering how this library works.
My code is trying to download the SSL certificates from S3, then create the two new files based on the exiting ones using child_process library.
const AWS = require('aws-sdk');
const fs = require('fs');
const child_process = require("child_process");
const exec = require('child_process').exec;
var s3 = new AWS.S3();
var filePath = '../Desktop/Certs/'
var bucketName = 'neb.certificates' //replace example bucket with your s3 bucket name
var params = {
Bucket: bucketName,
Prefix: 'dev/jenkins.secure.care/',
};
s3.listObjectsV2(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else {
// console.log(data.Contents)
var len = data.Contents.length
for(var i=0; i<len; i++){
var key = data.Contents[i]["Key"]
var newPath = filePath.concat(key.substring(31))
const downloadFile = (newPath, bucketName, key) => {
//construct getParam
var getParams = {
Bucket: bucketName,
Key: key
};
s3.getObject(getParams, (err, data) => {
if (err) console.error(err)
fs.writeFileSync(newPath, data.Body.toString())
// console.log(`${newPath} has been created!`)
})
}
downloadFile(newPath, bucketName, key)
}
}
});
exec('mv ../Desktop/Certs/cert.pem ../Desktop/Certs/jenkins.crt', (err, stdout, stderr) => {
if (err) {
console.error(err);
return;
}
console.log(stdout);
});
exec('mv ../Desktop/Certs/privkey.pem ../Desktop/Certs/jenkins.key', (err, stdout, stderr) => {
if (err) {
console.error(err);
return;
}
console.log(stdout);
});
So when I run the code the first time, it only downloads the certificates from S3 to the local folder, it did not create the other 2 files. Then I have to run it the second time to create additional files.
However, I just want to run it once, and it has everything what I expect.
I have added a code which to sleep 5 seconds, then create the additional files, But it did not solve my problem, which means I still run the code twice to get everything.
child_process.execSync("sleep 5");
Please helps

Answer 1

It looks like the problem here is the async execution of your code. In this example here's what's happening:
Run s3.listObjectsV2() and when finished, fire callback (but not now, in the future)
Run first exec() and when finished, fire callback (but not now, in the future)
Run second exec() and when finished, fire callback (but not now, in the future)
And those three steps are fired immediately, one by one. And each of them has its own callback which fires in the future. Ok, but when is the future? - exactly! You don't know. In your case probably those two callbacks in exec()'s are fired before the callback from s3 and this is why it does not work.
The solution here is to make sure that those exec()s are fired after s3.listObjects. So you have two options: first is to make a promise out of the s3, like this: s3.listObjectsV2(params).promise() and await for it, then in .then((data) => {}) you have your data and in .catch((error) => {}) you have your error. Or you can simply put those exec()s in the callback of the s3 call.
Your code should look like this according to the solution 2 (from the comments):
const AWS = require('aws-sdk');
const fs = require('fs');
const child_process = require("child_process");
const exec = require('child_process').exec;
var s3 = new AWS.S3();
var filePath = '../Desktop/Certs/'
var bucketName = 'neb.certificates' //replace example bucket with your s3 bucket name
var params = {
Bucket: bucketName,
Prefix: 'dev/jenkins.secure.care/',
};
s3.listObjectsV2(params, async function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else {
// console.log(data.Contents)
var len = data.Contents.length
for(var i=0; i<len; i++){
var key = data.Contents[i]["Key"]
var newPath = filePath.concat(key.substring(31))
const downloadFile = (newPath, bucketName, key) => {
//construct getParam
var getParams = {
Bucket: bucketName,
Key: key
};
return s3.getObject(getParams).promise();
};
const downloadData = await downloadFile(newPath, bucketName, key).catch(console.error);
fs.writeFileSync(newPath, downloadData.Body.toString());
console.log(newPath, 'created');
}
//rename files
console.log('renaming first cert.pem');
exec('mv ../Desktop/Certs/cert.pem ../Desktop/Certs/jenkins.crt', (err, stdout, stderr) => {
if (err) {
console.error(err);
return;
}
console.log(stdout);
});
console.log('renaming second privkey.pem');
exec('mv ../Desktop/Certs/privkey.pem ../Desktop/Certs/jenkins.key', (err, stdout, stderr) => {
if (err) {
console.error(err);
return;
}
console.log(stdout);
});
}
});