node csv files upload synchronous import followed by mongoose find - node.js

Question

I am not able to get results from a mongoose find query after successfully uploading csv files. The data from the csvs is being saved in my collection OK. But I am getting no results from a following find query. What causes promises to not conform to the sequence? I feel I may be close to understanding promises as they are handled in the event loop but this case escapes me?
let uploadfiles = function (req,res) {
return new Promise( function (resolve, reject) {
upload(req, res, (err) => {
promises = [];
if (!err) {
if (Object.keys(req.files).length>0) {
simpleLoop(req.files, function (value, key, lovey) {
promises.push(insertStreamData(req.files[key]["originalname"]));
});
console.log('hither');
}
}
return Promise.all(promises);
});
resolve('something');
});
};
// insert uploaded csv data from files into db
function insertStreamData(filename){
var originalFileName = filename;
var thefilename = originalFileName.split('.');
var csvfile = "./public/files/"+originalFileName;
var stream = fs.createReadStream(csvfile, { headers: true});
var csvStream = csv().on("data", function(data){
if(data[0] != 'Product #' && data[7] != 0){
var item = new Product({
'Invoice':thefilename[0],
'Product #': data[0],
'SKU': data[1],
'UpcCode': data[2],
'Description': data[3],
'MfgNo': data[4],
'Vendor': data[5],
'Order Qty': data[6],
'Ship Qty': data[7],
'Min Sell': data[8],
'Retail': data[9],
'Cost': data[10],
'Ext Cost': data[11],
'Box': data[12]
});
item.save(function(error){ if(error){ throw error; } });
}
}).on("end", function(){
console.log('the end');
});
stream.pipe(csvStream);
}
let getincomingproducts = function(){
return new Promise(function (resolve, reject) {
resolve('done');
});
};
router.post('/uploaddata', function(req, res) {
uploadfiles(req,res).then(function(result){
return getincomingproducts();
}).then(function(result){
console.log(result);
res.redirect('/showdata');
}).catch(err => console.log("Caught " + err));
});
Output 'done' is logged to the console prior to the first promise. I want to replace done with mongoose find results. But a simple console log demonstrates the unsynchronous result.
done
hither
the end
the end
the end
the end

Answer 1

Please don't mix Promise constructors and such with async and await. This could be a lot easier if you just use await and async functions. And only wrap the callback taking functions once.
A clean version wouldn't have any business logic in the Promise constructors and would used await rather than then in an async function.
I found the second point in the program where it doesn't wait for the completion of an action that is that the insertStreamData does not return a promise that is fulfilled when it is done which is what is causing the out of order issue. I will update my answer.
I see a issue in upload files where it has:
return Promise.all(promises);
});
resolve('something');
which should be
return Promise.all(promises).then(function () {
resolve('something');
});
This change should cause the output to be hither, the end * 4, done. If you want the insertStreamData to occur in serial rather than parallel, that requires a different change.
My version of the script:
function upload_wrapper(req, res) {
return new Promise(function(resolve, reject) {
upload(req, res, function (err) {
if (err) {
reject(err);
} else {
resolve();
}
});
})
}
let uploadfiles = async function(req, res) {
await upload_wrapper(req, res);
let promises = [];
if (Object.keys(req.files).length > 0) {
simpleLoop(req.files, function(value, key, lovey) {
promises.push(insertStreamData(req.files[key]["originalname"]));
});
console.log('hither');
}
await Promise.all(promises);
return 'something';
};
// insert uploaded csv data from files into db
function insertStreamData(filename) {
return new Promise(function(resolve, reject) {
var originalFileName = filename;
var thefilename = originalFileName.split('.');
var csvfile = "./public/files/" + originalFileName;
var stream = fs.createReadStream(csvfile, {
headers: true
});
var csvStream = csv();
csvStream.on("data", function(data) {
if (data[0] != 'Product #' && data[7] != 0) {
var item = new Product({
'Invoice': thefilename[0],
'Product #': data[0],
'SKU': data[1],
'UpcCode': data[2],
'Description': data[3],
'MfgNo': data[4],
'Vendor': data[5],
'Order Qty': data[6],
'Ship Qty': data[7],
'Min Sell': data[8],
'Retail': data[9],
'Cost': data[10],
'Ext Cost': data[11],
'Box': data[12]
});
item.save(function(error) {
if (error) {
//throw error;
csvStream.pause(); // here the stream should be destroyed if this stops at the first error
reject(error);
}
});
}
}).on("end", function() {
console.log('the end');
resolve('the end');
});
stream.pipe(csvStream);
});
}
let getincomingproducts = function() {
return Promise.resolve('done');
};
router.post('/uploaddata', async function(req, res) {
try {
let result = await uploadfiles(req, res);
let result = await getincomingproducts();
console.log(result);
res.redirect('/showdata');
} catch (err) {
console.log("Caught " + err)
}
});
Untested. It still could be improved.