has any one of you tried to upload 50+ images to cloundinary? I been trying but the issue is that promise does not get resolved (even with .reflect() and fails to upload all images. Depending on the speed of uploads, it will fail 30% ~ 70%.
Is there any ways to completely make it async and make sure all images are uploaded correctly? Only modules that I am using is bluebird and cloudinary module from their doc.
Promisebd = require('bluebird');
function uploadimg(img, i, itemId) {
var pubID = 'a2z/toys/' + itemId + '/' + i;
// return new cloudImg.v2.uploader.upload(img, {
return cloudinary.v2.uploader.upload(img, { // works
public_id: pubID,
quality: 90
// use_filename: true,
} , function(err, res){
if(err) {
console.log(err);
}
console.log(res);
});
}
promiseArr.push(uploadimg(fullimg, i, d[0].details.detailsProductId)); // pushing the promises to Arr
Promisebd.all(promiseArr.map(function(promise) {
return promise.reflect();
})).each(function(inspection) {
if(inspection.isFulfilled()) {
console.log('The promise is the arr was fulfilled with ', inspection.value());
}else{
console.log('The promise is NOT the arr was NOT fulfilled with ', inspection.reason());
}
})
promsify your upload img function and try to use it
function uploadimgAsync(img, i, itemId) {
return new Promise(function (resolve, reject) {
var pubID = 'az/toys/' + itemId + '/' + i;
cloudinary.v2.uploader.upload(img, { // works
public_id: pubID,
quality: 90
},
function(err, res){
if(err) {
reject(err);
}
resolve(res);
});
});
}
Related
I want to generate multiple pdf files and attach to the email. But await seems not working on res.app.render.
route.get('/:id/receipts', async function (req, res) {
...
let attachments = [];
for await(let item of items){
res.view.item = item;
console.log(1)
await res.app.render('pdfs/receipt', res.view, async function(err, html){
console.log(2)
if (err) return res.end(err.stack)
return await pdf.create(html).toBuffer(async function(err, buffer){
console.log(3)
attachments.push({
content: buffer,
filename: 'receipt.pdf',
})
});
});
}
console.log(4)
...
})
Expect Result:
1
2
3
4
Actually Result:
1
4
2
3
I think res.app.render is not returning a promise that's why you are facing this issue. You have to make a custom promise. I hope following code will help you.
oute.get('/:id/receipts', async function (req, res) {
...
let attachments = [];
for await(let item of items){
res.view.item = item;
console.log(1)
const customPromise = new Promise((resolve, reject) => {
res.app.render('pdfs/receipt', res.view, async function(err, html){
console.log(2)
if (err) { res.end(err.stack);reject()}
else {
await pdf.create(html).toBuffer(async function(err, buffer){
console.log(3)
attachments.push({
content: buffer,
filename: 'receipt.pdf',
})
});
resolve();
}
});
})
}
console.log(4)
...
})
I've been struggling to figure out how to do this for the past two hours. But I can't seem to get the Promise to wait for my searchFacesFunc to complete before it solves. What is the correct way to do so?
async function searchFacesFunc(faceId){
var searchFacesParams = {
CollectionId: "my-collection",
FaceId: faceId,
FaceMatchThreshold: 80,
MaxFaces: 10
};
await rekognition.searchFaces(searchFacesParams, function(err, data) {
if(err){
throw err;
}else{
var matching_percent = data.FaceMatches[0].Similarity;
console.log('Matching Percent: ' + matching_percent);
}
});
}
return new Promise((resolve, reject) => {
rekognition.indexFaces(indexParams, function(err, data) {
if(err){
throw err;
}else{
const faceRecords = data.FaceRecords;
for(let i = 0; i < faceRecords.length; i++){
var faceId = faceRecords[i].Face.FaceId;
console.log('FaceId: ' + faceId);
searchFacesFunc(faceId); //The promise is finished before these multiple functions finish
}
resolve(null, 'success');
}
});
});
If the rekognition.indexFaces function accepts an asynchronous callback, you can solve this issue easily:
return rekognition.indexFaces(indexParams, async (err, data) => {
if (err) {
throw err;
} else {
const faceRecords = data.FaceRecords;
for (let i = 0; i < faceRecords.length; i++) {
var faceId = faceRecords[i].Face.FaceId;
console.log("FaceId: " + faceId);
await searchFacesFunc(faceId); // Await the promise
}
return "success";
}
});
};
However, if this is not the case, you can still solve this the following way:
Use util.promisify to "promisify" the rekognition.indexFaces function
Construct a recursive callback function that only resolves the original promise you constructed when it executed faceRecords.length times.
I am not able to get results from a mongoose find query after successfully uploading csv files. The data from the csvs is being saved in my collection OK. But I am getting no results from a following find query. What causes promises to not conform to the sequence? I feel I may be close to understanding promises as they are handled in the event loop but this case escapes me?
let uploadfiles = function (req,res) {
return new Promise( function (resolve, reject) {
upload(req, res, (err) => {
promises = [];
if (!err) {
if (Object.keys(req.files).length>0) {
simpleLoop(req.files, function (value, key, lovey) {
promises.push(insertStreamData(req.files[key]["originalname"]));
});
console.log('hither');
}
}
return Promise.all(promises);
});
resolve('something');
});
};
// insert uploaded csv data from files into db
function insertStreamData(filename){
var originalFileName = filename;
var thefilename = originalFileName.split('.');
var csvfile = "./public/files/"+originalFileName;
var stream = fs.createReadStream(csvfile, { headers: true});
var csvStream = csv().on("data", function(data){
if(data[0] != 'Product #' && data[7] != 0){
var item = new Product({
'Invoice':thefilename[0],
'Product #': data[0],
'SKU': data[1],
'UpcCode': data[2],
'Description': data[3],
'MfgNo': data[4],
'Vendor': data[5],
'Order Qty': data[6],
'Ship Qty': data[7],
'Min Sell': data[8],
'Retail': data[9],
'Cost': data[10],
'Ext Cost': data[11],
'Box': data[12]
});
item.save(function(error){ if(error){ throw error; } });
}
}).on("end", function(){
console.log('the end');
});
stream.pipe(csvStream);
}
let getincomingproducts = function(){
return new Promise(function (resolve, reject) {
resolve('done');
});
};
router.post('/uploaddata', function(req, res) {
uploadfiles(req,res).then(function(result){
return getincomingproducts();
}).then(function(result){
console.log(result);
res.redirect('/showdata');
}).catch(err => console.log("Caught " + err));
});
Output 'done' is logged to the console prior to the first promise. I want to replace done with mongoose find results. But a simple console log demonstrates the unsynchronous result.
done
hither
the end
the end
the end
the end
Please don't mix Promise constructors and such with async and await. This could be a lot easier if you just use await and async functions. And only wrap the callback taking functions once.
A clean version wouldn't have any business logic in the Promise constructors and would used await rather than then in an async function.
I found the second point in the program where it doesn't wait for the completion of an action that is that the insertStreamData does not return a promise that is fulfilled when it is done which is what is causing the out of order issue. I will update my answer.
I see a issue in upload files where it has:
return Promise.all(promises);
});
resolve('something');
which should be
return Promise.all(promises).then(function () {
resolve('something');
});
This change should cause the output to be hither, the end * 4, done. If you want the insertStreamData to occur in serial rather than parallel, that requires a different change.
My version of the script:
function upload_wrapper(req, res) {
return new Promise(function(resolve, reject) {
upload(req, res, function (err) {
if (err) {
reject(err);
} else {
resolve();
}
});
})
}
let uploadfiles = async function(req, res) {
await upload_wrapper(req, res);
let promises = [];
if (Object.keys(req.files).length > 0) {
simpleLoop(req.files, function(value, key, lovey) {
promises.push(insertStreamData(req.files[key]["originalname"]));
});
console.log('hither');
}
await Promise.all(promises);
return 'something';
};
// insert uploaded csv data from files into db
function insertStreamData(filename) {
return new Promise(function(resolve, reject) {
var originalFileName = filename;
var thefilename = originalFileName.split('.');
var csvfile = "./public/files/" + originalFileName;
var stream = fs.createReadStream(csvfile, {
headers: true
});
var csvStream = csv();
csvStream.on("data", function(data) {
if (data[0] != 'Product #' && data[7] != 0) {
var item = new Product({
'Invoice': thefilename[0],
'Product #': data[0],
'SKU': data[1],
'UpcCode': data[2],
'Description': data[3],
'MfgNo': data[4],
'Vendor': data[5],
'Order Qty': data[6],
'Ship Qty': data[7],
'Min Sell': data[8],
'Retail': data[9],
'Cost': data[10],
'Ext Cost': data[11],
'Box': data[12]
});
item.save(function(error) {
if (error) {
//throw error;
csvStream.pause(); // here the stream should be destroyed if this stops at the first error
reject(error);
}
});
}
}).on("end", function() {
console.log('the end');
resolve('the end');
});
stream.pipe(csvStream);
});
}
let getincomingproducts = function() {
return Promise.resolve('done');
};
router.post('/uploaddata', async function(req, res) {
try {
let result = await uploadfiles(req, res);
let result = await getincomingproducts();
console.log(result);
res.redirect('/showdata');
} catch (err) {
console.log("Caught " + err)
}
});
Untested. It still could be improved.
I'm trying to use Promise.all function but actually as I start nodeJS and I discover asynchronous technology I don't know where is my problem in my code...
Basically I would like to use Promise.all to make my own callback in a function then In my loop For I create several Promise and if I can save my data then I'll resolve my current Promise.
But apparently my promise.all is executed immediately and it don't wait for my others Promise..
See below ..
function persistMAP(jsonData,callback){
//Deck persistance
const promises =[];
for(var i=0; i<1; i++){
(function(i) {
var rowData = new DeckDatabase({
_id: new mongoose.Types.ObjectId(),
DeckNumber: Number(jsonData.Deck[i].DeckNumber),
x: Number(jsonData.Deck[i].x),
y: Number(jsonData.Deck[i].y),
});
rowData.save(function (err) {
if (err) return console.log(err);
for(var index=0; j=jsonData.Units.length,index<j; index++){
(function(index) {
promises.push(
new Promise(function(resolve,reject){
var unit = new MapDatabase({
UnitID:jsonData.Units[index].UnitID,
TypeID: Number(jsonData.Units[index].TypeID),
x: Number(jsonData.Units[index].x),
y: Number(jsonData.Units[index].y),
_id: mongoose.Types.ObjectId(((jsonData.Units[index].Code).toLowerCase()) + 'dd40c86762e0fb12000003'), // mongoose.Types.ObjectId(jsonData.Units[i].Code + 'dd40c86762e0fb12000003')
MainClass: jsonData.Units[index].MainClass,
Orientation: jsonData.Units[index].Orientation,
Postion: jsonData.Units[index].Postion,
Deck : String(rowData._id)
});
unit.save(function (err) {
if (err) {
reject();
throw err
}
else{
console.log('save');
resolve();
}
});
})
);
})(index);
}
});
})(i);
}
Promise.all(promises)
.then(()=>{
console.log('start find');
callback();
})
};
and here is where I call my function
else{
var jobj = JSON.parse(response.body);
console.log("persist begin");
persistMAP(jobj,function(){
console.log('retrieve Done');
MapDatabase.find()
.populate('Deck')
.exec(function(err, finalData){
console.log('send');
res.send(finalData);
})
});
}
So why It doesn't wait ? :(
You do an async operation in your for:
for(var i=0; i<1; i++){
....
rowData.save(function (err) {
So your promises array isn't filled with all the promise.
To loop async use async module :
var async = require("async");
var models = []; // my models array
// async loop, with 10 in same time, e is the element, i the index in models
async.eachOfLimit(models, 10, function(e, i, cb){
anAsyncFunction(function(err){
return cb(err);
});
}, function(err, result{
// all done
});
OK I have a NodeJS app and I'm trying to download lots of images from a web server (about 500 for now but the number will increase). The problem I get is a "Unhandled stream error in pipe Error: EMFILE" because it seems that too much files get opened at the same time.
So I'm trying to use async.queue to process files by batches of 20. But I still get the error.
SomeModel.find({}, function(err, photos){
if (err) {
console.log(err);
}
else {
photos.forEach(function(photo){
var url = photo.PhotoURL;
var image = url.replace('http://someurl.com/media.ashx?id=', '').replace('&otherstuffattheend', '.jpg');
photo.target = image;
var q = async.queue(function (task) {
request
.get(task.PhotoURL)
.on('response', function(response) {
console.log(task.PhotoURL + ' : ' + response.statusCode, response.headers['content-type']);
console.log(task.target);
})
.on('error', function(err) {
console.log(err);
})
.pipe(fs.createWriteStream(task.target));
}, 20);
q.push(photo, function(err) {
if (err) {
console.log(err);
}
});
q.drain = function() {
console.log('Done.')
}
});
}
});
What am I doing wrong ? Many thanks for your time and help.
The problem is that you're creating a new queue for each photo and each queue receives just one photo. Instead, only create a queue once (outside of the forEach()) and push the photo objects to it. You're also missing the callback in your task handler. For example:
var q = async.queue(function(task, cb) {
request
.get(task.PhotoURL)
.on('response', function(response) {
console.log(task.PhotoURL + ' : ' + response.statusCode, response.headers['content-type']);
console.log(task.target);
// the call to `cb` could instead be made on the file stream's `finish` event
// if you want to wait until it all gets flushed to disk before consuming the
// next task in the queue
cb();
})
.on('error', function(err) {
console.log(err);
cb(err);
})
.pipe(fs.createWriteStream(task.target));
}, 20);
q.drain = function() {
console.log('Done.')
};
photos.forEach(function(photo) {
var url = photo.PhotoURL;
var image = url.replace('http://someurl.com/media.ashx?id=', '').replace('&otherstuffattheend', '.jpg');
photo.target = image;
q.push(photo, function(err) {
if (err) {
console.log(err);
}
});
});