I have the following piece of code:
array.forEach(function (item) {
// *** some processing on each item ***
var params = {Key: item.id, Body: item.body};
s3bucket.upload(params, function(err, data) {
if (err) {
console.log("Error uploading data. ", err);
} else {
console.log("Success uploading data");
}});
});
Because s3bucket.upload is being executed asynchronously - the loop finishes before uploading all the items.
How can I force s3bucket.upload to be synchronous?
Meaning don't jump to next iteration until this item was uploaded (or failed) to S3.
Thanks
you can use https://github.com/caolan/async#each each or eachSeries
function upload(array, next) {
async.eachSeries(array, function(item, cb) {
var params = {Key: item.id, Body: item.body};
s3bucket.upload(params, function(err, data) {
if (err) {
console.log("Error uploading data. ", err);
cb(err)
} else {
console.log("Success uploading data");
cb()
}
})
}, function(err) {
if (err) console.log('one of the uploads failed')
else console.log('all files uploaded')
next(err)
})
}
Better to use promises as suggested in one of the comments:
const uploadToS3 = async (items) => {
for (const item of array) {
const params = { Key: item.id, Body: item.body };
try {
const data = await s3bucket.upload(params).promise();
console.log("Success uploading data");
} catch (err) {
console.log("Error uploading data. ", err);
}
}
}
You could pass a post back function, this way the rest of the code is executed only when the upload has been completed. This does not answer your question but could be an alternative option:
array.forEach(function (item) {
// *** some processing on each item ***
var params = {Key: item.id, Body: item.body};
var f1=function(){
// stuff to do when upload is ok!
}
var f2=function(){
// stuff to do when upload fails
}
s3bucket.upload(params, function(err, data) {
if (err) {
f2();
console.log("Error uploading data. ", err);
// run my function
} else {
// run my function
f1();
console.log("Success uploading data");
}});
});
Related
So, I have this:
async function uploadImageToFtp(fileName, path) {
const client = new ftp.Client()
client.ftp.verbose = true
try {
await client.access({
host: process.env.FTP_HOST,
user: process.env.FTP_USER,
password: '123',
secure: false
})
await client.uploadFrom(path, "tables/" + fileName)
} catch (err) {
console.log(err)
}
client.close()
}
fs.readdir('plates', function(err, files) {
//handling error
if (err) {
return console.log('Unable to scan directory: ' + err);
}
//listing all files using forEach
files.forEach(function(file) {
uploadImageToFtp(file, 'plates/' + file);
console.log(file);
});
});
But I get "too many FTP connections...".
So, how to wait for 1 file to upload and then to continue with seconds etc...?
Thank you!
Use for-loop instead of forEach, and use async/await completely for your example:
fs.readdir('plates', async function (err, files) { // async function, carefully this line, `readdir` still is a callback function
//handling error
if (err) {
return console.log('Unable to scan directory: ' + err);
}
//listing all files using forEach
for (const file of files) {
await uploadImageToFtp(file, 'plates/' + file); // wait until it done
console.log(file);
}
});
I am new to nodejs and async. Having trouble understanding how can I wrap the two separate async.each methods to have one res.render...I am trying to display a list of valid account ids, and valid user ids on the front end.
The two separate async.each methods are:
async.each(account_ids, function(accountId, callback) {
console.log('Processing accountId ' + accountId);
callingExternalApi(accountId, callback, function(err, response){
if(err){
console.log("error account");
}
console.log("account response is: ", response);
});
}, function(err) {
if( err ) {
console.log('An account failed to process');
} else {
console.log('All accounts have been processed successfully');
}
});
and
async.each(email_ids, function(emailId, callback) {
console.log('Processing email id ' + emailId);
request({
url: emailIdlookupUrl,
method: 'POST',
json: {
email_address: emailId
}
}, function (err, response, body) {
if (err) {
logger.error(err);
req.flash('error', err.message);
return res.redirect('?');
}
if (response.statusCode !== 200) {
const msg = 'Unable to verify user';
req.flash('error', msg);
return res.redirect('?');
}
console.log("user id is: ", body.user.id);
callback();
});
}, function(err) {
if( err ) {
console.log('An email failed to process');
} else {
console.log('All user emails have been processed successfully');
}
});
Any help is highly appreciated. Please excuse me for any redundant callbacks or error logging. Still learning nodejs.
Thanks!!
The main issue is not that you are invoking both of these async.each calls. The problem is that they will run in parallel, and the fastest one to invoke req.* functions or callback function will return a response to the connection.
Both of these functions return promises if their callback parameters are omitted.
I recommend reading up on both the async library and JS async/await in general:
https://javascript.info/async-await
https://caolan.github.io/async/v3/docs.html#each
https://zellwk.com/blog/async-await-express/
Note that async also accepts native async functions, which many finder cleaner and easier to understand.
Here is what I think you want from the code above, including compiling the results into lists:
var request = require("request-promise");
async function checkAccounts(account_ids) {
const valid_accounts = [];
await async.each(account_ids, async function(accountId) {
console.log("Processing accountId " + accountId);
const extAPIresult = await callingExternalApi(accountId);
console.log("account response is: ", extAPIresult);
});
valid_accounts.push(extAPIresult);
console.log("All accounts have been processed successfully");
return valid_accounts;
}
async function checkEmails(email_ids) {
const valid_emails = [];
await async.each(email_ids, async function(emailId) {
console.log("Processing email id " + emailId);
const reqresult = await request({
url: emailIdlookupUrl,
method: "POST",
json: {
email_address: emailId
}
});
if (reqresult.statusCode !== 200) {
throw new Error("Unable to verify user");
}
valid_emails.push(reqresult.body.user.id);
console.log("user id is: ", reqresult.body.user.id);
});
console.log("All emails have been processed successfully");
return valid_emails;
}
async function doChecks() {
const accounts = checkAccounts(account_ids);
const emails = checkEmails(email_ids);
const responses = await Promises.all([accounts, emails]);
console.log("All checks have been processed successfully");
return responses;
}
function get(req, res) {
doChecks()
.then(responses => {
res.send("All checks have been processed successfully");
res.send(String(responses));
})
.catch(err => {
req.flash("error", err.message);
res.redirect("?");
});
}
I have a lambda function that does work successfully. I am able to create a file on S3 and another FTP server. The issue is even after successfull completion it time outs. It just doesn't finish executing. I could see from the logs it takes around 2 seconds and the timeout specified is 5 seconds. How do I make my function stops as soon as it is done? Here is the complete code:
"use strict";
var config = require('./config/dev');
var sql = require("mssql");
var AWS = require('aws-sdk');
var PromiseFtp = require('promise-ftp');
var fs = require('fs');
const request = require('request');
exports.handler = (event, context, callback) => {
GetDataFromServer(event, context, callback);
};
function GetDataFromServer(event, context, callback) {
console.log("Fetching data from database...");
var keys = [], outputText = '';
sql.connect(config.db, err => {
if (err) {
console.log("Error while connecting database :- " + err);
return callback(true, 'Error in fetching records from database...');
}
else {
new sql.Request()
.input('ClientId', sql.Int, 469)
.execute('ForesightDailyDataPull', (err, result) => {
if (!err) {
//Create Text here and assign to "outputText"
CreateFileOnS3(outputText, event, context, callback);
}
else {
console.log('Error in fetching records from database...');
return callback(true, 'Error in fetching records from database...');
}
})
}
});
sql.on('error', err => {
console.log('Error in fetching records from database...');
return callback(true, 'Error in fetching records from database...');
})
}
function CreateFileOnS3(fileData, event, context, callback) {
const fileName = generateFileName();
console.log('Sending file to S3...');
const s3 = new AWS.S3(config.awsCredentials);
const params = {
Bucket: config.app.s3Bucket,
Key: fileName,
Body: fileData
};
s3.upload(params, function (s3Err, data) {
if (s3Err) {
console.log('There was an error creating file on S3');
return callback(true, 'There was an error creating file on S3');
}
else {
console.log(`File uploaded successfully at ${data.Location}`);
CreatefileOnFTP(fileData, fileName, event, context, callback);
}
});
}
function CreatefileOnFTP(fileData, fileName, event, context, callback) {
console.log('Sending file to FTP...');
var ftpObject = {
"fileData": fileData,
"fileName": fileName,
"ftpURL": config.ftpDetails.ftpProtocol + "://" + config.ftpDetails.host,
"ftpUserName": config.ftpDetails.user,
"ftpPassword": config.ftpDetails.password
};
request({
url: config.ftpUploadURL,
method: "POST",
json: true,
body: ftpObject
}, function (error, response, body) {
if (!error) {
console.log('File sent successfully to FTP server.');
return callback(null, 'File sent successfully to FTP...');
}
else {
console.log('An error occurred while sending file to FTP.');
return callback(true, 'Error in sending file to FTP...');
}
});
}
function generateFileName() {
var _d = new Date(),
y = _d.getFullYear(),
m = _d.getMonth() + 1,
d = _d.getDate();
return y + '-' + (m < 10 ? '0' + m : m) + '-' + (d < 10 ? '0' + d : d) + '.txt';
}
You should close the open sql connection after the function completes by calling sql.close() after the action resolves.
function GetDataFromServer(event, context, callback) {
console.log("Fetching data from database...");
var keys = [], outputText = '';
sql.connect(config.db, err => {
if (err) {
console.log("Error while connecting database :- " + err);
return callback(true, 'Error in fetching records from database...');
}
else {
new sql.Request()
.input('ClientId', sql.Int, 469)
.execute('ForesightDailyDataPull', (err, result) => {
if (!err) {
//Create Text here and assign to "outputText"
CreateFileOnS3(outputText, event, context, callback);
sql.close() //HERE
}
else {
console.log('Error in fetching records from database...');
sql.close() //HERE
return callback(true, 'Error in fetching records from database...');
}
})
}
});
sql.on('error', err => {
console.log('Error in fetching records from database...');
sql.close() //HERE
return callback(true, 'Error in fetching records from database...');
})
}
I have download a zip file from s3 bucket then extracting the zip file
and finally upload one file to s3 bucket in Lambda function using
Node JS.But am getting the error
==> Error: EROFS: read-only file system, open './tmp/test.zip'
"Process exited before completing> request"
exports.handler = function (callback) {
downloadZipFile(params, downloadPath, function (err) {
if (err) {
callback(err);
} else {
processZipFile(downloadPath, function (err) {
if (err) {
callback(err);
} else {
callback(null);
}
});
}
});
};
function downloadZipFile(params, downloadPath, callback) {
const file = fs.createWriteStream(downloadPath);
s3.getObject(params)
.on('httpData', function (chunk) {
file.write(chunk);
})
.on('success', function () {
callback(null);
})
.on('error', function (err) {
callback(err);
})
.on('complete', function () {
file.end();
})
.send();
}
function processZipFile(filePath) {
const stats = fs.statSync(filePath)
const fileSizeInBytes = stats.size
if (fileSizeInBytes > 0) {
var srcPath = filePath;
var destPath = "./tmp";
targz.decompress({
src: srcPath,
dest: destPath
}, function (err) {
if (err) {
console.log(err);
} else {
console.log("Done!");
UploadFile();
}
});
}
}
function UploadFile() {
var body = fs.createReadStream('./tmp/SampleFile.txt')
var srcfileKey = "SampleFile.txt";
// Upload the stream
var s3obj = new AWS.S3({ params: { Bucket: bucketName, Key: srcfileKey } });
s3obj.upload({ Body: body }, function (err, data) {
if (err) {
console.log("An error occurred", err);
}
console.log("Uploaded the file at", data.Location);
})
}
You need to change the file path to just /tmp instead of ./tmp. Lambda only allows you to write to the /tmp directory.
OK I have a NodeJS app and I'm trying to download lots of images from a web server (about 500 for now but the number will increase). The problem I get is a "Unhandled stream error in pipe Error: EMFILE" because it seems that too much files get opened at the same time.
So I'm trying to use async.queue to process files by batches of 20. But I still get the error.
SomeModel.find({}, function(err, photos){
if (err) {
console.log(err);
}
else {
photos.forEach(function(photo){
var url = photo.PhotoURL;
var image = url.replace('http://someurl.com/media.ashx?id=', '').replace('&otherstuffattheend', '.jpg');
photo.target = image;
var q = async.queue(function (task) {
request
.get(task.PhotoURL)
.on('response', function(response) {
console.log(task.PhotoURL + ' : ' + response.statusCode, response.headers['content-type']);
console.log(task.target);
})
.on('error', function(err) {
console.log(err);
})
.pipe(fs.createWriteStream(task.target));
}, 20);
q.push(photo, function(err) {
if (err) {
console.log(err);
}
});
q.drain = function() {
console.log('Done.')
}
});
}
});
What am I doing wrong ? Many thanks for your time and help.
The problem is that you're creating a new queue for each photo and each queue receives just one photo. Instead, only create a queue once (outside of the forEach()) and push the photo objects to it. You're also missing the callback in your task handler. For example:
var q = async.queue(function(task, cb) {
request
.get(task.PhotoURL)
.on('response', function(response) {
console.log(task.PhotoURL + ' : ' + response.statusCode, response.headers['content-type']);
console.log(task.target);
// the call to `cb` could instead be made on the file stream's `finish` event
// if you want to wait until it all gets flushed to disk before consuming the
// next task in the queue
cb();
})
.on('error', function(err) {
console.log(err);
cb(err);
})
.pipe(fs.createWriteStream(task.target));
}, 20);
q.drain = function() {
console.log('Done.')
};
photos.forEach(function(photo) {
var url = photo.PhotoURL;
var image = url.replace('http://someurl.com/media.ashx?id=', '').replace('&otherstuffattheend', '.jpg');
photo.target = image;
q.push(photo, function(err) {
if (err) {
console.log(err);
}
});
});