Using the below code, I'm trying to download a file from one S3 bucket and upload it to another S3 bucket programmatically. The code executing without any issues/exceptions but the file is not getting processed.
const AWS = AWSXRay.captureAWS(require('aws-sdk'))
const S3 = new AWS.S3()
const fs = require('fs')
exports.index = async (event, context) => {
var getParams = {
Bucket: 'my-s3-test-bucket1',
Key: 'SampleVideo.mp4'
}
const inputFilename = '/tmp/SampleVideo.mp4';
const writeStream = fs.createWriteStream(inputFilename)
new Promise ((resolve, reject) => {S3.getObject(getParams).createReadStream().pipe(writeStream).on('end', () => {console.log('end');return resolve();}).on('error',(error) => {console.log('error');return reject(error);})});
writeStream.on('finish', function () {
var putParams = {
Body: fs.createReadStream(inputFilename),
Bucket: 'my-s3-test-bucket2',
Key: 'transfer-' + 'OutputVideo.mp4',
}
S3.upload(putParams, function (err, data) {
if (err) console.log(err, err.stack)
else console.log('logging data' + data)
})
})
}
It seems that because you are using async handler, your function completes prematurely before its body has a chance to fully execute.
You can wrap your code in a promise, as shown in the AWS docs, to actually tell your function to wait for its entire body to execute:
const AWS = AWSXRay.captureAWS(require('aws-sdk'))
const S3 = new AWS.S3()
const fs = require('fs')
exports.index = async (event, context) => {
const promise = new Promise(function(resolve, reject) {
var getParams = {
Bucket: 'my-s3-test-bucket1',
Key: 'SampleVideo.mp4'
}
const inputFilename = '/tmp/SampleVideo.mp4';
const writeStream = fs.createWriteStream(inputFilename)
new Promise ((resolve, reject) => {S3.getObject(getParams).createReadStream().pipe(writeStream).on('end', () => {console.log('end');return resolve();}).on('error',(error) => {console.log('error');return reject(error);})});
writeStream.on('finish', function () {
var putParams = {
Body: fs.createReadStream(inputFilename),
Bucket: 'my-s3-test-bucket2',
Key: 'transfer-' + 'OutputVideo.mp4',
}
S3.upload(putParams, function (err, data) {
if (err) console.log(err, err.stack)
else console.log('logging data' + data)
})
})
}
return promise
}
Related
I have written a Lambda function which opens an image and modifies it, before saving it to the /tmp folder and uploading it to an S3 bucket. This works locally, but when I run in in Lambda I get an error stating no such file or directory, open '/tmp/nft.png. What could be causing this? I originally thought that it was an issue with the write function not being awaited, but I don't think this can be the case since it works fine locally.
var Jimp = require("jimp")
var fs = require('fs')
var path = require("path")
var AWS = require('aws-sdk')
AWS.config.update({
accessKeyId: <removed>,
secretAccessKey: <removed>
})
var s3 = new AWS.S3()
async function updateImage() {
var img = await Jimp.read("base_img.png")
var font = await Jimp.loadFont("fonts/Audiowide.fnt")
img.print(<removed for simplicity>)
return img.write("/tmp/nft.png")
}
function uploadFile(id) {
return new Promise((resolve, reject) => {
fs.readFile("/tmp/nft.png", function (err, data) {
if (err) { throw err; }
params = {<removed>};
s3.putObject(params, function(err, data) {
if (err) {
console.log(err)
reject(err)
} else {
console.log("Successfully uploaded data");
resolve()
}
});
});
})
}
exports.handler = async (event) => {
await updateImage()
await uploadFile(event.queryStringParameters.id)
return {
statusCode: 200,
body: JSON.stringify("Generated image")
}
}
I have a Node.js AWS Lambda function and I am trying to read records from a CSV file in S3 and print its contents.
Below is my code to achieve the same however I am getting Null as output.
Code:
const AWS = require('aws-sdk');
const s3 = new AWS.S3();
const csv = require('csv-parser');
const bucket = 'awslambdabuckets';
const objectkey = 'record.csv';
const params = { Bucket: bucket, Key: objectkey };
const results = [];
exports.handler = async function (event, ctx, callback) {
try {
const file = s3.getObject(params).createReadStream();
file
.pipe(csv())
.on('data', function (data) {
results.push;
})
.on('end', () => {
console.log(results);
callback(null, results);
});
} catch (err) {
console.log(err);
callback(Error(err));
}
};
Output:
Can someone help me point out what's the problem and how to fix it.
You are not pushing the data to the result, see and make changes as below
exports.handler = async function (event, ctx, callback) {
try {
const file = s3.getObject(params).createReadStream();
file
.pipe(csv())
.on('data', function (data) {
results.push(data);
})
.on('end', () => {
console.log(results);
callback(null, results);
});
} catch (err) {
console.log(err);
callback(Error(err));
}
};
You are not pushing data to the array:
const AWS = require('aws-sdk');
const s3 = new AWS.S3();
const csv = require('csv-parser');
const bucket = 'awslambdabuckets';
const objectkey = 'record.csv';
const params = { Bucket: bucket, Key: objectkey };
const results = [];
exports.handler = function (event, ctx, callback) {
try {
const file = s3.getObject(params).createReadStream();
file
.pipe(csv())
.on('data', function (data) {
results.push(data); // --> here
})
.on('end', () => {
console.log(results);
callback(null, results);
});
} catch (err) {
console.log(err);
callback(Error(err));
}
};
I have json file uploaded to s3
then I wrote the following code to Query this file
const aws = require('aws-sdk');
const s3 = new aws.S3();
const bucket = 'hotels.mserver.online';
const objectKey = 'hotelsrates.json';
exports.handler = (event,context,callback) => {
// TODO implement
const response = getS3Objects(bucket,objectKey); //s3.listObjectsV2({}).promise();
console.log(response);
};
function getS3Objects(bucket,key) {
return s3.getObject({ Bucket:bucket, Key:key, ResponseContentType:'application/json '})
.promise().then(file => { return file })
.catch(error => { return error });
}`
but the result is getting null .
I understand what you are trying to accomplish here but that is not the right way to do it.
function getS3Objects(bucket,key){
return s3.getObject({Bucket:bucket,Key:key,ResponseContentType:'application/json'})
.promise().then(file=>{return file})
.catch(error =>{return error});
}`
The part above will still return a promise object, which means that you need to handle it accordingly. Instead of const response = getS3Objects(bucket,objectKey); you want to do
getS3Objects(bucket,objectKey).then(response => console.log(response));
Inside of your handler function.
Furthermore, your usage of s3.getObject function is incorrect. Where first argument is an object - parameters, and the second argument is a callback function.
s3.getObject(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data);
Therefore in your case, you want to modify your getS3Objects function a bit. If you want to use promises, then you can do it like this.
function getS3Objects(bucket, key) {
return new Promise((resolve, reject) => {
s3.getObject(
{
Bucket: bucket,
Key: key,
ResponseContentType: 'application/json'
},
(err, data) => {
if (err) {
reject(err);
} else {
resolve(data);
}
}
);
});
}
Another way that you can do this is as follows:
const AWS = require('aws-sdk');
const s3 = new AWS.S3();
async function readFile(Bucket, Key) {
const params = {
Bucket,
Key,
ResponseContentType: 'application/json',
};
const f = await s3.getObject(params).promise();
return f.Body.toString('utf-8');
}
readFile('mybucket', 'xyz.json').then(console.log);
I am trying to run a lambda function attached to an API gateway GET request and below is the code
const AWS = require('aws-sdk');
const s3 = new AWS.S3();
const bucketName = "dhaval-upload";
let params = {
Bucket: bucketName,
Key: event.fileName
};
exports.handler = async (event, context, callback) => {
return await s3.getObject(params).promise()
.then((res) => {
return "abcd";
// return res.Body.toString('utf-8');
})
.catch((err) => {
return err;
});
};
but I am getting the below error
errorMessage: "event is not defined"
errorType: "ReferenceError"
But I don't understand the reason for this as I have another POST request running perfectly..
Any help will be highly appreciated
You need to place params inside your handler, like this:
exports.handler = async (event, context, callback) => {
let params = {
Bucket: bucketName,
Key: event.fileName
};
return await s3.getObject(params).promise()
.then((res) => {
return "abcd";
// return res.Body.toString('utf-8');
})
.catch((err) => {
return err;
});
};
I want to download some image files from s3 bucket on my local system using Promises in node.js.
var params = {
Bucket: bucket_name',
Key: 'key'
};
var fileStream = fs.createWriteStream('path/to/file.jpg');
I tried this which is working
s3.getObject(params).createReadStream.pipe(fileStream);
But I want my code look like this
return s3.getObject(params).promise()
.then(function(data) {
//console.log(data.Body);
// No idea about this section
})
.catch(function(err) {
throw err;
});
I have to use Promise to ensure all images should be downloaded.
One possible solution is to use bluebird and create a function that returns a promise on the end of the stream:
const B = require('bluebird');
function downloadFromS3 (object) {
var p = B.Promise.defer();
var stream = s3.getObject(params).createReadStream()
stream.pipe(fileStream);
stream.on('error', (e) => p.reject(e))
stream.on('end', () => p.resolve())
return p.promise;
}
downloadFromS3(params)
.then(() => console.log('finished'))
.catch(() => console.log('failed'))
Not sure if this code specifically would work, but it may give you a direction to look into.
The below snippet worked for me;
async function getObjectStreamSync(params, dest) {
return new Promise((resolve, reject) => {
// create read stream for object
let stream = s3.getObject(params).createReadStream();
var fileStream = fs.createWriteStream(dest);
stream.pipe(fileStream);
// on error reject the Promise
stream.on('error', (err) => reject(new Error(err)));
// on end resolve the Promise
stream.on('end', () => resolve());
});
}
await getObjectStreamSync(params, "path/to/file/file.ext");
Here, wrapped the stream within Promise. And by listening to the emitted events reject/resolve the Promise.
streamToPromise = require('stream-to-promise');
var fileStream = fs.createWriteStream('path/to/file.jpg');
streamToPromise(fileStream).then(function () {
console.log('Image saved to file.');
});
s3.getObject(params).createReadStream.pipe(fileStream);
Here's a native promise solution with error detection on the read stream and on the write stream.
function streamPromise(stream) {
return new Promise((resolve, reject) => {
stream.on('end', () => {
resolve('end');
});
stream.on('finish', () => {
resolve('finish');
});
stream.on('error', (error) => {
reject(error);
});
});
}
async function s3Download(srcBucket, srcKey, outputPath) {
var objReq = s3.getObject({
Bucket: srcBucket,
Key: srcKey
});
let outStream = fs.createWriteStream(outputPath);
let readStream = objReq.createReadStream();
readStream.on('error', (err) => {
console.warn('s3download error', err);
outStream.emit("error", err);
});
readStream.pipe(outStream);
return streamPromise(outStream);
}
Here is a snippet to use async/await with NodeJS 8:
const AWS = require('aws-sdk');
const fs = require('fs-extra');
const decompress = require('decompress');
const s3 = new AWS.S3();
const s3Params = {
Bucket: s3Location.bucketName,
Key: s3Location.objectKey,
};
const s3Object = await s3.getObject(s3Params).promise();
await fs.writeFile('myfile.zip', s3Object.Body);
await decompress('myfile.zip', 'myFileDir');
/* The compressed file is retrieved as "myfile.zip".
Content will be extracted in myFileDir directory */