AWS Lambda gives error on putting s3 object - node.js

I am working on a function which creates a thumbnail by saving a thumbnail version of the image in the screenshot folder when any image is uploaded to the images folder in the bucket. I am using serverless framework. I keep getting an error shown below. I have pasted exact code so anyone can copy paste and implement this solution. Serverless.yml, handler function file as well as any supporting file is included as well.
I can't figure out when i am referring to buffer why do i get this error that object type is not buffer etc.
{ InvalidParameterType: Expected params.Body to be a string, Buffer, Stream, Blob, or typed array object
at ParamValidator.fail (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:50:37)
at ParamValidator.validatePayload (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:255:10)
at ParamValidator.validateScalar (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:133:21)
at ParamValidator.validateMember (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:94:21)
at ParamValidator.validateStructure (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:75:14)
at ParamValidator.validateMember (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:88:21)
at ParamValidator.validate (/var/runtime/node_modules/aws-sdk/lib/param_validator.js:34:10)
at Request.VALIDATE_PARAMETERS (/var/runtime/node_modules/aws-sdk/lib/event_listeners.js:125:42)
at Request.callListeners (/var/runtime/node_modules/aws-sdk/lib/sequential_executor.js:105:20)
at callNextListener (/var/runtime/node_modules/aws-sdk/lib/sequential_executor.js:95:12)
message: 'Expected params.Body to be a string, Buffer, Stream, Blob, or typed array object',
code: 'InvalidParameterType',
time: 2019-03-12T16:37:26.910Z }
Code:
Handler.js
'use strict';
const resizer = require('./resizer');
module.exports.resizer = (event, context, callback) => {
console.log(event.Records[0].s3);
const bucket = event.Records[0].s3.bucket.name;
const key = event.Records[0].s3.object.key;
console.log(`A file named ${key} was put in a bucket ${bucket}`);
resizer(bucket, key)
.then(() => {
console.log(`The thumbnail was created`);
callback(null, {
message: 'The thumbnail was created'
});
})
.catch(error => {
console.log(error);
callback(error);
});
};
module.exports.thumbnails = (event, context, callback) => {
const bucket = event.Records[0].s3.bucket.name;
const key = event.Records[0].s3.object.key;
console.log(bucket);
console.log(key);
console.log(`A new file ${key} was created in the bucket ${bucket}`);
callback(null, {
message: `A new file ${key} was created in the bucket ${bucket}`
});
};
Resizer.js
'use strict';
const AWS = require('aws-sdk');
const S3 = new AWS.S3();
const Jimp = require('jimp'); //https://github.com/oliver-moran/jimp
module.exports = (bucket, key) => {
const newKey = replacePrefix(key);
const height = 512;
return getS3Object(bucket, key).then(data => resizer(data.Body, height)).then(buffer => putS3Object(bucket, newKey, buffer));
};
function getS3Object(bucket, key) {
return S3.getObject({
Bucket: bucket,
Key: key
}).promise();
}
function putS3Object(bucket, key, body) {
return S3.putObject({
Body: body,
Bucket: bucket,
ContentType: 'image/jpg',
Key: key
}).promise();
}
function replacePrefix(key) {
const uploadPrefix = 'uploads/';
const thumbnailsPrefix = 'thumbnails/';
return key.replace(uploadPrefix, thumbnailsPrefix);
}
function resizer(data, height) {
return Jimp.read(data)
.then(image => {
return image
.resize(Jimp.AUTO, height)
.quality(100) // set JPEG quality
.getBuffer(Jimp.MIME_JPEG, (err, buffer) => {
return buffer;
});
})
.catch(err => err);
}
Serverless.yml
service: serverless-resizer-project # NOTE: update this with your service name
provider:
name: aws
runtime: nodejs6.10
profile: student1
iamRoleStatements:
- Effect: "Allow"
Action:
- "s3:ListBucket"
- "s3:GetObject"
- "s3:PutObject"
Resource: "arn:aws:s3:::serverless-resizer-project-images/*"
functions:
resizer:
handler: handler.resizer
events:
- s3:
bucket: serverless-resizer-project-images
event: s3:ObjectCreated:*
rules:
- prefix: uploads/
- suffix: .jpg
thumbnails:
handler: handler.thumbnails
events:
- s3:
bucket: serverless-resizer-project-images
event: s3:ObjectCreated:*
rules:
- prefix: thumbnails/
- suffix: .jpg

The return value of your resizer function is not what you expect. You're using the getBuffer function with a callback, which means that the buffer of the image is not resolved by the promise, but instead is used in the callback, which is not your intention. You should instead use getBufferAsync, which returns a promise that resolves to the image buffer. Your resizer function should look something like this:
function resizer(data, height) {
return Jimp.read(data)
.then(image => image
.resize(Jimp.AUTO, height)
.quality(100) // set JPEG quality
.getBufferAsync(Jimp.MIME_JPEG)
)
.catch(err => err);
}

Related

how do I rename a folder?

I want to do this with aws-sdk library.
I have a folder on my S3 bucket called "abcd/", it has 3 files on it (e.g. abcd/1.jpg, abcd/2.jpg).
I want to rename the folder to 1234/
^ I want there to be 1234/ only
const awsMove = async (path) => {
try {
const s3 = new AWS.S3();
const AWS_BUCKET = 'my-bucket-test';
const copyParams = {
Key: path.newPath,
Bucket: AWS_BUCKET,
CopySource: encodeURI(`/${AWS_BUCKET}/${path.oldPath}`),
};
await s3.copyObject(copyParams).promise();
const deleteParams = {
Key: path.oldPath,
Bucket: AWS_BUCKET,
};
await s3.deleteObject(deleteParams).promise();
} catch (err) {
console.log(err);
}
};
const changePath = { oldPath: 'abcd/', newPath: '1234/' };
awsMove(changePath);
The above code errors with "The specified key does not exist" what am I doing wrong?
AWS S3 does not have the concept of folders as in a file system. You have a bucket and a key that identifies the object/file stored at that location. The pattern of the key is usually a/b/c/d/some_file and the way it is showed on AWS console, it might give you an impression that a, b, c or d are folders but indeed they aren't.
Now, you can't change the key of an object since it is immutable. You'll have to copy the file existing at the current key to the new key and delete the file at current key.
This implies renaming a folder say folder/ is same as copying all files located at key folder/* and creating new ones at newFolder/*. The error:
The specified key does not exist
says that you've not specified the full object key during the copy from source as well as during deletion. The correct implementation would be to list all files at folder/* and copy and delete them one by one. So, your function should be doing something like this:
const awsMove = async (path) => {
try {
const s3 = new AWS.S3();
const AWS_BUCKET = 'my-bucket-test';
const listParams = {
Bucket: AWS_BUCKET,
Delimiter: '/',
Prefix: `${path.oldPath}`
}
await s3.listObjects(listParams, function (err, data) {
if(err)throw err;
data.Contents.forEach(async (elem) => {
const copyParams = {
Key: `${path.newPath}${elem.Key}`,
Bucket: AWS_BUCKET,
CopySource: encodeURI(`/${AWS_BUCKET}/${path.oldPath}/${elem.Key}`),
};
await s3.copyObject(copyParams).promise();
const deleteParams = {
Key: `${path.newPath}${elem.Key}`,
Bucket: AWS_BUCKET,
};
await s3.deleteObject(deleteParams).promise();
});
}).promise();
} catch (err) {
console.log(err);
}
};
Unfortunately, you will need to copy the old ones to the new name and delete them from the old one.
BOTO 3:
AWS_BUCKET ='my-bucket-test'
s3 = boto3.resource('s3')
s3.Object(AWS_BUCKET,'new_file').copy_from(CopySource='AWS_BUCKET/old_file')
s3.Object(AWS_BUCKET,'old_file').delete()
Node :
var s3 = new AWS.S3();
AWS_BUCKET ='my-bucket-test'
var OLD_S3_KEY = '/old-file.json';
var NEW_S3_KEY = '/new-file.json';
s3.copyObject({
Bucket: BUCKET_NAME,
CopySource: `${BUCKET_NAME}${OLD_KEY}`,
Key: NEW_KEY
})
.promise()
.then(() =>
s3.deleteObject({
Bucket: BUCKET_NAME,
Key: OLD_KEY
}).promise()
)
.catch((e) => console.error(e))

NodeJS - Identify file type when getObject from Amazon S3

I have a S3 Bucket with many type of files (since images to PDF).
I need to identify the file type when I am using s3.getObject to return it appropriately.
I have this code:
module.exports.getAttachedFile = async (fileName) => {
try {
let s3 = new AWS.S3({
accessKeyId: 'my_access_key',
secretAccessKey: 'my_secret_key'
});
let params = {
Bucket: 'attached-files',
Key: fileName
};
const data = await s3.getObject(params).promise();
return data.Body;
} catch (e) {
console.log(e);
return e;
}
}
You can get the MIME type by calling data.ContentType.
Here you can see all the attributes of the GetObject Response.

AWS lambda never completes but doesn't appear to timeout either

I'm attempting to create a simple application. A user emails an attachment to a special inbox, AWS SES gets the email and stores it in S3, a lambda function is triggered and the lambda finds the email in S3, parses out the attachment (in this case a jpg) and then stores it in a different S3 bucket. Finally, the application creates a new row in an Airtable with the image as an attachment.
When I invoked this function locally using serverless, everything works fine. The email with the image are already stored in an S3 bucket so I've created a mock which passes the key explicitly to my lambda. However, when I deploy the application and send a test email, the following happens:
Email is stored in S3 bucket 'to-airtable-temp'
Lambda function is called
Before the email can be found and the attachment striped off and stored in a seperate S3 bucket the function just stops. No error message or timeout. It just stops. Cloudwatch logs look like the following:
START RequestId: 6a0364ae-0879-4ee1-9dcd-c8747de1a650 Version: $LATEST
2020-02-17T07:39:55.465Z 6a0364ae-0879-4ee1-9dcd-c8747de1a650 INFO {
s3SchemaVersion: '1.0',
configurationId: 'emailtoairtable-dev-parse-224c3963d3a3f9c35018ae93a9fffea4',
bucket: {
name: 'to-airtable-temp',
ownerIdentity: { principalId: 'AI5RGHFD5AFHE' },
arn: 'arn:aws:s3:::to-airtable-temp'
},
object: {
key: 'mtvuuiokqj55l2a8b0qser7tn9dhfignoh9c1vg1',
size: 3804230,
eTag: 'c821fb0a2a9c3b060e20e7d177f8b972',
sequencer: '005E4A434810147365'
}
}
2020-02-17T07:39:55.465Z 6a0364ae-0879-4ee1-9dcd-c8747de1a650 INFO key mtvuuiokqj55l2a8b0qser7tn9dhfignoh9c1vg1
2020-02-17T07:39:55.465Z 6a0364ae-0879-4ee1-9dcd-c8747de1a650 INFO Key pushed to list. mtvuuiokqj55l2a8b0qser7tn9dhfignoh9c1vg1
END RequestId: 6a0364ae-0879-4ee1-9dcd-c8747de1a650
REPORT RequestId: 6a0364ae-0879-4ee1-9dcd-c8747de1a650 Duration: 1113.17 ms Billed Duration: 1200 ms Memory Size: 1024 MB Max Memory Used: 114 MB Init Duration: 119.56 ms
Here is my handler.js file:
'use strict';
module.exports.parse = async event => {
try {
const aws = require('aws-sdk');
const s3 = new aws.S3();
const simpleParser = require('mailparser').simpleParser;
const Airtable = require('airtable');
const dateformat = require('dateformat');
var base = new Airtable({ apiKey: process.env.airtableApiKey}).base(process.env.airtableBaseId);
var data = [];
var keys = [];
event["Records"].forEach(async record => {
console.log(record["s3"]);
console.log('key', record["s3"]["object"]["key"]);
keys.push(record["s3"]["object"]["key"]);
console.log('Key pushed to list. ', record["s3"]["object"]["key"]); // <-- this is the last line that I am sure processes because I see it in the CloudWatch logs.
var temp_data = await s3.getObject(
{
Bucket: 'to-airtable-temp',
Key: record["s3"]["object"]["key"]
}).promise();
console.log('temp_data', temp_data);
data.push(temp_data);
});
setTimeout( async function() {
// console.log('data', data[0].Body.toString());
let parsed = await simpleParser(data[0].Body.toString());
console.log(parsed);
// save the file to a public S3 bucket so it can be uploaded to airtable
parsed["attachments"].forEach(function(attachment) {
let now = new Date();
s3.upload({
Bucket: 'to-airtable-images',
Key: keys[0] + dateformat(now, "yyyy-mm-dd") + '.jpg',
Body: attachment.content,
ContentType: "image/jpeg"
},
function(error, data) {
if (error) {
throw error;
}
console.log('File uploaded successfully. ' +data.Location);
// Now upload to airtable
base('Input').create([
{
"fields": {"Attachments": [
{
"url": data.Location
}
]}
}
], function(err, records) {
if (err) {
console.error(err);
return;
}
records.forEach(function (record) {
console.log(record.getId());
});
});
});
});
return {
statusCode: 200,
body: JSON.stringify(
{
message: 'Go Serverless v1.0! Your function executed successfully!',
input: event,
data: JSON.stringify(data),
},
null,
2
),
};
}, 500); // I've tried increasing this time but it still hangs.
} catch (error) {
console.error(error);
}
};
You shouldn't use async/await with the forEach function. Using async/await with a forEach loop. Instead, use the more modern for of syntax:
for (let record of event["Records"]) {
// you can include await calls in this block
}

Serverless lambda trigger read json file

I have lambda (Node) which has trigger to fire when a new JSON file added to our S3 bucket. Here is my lambda code
module.exports.bookInfo = (event, context) => {
console.log('Events ', JSON.stringify(event));
event.Records.forEach((record) => {
const filename = record.s3.object.key;
const bucketname = record.s3.bucket.name;
let logMsg = [];
const s3File = `BucketName: [${bucketname}] FileName: [${filename}]`;
console.log(s3File)
logMsg.push(`Lambda execution started for ${s3File}, Trying to download file from S3`);
try {
s3.getObject({
Bucket: bucketname,
Key: filename
}, function(err, data) {
logMsg.push('Data is ', JSON.stringify(data.Body))
if (err) {
logMsg.push('Generate Error :', err);
console.log(logMsg)
return null;
}
logMsg.push(`File downloaded successfully. Processing started for ${s3File}`);
logMsg.push('Data is ', JSON.stringify(data.Body))
});
} catch (e) {console.log(e)}
});
}
When i run this, i don't get file content and i suspect that lambda finishes execution before file read operation complete. I tried with async await without success. What i am missing here ? I was able to read small file of 1 kb but when my file grows like 100 MB, it causes issue.
Thanks in advance
I was able to do it through async/await. Here is my code
module.exports.bookInfo = (event, context) => {
event.Records.forEach(async(record) => {
const filename = record.s3.object.key;
const bucketname = record.s3.bucket.name;
const s3File = `BucketName: [${bucketname}] FileName: [${filename}]`;
logMsg.push(`Lambda execution started for ${s3File}, Trying to download file from S3`);
let response = await s3.getObject({
Bucket: bucketname,
Key: filename
}).promise();
})
}

Javascript AWS SDK S3 upload method with Body stream generating empty file

I'm trying to use the method upload from s3 using a ReadableStream from the module fs.
The documentation says that a ReadableStream can be used at Bodyparam:
Body — (Buffer, Typed Array, Blob, String, ReadableStream) Object data.
Also the upload method description is:
Uploads an arbitrarily sized buffer, blob, or stream, using intelligent concurrent handling of parts if the payload is large enough.
Also, here: Upload pdf generated to AWS S3 using nodejs aws sdk the #shivendra says he can use a ReadableStream and it works.
This is my code:
const fs = require('fs')
const S3 = require('aws-sdk/clients/s3')
const s3 = new S3()
const send = async () => {
const rs = fs.createReadStream('/home/osman/Downloads/input.txt')
rs.on('open', () => {
console.log('OPEN')
})
rs.on('end', () => {
console.log('END')
})
rs.on('close', () => {
console.log('CLOSE')
})
rs.on('data', (chunk) => {
console.log('DATA: ', chunk)
})
console.log('START UPLOAD')
const response = await s3.upload({
Bucket: 'test-bucket',
Key: 'output.txt',
Body: rs,
}).promise()
console.log('response:')
console.log(response)
}
send().catch(err => { console.log(err) })
It's getting this output:
START UPLOAD
OPEN
DATA: <Buffer 73 6f 6d 65 74 68 69 6e 67>
END
CLOSE
response:
{ ETag: '"d41d8cd98f00b204e9800998ecf8427e"',
Location: 'https://test-bucket.s3.amazonaws.com/output.txt',
key: 'output.txt',
Key: 'output.txt',
Bucket: 'test-bucket' }
The problem is that my file generated at S3 (output.txt) has 0 Bytes.
Someone know what am I doing wrong?
If I pass a buffer on Body it works.
Body: Buffer.alloc(8 * 1024 * 1024, 'something'),
But it's not what I want to do. I'd like to do this using a stream to generate a file and pipe a stream to S3 as long as I generate it.
It's an API interface issue using NodeJS ReadableStreams.
Just comment the code related to listen event 'data', solves the problem.
const fs = require('fs')
const S3 = require('aws-sdk/clients/s3')
const s3 = new S3()
const send = async () => {
const rs = fs.createReadStream('/home/osman/Downloads/input.txt')
rs.on('open', () => {
console.log('OPEN')
})
rs.on('end', () => {
console.log('END')
})
rs.on('close', () => {
console.log('CLOSE')
})
// rs.on('data', (chunk) => {
// console.log('DATA: ', chunk)
// })
console.log('START UPLOAD')
const response = await s3.upload({
Bucket: 'test-bucket',
Key: 'output.txt',
Body: rs,
}).promise()
console.log('response:')
console.log(response)
}
send().catch(err => { console.log(err) })
Though it's an strange API, when we listen to 'data' event, the ReadableStream starts the flowing mode (listening to an event changing publisher/EventEmitter state? Yes, very error prone...). For some reason the S3 need a paused ReadableStream. If whe put rs.on('data'...) after await s3.upload(...) it works. If we put rs.pause() after rs.on('data'...) and befote await s3.upload(...), it works too.
Now, what does it happen? I don't know yet...
But the problem was solved, even it isn't completely explained.
Check if file /home/osman/Downloads/input.txt actually exists and accessible by node.js process
Consider to use putObject method
Example:
const fs = require('fs');
const S3 = require('aws-sdk/clients/s3');
const s3 = new S3();
s3.putObject({
Bucket: 'test-bucket',
Key: 'output.txt',
Body: fs.createReadStream('/home/osman/Downloads/input.txt'),
}, (err, response) => {
if (err) {
throw err;
}
console.log('response:')
console.log(response)
});
Not sure how this will work with async .. await, better to make upload to AWS:S3 work first, then change the flow.
UPDATE:
Try to implement upload directly via ManagedUpload
const fs = require('fs');
const S3 = require('aws-sdk/clients/s3');
const s3 = new S3();
const upload = new S3.ManagedUpload({
service: s3,
params: {
Bucket: 'test-bucket',
Key: 'output.txt',
Body: fs.createReadStream('/home/osman/Downloads/input.txt')
}
});
upload.send((err, response) => {
if (err) {
throw err;
}
console.log('response:')
console.log(response)
});

Resources