I have a lambda function to ingest images from an S3 bucket, get some metadata, store this to an AWS RDS instance and then re-upload the image. Should be simple but I fear one of the following is causing issues.
Sometimes creates duplicates
Sometime misses files
It seems to happen with larger sets of images. I uploaded sub-1000 assets and it seems to work reasonably well. 3000+ it seems to be unreliable. The function is not set to time out too early (30 seconds should be fine) it has good memory allocation 512MB (please tell me if these are false assumptions. I am an amateur at this and a novice with Lambda so please so let me know what you think I have done.
const AWS = require('aws-sdk')
const uuidv4 = require('uuid/v4');
AWS.config.update({
accessKeyId: 'XXX',
secretAccessKey: 'XXX'
})
const s3 = new AWS.S3({
signatureVersion: 'v4',
region: 'eu-west-2'
})
const hasha = require('hasha')
const { Pool, Client } = require('pg')
const pool = new Pool({
user: 'XXX',
host: 'XXX',
database: 'XXX',
password: 'XXX',
port: 5432,
})
exports.handler = async (event, context) => {
const bucket = event.Records[0].s3.bucket.name;
const key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
console.log("Processing: " + key)
//Get file
try {
const file = await s3.getObject({
Bucket: bucket,
Key: key
}).promise()
const hash = await hasha(file.Body, { algorithm: 'md5' })
const id = uuidv4()
newfile = await s3.putObject({
Bucket: 'XXX',
Key: id,
Body: file.Body,
ContentType: file.ContentType
}).promise()
var fileString = key.split('/')
var fileName = fileString[fileString.length - 1]
const text = 'INSERT INTO original(original_filename, mime, file_size, file_path, file_name, hash) VALUES($1, $2, $3, $4, $5, $6) RETURNING *'
const values = [fileName, file.ContentType, file.ContentLength, key, id, hash]
const res = await pool.query(text, values)
console.log(res.rows[0])
return "Done"
} catch (err) {
console.log("####### Error #######: ", err)
return "Error"
}
}
I am expecting that X numbers of files are uploaded and the same number are in the target bucket and within my DB table. This is not always the case and is tricky to unpick where it is going wrong. I am sure there is a more elegant way to do this.
Related
I want to do this with aws-sdk library.
I have a folder on my S3 bucket called "abcd/", it has 3 files on it (e.g. abcd/1.jpg, abcd/2.jpg).
I want to rename the folder to 1234/
^ I want there to be 1234/ only
const awsMove = async (path) => {
try {
const s3 = new AWS.S3();
const AWS_BUCKET = 'my-bucket-test';
const copyParams = {
Key: path.newPath,
Bucket: AWS_BUCKET,
CopySource: encodeURI(`/${AWS_BUCKET}/${path.oldPath}`),
};
await s3.copyObject(copyParams).promise();
const deleteParams = {
Key: path.oldPath,
Bucket: AWS_BUCKET,
};
await s3.deleteObject(deleteParams).promise();
} catch (err) {
console.log(err);
}
};
const changePath = { oldPath: 'abcd/', newPath: '1234/' };
awsMove(changePath);
The above code errors with "The specified key does not exist" what am I doing wrong?
AWS S3 does not have the concept of folders as in a file system. You have a bucket and a key that identifies the object/file stored at that location. The pattern of the key is usually a/b/c/d/some_file and the way it is showed on AWS console, it might give you an impression that a, b, c or d are folders but indeed they aren't.
Now, you can't change the key of an object since it is immutable. You'll have to copy the file existing at the current key to the new key and delete the file at current key.
This implies renaming a folder say folder/ is same as copying all files located at key folder/* and creating new ones at newFolder/*. The error:
The specified key does not exist
says that you've not specified the full object key during the copy from source as well as during deletion. The correct implementation would be to list all files at folder/* and copy and delete them one by one. So, your function should be doing something like this:
const awsMove = async (path) => {
try {
const s3 = new AWS.S3();
const AWS_BUCKET = 'my-bucket-test';
const listParams = {
Bucket: AWS_BUCKET,
Delimiter: '/',
Prefix: `${path.oldPath}`
}
await s3.listObjects(listParams, function (err, data) {
if(err)throw err;
data.Contents.forEach(async (elem) => {
const copyParams = {
Key: `${path.newPath}${elem.Key}`,
Bucket: AWS_BUCKET,
CopySource: encodeURI(`/${AWS_BUCKET}/${path.oldPath}/${elem.Key}`),
};
await s3.copyObject(copyParams).promise();
const deleteParams = {
Key: `${path.newPath}${elem.Key}`,
Bucket: AWS_BUCKET,
};
await s3.deleteObject(deleteParams).promise();
});
}).promise();
} catch (err) {
console.log(err);
}
};
Unfortunately, you will need to copy the old ones to the new name and delete them from the old one.
BOTO 3:
AWS_BUCKET ='my-bucket-test'
s3 = boto3.resource('s3')
s3.Object(AWS_BUCKET,'new_file').copy_from(CopySource='AWS_BUCKET/old_file')
s3.Object(AWS_BUCKET,'old_file').delete()
Node :
var s3 = new AWS.S3();
AWS_BUCKET ='my-bucket-test'
var OLD_S3_KEY = '/old-file.json';
var NEW_S3_KEY = '/new-file.json';
s3.copyObject({
Bucket: BUCKET_NAME,
CopySource: `${BUCKET_NAME}${OLD_KEY}`,
Key: NEW_KEY
})
.promise()
.then(() =>
s3.deleteObject({
Bucket: BUCKET_NAME,
Key: OLD_KEY
}).promise()
)
.catch((e) => console.error(e))
My call recordings are being pushed at S3 and stored with contactId_timestamp.wav as filename.
For now i can get/download the files by specifically providing the file name as key, now i wanted to create the filename by myself as contactId + disconnecttimestamp i can get ge contactId through getContactId() but how to get the disconnecttimestamp?
My goal is same what we are experiencing in Contact Flow Search the recordings can be played with respect to contactId.
Here is how i am downloading the recordings from S3.
require("dotenv").config();
const expres = require("express");
const app = expres();
app.listen(3001);
const aws = require("aws-sdk");
aws.config.update({
secretAccessKey: process.env.ACCESS_SECRET,
accessKeyId: process.env.ACCESS_KEY,
region: process.env.REGION
})
const BUCKET = process.env.BUCKET
const s3 = new aws.S3(secretAccessKey = process.env.ACCESS_SECRET, accessKeyId = process.env.ACCESS_KEY);
app.get("/download/filename", async(req, res)=>{
const filename = req.params.filename
let x = await s3.getObject({Bucket:BUCKET, Key:filename}).promise();
res.send(x.Body);
})
And Than hitting the http://localhost:3001/download/0989c085-16d1-478b-8858-1ccddb2990f4_20220303T16:46_UTC.wav
If you have the ContactID for the call you can use describeContact to get the contact info which includes the DisconnectTimestamp.
Something along these lines should work.
const AWS = require('aws-sdk');
aws.config.update({
secretAccessKey: process.env.ACCESS_SECRET,
accessKeyId: process.env.ACCESS_KEY,
region: process.env.REGION
})
const connect = new AWS.Connect({ region: process.env.REGION });
var params = {
ContactId: 'STRING_VALUE', /* required */
InstanceId: 'STRING_VALUE' /* required - the connect instance ID */
};
connect.describeContact(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else var DisconnectTimestamp = data.Contact.DisconnectTimestamp); // successful response
});
more info here https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Connect.html#describeContact-property
I used s3 to trigger my lambda function every time I uploaded a file in s3 bucket, I've tested it and it works but I noticed that S3 is triggering my lambda function twice with unique request id, even if I removed the entire codes and just add console.info('something') it's still trigger twice so clearly it's not lambda making retries if there's error. I also set retry to 0.
1st request id : awsRequestId: '9f73e49f-6cc7-454e-a89f-7d88122a7166'
2nd request id : awsRequestId: '1c8572d5-61ee-4b0b-93d9-4f8a3dcd28bf'
here's my code
const AWS = require('aws-sdk');
const s3 = new AWS.S3({
region: process.env.region,
accessKeyId: process.env.accessKeyId,
secretAccessKey: process.env.secretAccessKey
});
const axios = require('axios');
const csvtojson = require('csvtojson');
const _ = require('lodash');
exports.handler = async (event, context) => {
const { normalizeAndHashData } = require('./hash');
const params = {Bucket: 'myBucket', 'myKey'}
const data = await s3.getObject(params).promise();
const eventsJson = await csvtojson().fromString(data.Body.toString());
const result1 = await axios.post(`https://myurl`, eventJson);
if (result1) {
const sampleDataToUpload = {id: 1, name: 'test'}
const result2 = await axios.post(`https://myurl2`, sampleDataToUpload);
}
return context.succeed()
};
If you send it with the right key, it will be deleted. I thought it was good because DeleteMarker got true, but if you send it with a key that is not in S3, DeleteMarker true comes out. What should we do with the validation?
const s3 = new AWS.S3({
accessKeyId:awsConfig.accessKeyId,
secretAccessKey: awsConfig.secretAccessKey,
region: awsConfig.region,
});
(async function () {
const params = {
Bucket: awsConfig.bucket,
Key: '17f316af9b011f08b68538a8ac76b73f.jpg',
};
const deleteS3Data = await s3.deleteObject(params).promise();
if(deleteS3Data.DeleteMarker !== true){
throw new Error("I don't have the key.");
}else{
return deleteS3Data;
}
}())
// deleteS3Data Whatever comes out true
I'm working with lambda function to resize the upload image in my s3 bucket, I'm encountering few problems with the following code, Can anyone explain what is "const match" and "originalKey" I tried several combination . none helped me.
'use strict';
const AWS = require('aws-sdk');
const S3 = new AWS.S3({
accessKeyId: "xxxxxxxxxxxx",
secretAccessKey: "yyyyyyyyyyy",
region: "us-east-1",
signatureVersion: 'v4',
});
const Sharp = require('sharp');
const BUCKET = "patientimg";
const URL = "https://s3.ap-south-1.amazonaws.com";
exports.handler = function(event, context, callback) {
const key = event.Records[0].s3.object.key;
const match = key.match(/(\d+)x(\d+)\/(.*)/);
const width =10;
const height =10;
const originalKey ="ImageName";
S3.getObject({Bucket: BUCKET, Key: originalKey}).promise()
.then(data => Sharp(data.Body)
.resize(width, height)
.toFormat('png')
.toBuffer()
)
.then(buffer => S3.putObject({
Body: buffer,
Bucket: BUCKET,
ContentType: "image/png",
Key: key,
}).promise()
)
.then(() => callback(null, {
statusCode: '301',
headers: {'location': "${URL}/${key}"},
body: "",
})
)
.catch(err => callback(err))
}
output:
error occured in match[1], match[2] and match[3]
Thanks in advance.....
This code assumes the uploaded object key matches a specific pattern /(\d+)x(\d+)\/(.*)/, which means:
\d+ 1 or more digits - match[1]
x a literal x
\d+ 1 or more digits - match[2]
\/ a literal slash
.* zero or more of any character - match[3]
If your object key doesn't match this pattern, the code would break.
In case someone else is struggling with this. Its not a * but its an x.
So the url for starting this should be like this
http://webpageendpointtobucket/200x200/earth.jpg