Kindly excuse my knowledge with NodeJs, as I've just started with it. I've following lambda function which isn't fetching list of objects (more than 1000) in S3 and stuck in infinite loop, resulting lambda timimg out. Not sure what's wrong here
Code:
console.log('Loading');
const AWS = require('aws-sdk');
var request=true;
const awsOptions = {
region: "us-east-1"
};
const s3 = new AWS.S3(awsOptions);
var list = [];
exports.handler = async (event, context, callback) => {
const SrcBucket = event.Records[0].s3.bucket.name;
const trigger_file = event.Records[0].s3.object.key;
var bucketParams = {
Bucket: SrcBucket,
Prefix: 'Test/'
};
do
{
s3.listObjects(bucketParams, (err, data) => {
if (err)
console.log("Error", err);
else
{
list.push(data.Contents);
if (data.IsTruncated)
bucketParams.Marker = data.NextMarker;
else
request = false;
}
});
} while (request);
callback(null, {
listLen: list.length
});
Related
#TBA gave the solution.
The root cause is not by runtime. It came from SDK v3.
Point: Do not update the code with mixed things (like both of runtime & SDK version together 🥲)
Thanks again, TBA.
I was using Node.js 14.x version runtime Lambda to read some json file from S3.
Brief code is below
const AWS = require("aws-sdk");
const s3 = new AWS.S3();
exports.handler = (event) => {
const { bucketName, objKey } = event
const params = {
Bucket: bucketName,
Key: objKey
};
return new Promise((resolve) => {
s3.getObject(params, async (err, data) =>{
if (err) console.log(err, err.stack);
else {
const contents = JSON.parse(data.Body)
resolve(contents);
}
});
})
};
and it returned the json data as I expected.
And today I tried to create a new lambda with runtime Node.js 18.x but it returned null or some errors...
Q) Could you give me some advice to solve this 🥲 ?
+) I used same json file for each lambda
+) Not sure why, but in my case, data.Body.toString() didn't work (I saw some answers in stackoverflow provide that and tried but no lucks)
Thanks in advance!
Case A (returns null)
import { S3Client, GetObjectCommand } from "#aws-sdk/client-s3";
const s3Client = new S3Client({ region: "ap-northeast-2" });
export const handler = (event) => {
const { objKey, bucketName } = event;
const params={
Bucket: bucketName,
Key: objKey
};
const getObjCommand = new GetObjectCommand(params);
return new Promise((resolve) => {
s3Client.send(getObjCommand, async (err, data) =>{
if (err) console.log(err, err.stack);
else {
const list = JSON.parse(data.Body)
resolve(list);
}
});
})
};
Case B (returns "Unexpected token o in JSON at position 1")
export const handler = async (event) => {
const { objKey, bucketName } = event;
const params={
Bucket: bucketName,
Key: objKey
};
const getObjCommand = new GetObjectCommand(params);
const response = await s3Client.send(getObjCommand)
console.log("JSON.parse(response.Body)", JSON.parse(response.Body))
};
Case C (returns "TypeError: Converting circular structure to JSON")
export const handler = async (event) => {
const { objKey, bucketName } = event;
const params={
Bucket: bucketName,
Key: objKey
};
const getObjCommand = new GetObjectCommand(params);
try {
const response = await s3Client.send(getObjCommand)
return JSON.stringify(response.Body)
} catch(err) {
console.log("error", err)
return err
}
};
I am not very familiar with node and trying to upload an array of media objects to an s3 bucket using an AWS Lambda node function.
the payload has an album which is an array of key/data dictionaries. My code is as below but I'm certain this is wrong.
const awsServerlessExpress = require('aws-serverless-express');
const app = require('./app');
const server = awsServerlessExpress.createServer(app);
const AWS = require("aws-sdk");
const docClient = new AWS.DynamoDB.DocumentClient();
var s3 = new AWS.S3();
var s3Params = {
Bucket: 'bucketid',
ContentEncoding: 'base64',
ContentType: 'image/jpeg'
};
exports.handler = async (event, context) => {
console.log(event);
var body = JSON.parse(event.body);
if (typeof body.album !== 'undefined' && body.album) {
body.album.forEach(function (value) {
var data = body.album.mediaString;
let mediaData = new Buffer(data, 'base64');
var mediaKey = body.album.mediaKey;
try {
s3Params = {
Bucket: 'bucketID',
Key: mediaKey,
Body: mediaData
};
try {
const stored = await s3.upload(s3Params).promise();
console.log("stored successfully");
return { body: JSON.stringify(data) };
} catch (err) {
console.log("error storing");
console.log(err);
return { error: err };
}
} catch (err) {
return { error: err };
}
});
return { body: JSON.stringify(data) };
} else {
return { error: 'error'};
}
};
I have an error that s3 not found. Just wondering if I'm going about this all wrong.
When I only upload one image with the following code everything works fine:
const awsServerlessExpress = require('aws-serverless-express');
const app = require('./app');
const server = awsServerlessExpress.createServer(app);
const AWS = require("aws-sdk");
const docClient = new AWS.DynamoDB.DocumentClient();
var s3 = new AWS.S3();
var s3Params = {
Bucket: 'bucketID',
ContentEncoding: 'base64',
ContentType: 'image/jpeg'
};
exports.handler = async (event, context) => {
var body = JSON.parse(event.body);
var data = body.mediaString;
let mediaData = new Buffer(data, 'base64');
var mediaKey = body.mediaKey;
try {
s3Params = {
Bucket: 'bucketID',
Key: mediaKey,
Body: mediaData
};
try {
const stored = await s3.upload(s3Params).promise();
console.log("stored successfully");
return { body: JSON.stringify(data) };
} catch (err) {
console.log("error storing");
console.log(err);
return { error: err };
}
} catch (err) {
return { error: err };
}
};
I tested on my localhost, then checked on s3 and saw that there was a new file created.
But when testing on Lambda, although there is no error, there is no file on S3. The log of s3.upload(params).promise() is also not displayed.
var fs = require('fs');
var AWS = require('aws-sdk');
exports.handler = async (event, context, callback) => {
context.callbackWaitsForEmptyEventLoop = false
try {
AWS.config.update({
accessKeyId: accessKeyId,
secretAccessKey: secretAccessKey
});
var s3 = new AWS.S3();
var path = 'myfile.txt';
var file_buffer = fs.readFileSync(path);
console.log(file_buffer);
var params = {
Bucket: 'bucket-dev',
Key: '2222.txt',
Body: file_buffer
};
console.log("1111");
s3.upload(params).promise()
.then(function(data) {
console.log("Successfully uploaded to");
callback(null, "All Good");
})
.catch(function(err) {
console.error(err, err.stack);
callback(err);
});
console.log("2222");
return context.logStreamName
} catch (err) {
console.log(err);
callback(err);
}
}
Thanks
Try not to mix and match async and callback. Something like this might be closer to what you want...
var fs = require("fs");
var AWS = require("aws-sdk");
exports.handler = async (event, context) => {
AWS.config.update({
accessKeyId,
secretAccessKey,
});
const s3 = new AWS.S3();
const path = "myfile.txt";
const file_buffer = fs.readFileSync(path);
const params = {
Bucket: "bucket-dev",
Key: "2222.txt",
Body: file_buffer,
};
console.log("1111");
const res = await s3.upload(params).promise();
console.log("Successfully uploaded", res);
return "All good";
};
I've following lambda function for fetching list of all the keys from S3, however, it ends up in error message as s3.listObjectsV2 is not a function. Not sure what's wrong. The code is taken from another SO post, however it doesn't seem to be working.
Code:
const AWS = require('aws-sdk');
var request=true;
const awsOptions = {
region: "us-east-1"
};
const s3 = new AWS.S3(awsOptions);
exports.handler = async (event, context, callback) => {
const SrcBucket = event.Records[0].s3.bucket.name;
const trigger_file = event.Records[0].s3.object.key;
var bucketParams = {
Bucket: SrcBucket,
Prefix: 'Temp/',
};
var allKeys = [];
listAllKeys();
function listAllKeys()
{
s3.listObjectsV2(bucketParams, function (err, data)
{
if (err)
{
console.log(err, err.stack); // an error occurred
}
else
{
var contents = data.Contents;
contents.forEach(function (content) {
allKeys.push(content.Key);
});
if (data.IsTruncated) {
bucketParams.ContinuationToken = data.NextContinuationToken;
console.log("get further list...");
listAllKeys();
}
}
});
}
console.log(allKeys.length);
}
What i'm trying to do here is to retrieve a basic CSV file in a S3 Bucket but i'm having hard time getting it.
I try first to only get the data from the bucket in a basic lambda function (not linked to an alexa skill) like that :
console.log('Loading function');
const aws = require('aws-sdk');
const s3 = new aws.S3({ apiVersion: '2006-03-01' });
exports.handler = async (event, context) => {
const bucket = 'mybucket';
const key = 'myfile';
const params = {
Bucket: bucket,
Key: key,
};
try {
const data = await s3.getObject(params).promise();
var content = data.Body.toString();
var lines = content.split('\r\n');
var headers = lines[0].split(',');
var result = {};
for(var i=1;i<lines.length;i++){
var currentline = lines[i].split(',');
var obj2 = {};
for(var j=2;j<headers.length;j++){
obj2[headers[j]] = currentline[j];
}
result[currentline[0]+currentline[1]] = obj2;
}
return result;
} catch (err) {
console.log(err);
const message = `Error getting object ${key} from bucket ${bucket}. Make sure they exist and your bucket is in the same region as this function.`;
console.log(message);
throw new Error(message);
}
};
And it worked perfectly but then i tried to put it inside a lambda function linked to my alexa skill and i'm not receiving anything from the getObject function :
const Alexa = require('alexa-sdk');
const aws = require('aws-sdk');
const s3 = new aws.S3({ apiVersion: '2006-03-01' });
var data;
function getData(){
// Setting the bucket and files parameters
const bucket = 'mybucket';
const key = 'myfile';
const params = {
Bucket: bucket,
Key: key,
};
try {
const data = s3.getObject(params);
// Getting the Body of the response
var content = data.Body.toString();
// Splitting into a proper data structure
var result = {};
var lines = content.split("\r\n");
var headers = lines[0].replace('\r','').split(",");
for(var i=1;i<lines.length;i++){
var currentline = lines[i].split(",");
var obj2 = {};
for(var j=2;j<headers.length;j++){
obj2[headers[j]] = currentline[j];
}
result[currentline[0].toLowerCase()+currentline[1].toLowerCase()] = obj2;
}
} catch (err) {
console.log(err);
const message = `Error getting object ${key} from bucket ${bucket}.`;
console.log(message);
throw new Error(message);
}
// Sending the response back
return result;
}
const handlers = {
'LaunchRequest': function () {
// GETTING THE DATA SET
data = getData();
// Stuff here
this.emit(':responseReady');
},
'getLocalisation': function () {
//Stuff here
},
'AMAZON.HelpIntent': function () {
this.response.speak(messages.HELP);
this.emit(':responseReady');
},
'AMAZON.CancelIntent': function () {
this.response.speak(messages.STOP);
this.emit(':responseReady');
},
'AMAZON.StopIntent': function () {
this.emit(':tell', 'Bye');
},
'AMAZON.FallbackIntent': function () {
this.response.speak(messages.ERROR);
this.emit(':responseReady');
},
};
exports.handler = (event, context, callback) => {
const alexa = Alexa.handler(event, context, callback);
alexa.APP_ID = APP_ID;
alexa.registerHandlers(handlers);
alexa.execute();
};
I'm running everything directly from the AWS Lambda IDE.
I think i'm missing something with Async/Await/Promise thing but i don't truely understand it.
I just had to put
async function getData()
and
exports.handler = async (event, context, callback) => {
// code here
};