I am trying to read the csv file from S3 in Lambda using NodeJS.
const AWS = require('aws-sdk');
const s3 = new AWS.S3();
const csv = require('csv-parser');
const bucket = '';
const objectkey = '';
const params = { Bucket: bucket, Key: objectkey }
const results = [];
exports.handler = async function (event) {
try {
const file = s3.getObject(params).createReadStream();
file
.pipe(csv())
.on('data', function (data) {
results.push(data)
})
.on('end', () => {
console.log(results);
})
} catch (err) {
console.log(err);
}
}
I am getting the result as null instead of csv data
you did not return the data at any point in your code. you need to call .succeed() once done.
const AWS = require('aws-sdk');
const s3 = new AWS.S3();
const csv = require('csv-parser');
const bucket = '';
const objectkey = '';
const params = { Bucket: bucket, Key: objectkey }
const results = [];
exports.handler = async function (event, ctx, callback) {
try {
const file = s3.getObject(params).createReadStream();
file
.pipe(csv())
.on('data', function (data) {
results.push(data)
})
.on('end', () => {
console.log(results);
callback(null, results);
})
} catch (err) {
console.log(err);
callback(Error(err))
}
}
Related
I am not very familiar with node and trying to upload an array of media objects to an s3 bucket using an AWS Lambda node function.
the payload has an album which is an array of key/data dictionaries. My code is as below but I'm certain this is wrong.
const awsServerlessExpress = require('aws-serverless-express');
const app = require('./app');
const server = awsServerlessExpress.createServer(app);
const AWS = require("aws-sdk");
const docClient = new AWS.DynamoDB.DocumentClient();
var s3 = new AWS.S3();
var s3Params = {
Bucket: 'bucketid',
ContentEncoding: 'base64',
ContentType: 'image/jpeg'
};
exports.handler = async (event, context) => {
console.log(event);
var body = JSON.parse(event.body);
if (typeof body.album !== 'undefined' && body.album) {
body.album.forEach(function (value) {
var data = body.album.mediaString;
let mediaData = new Buffer(data, 'base64');
var mediaKey = body.album.mediaKey;
try {
s3Params = {
Bucket: 'bucketID',
Key: mediaKey,
Body: mediaData
};
try {
const stored = await s3.upload(s3Params).promise();
console.log("stored successfully");
return { body: JSON.stringify(data) };
} catch (err) {
console.log("error storing");
console.log(err);
return { error: err };
}
} catch (err) {
return { error: err };
}
});
return { body: JSON.stringify(data) };
} else {
return { error: 'error'};
}
};
I have an error that s3 not found. Just wondering if I'm going about this all wrong.
When I only upload one image with the following code everything works fine:
const awsServerlessExpress = require('aws-serverless-express');
const app = require('./app');
const server = awsServerlessExpress.createServer(app);
const AWS = require("aws-sdk");
const docClient = new AWS.DynamoDB.DocumentClient();
var s3 = new AWS.S3();
var s3Params = {
Bucket: 'bucketID',
ContentEncoding: 'base64',
ContentType: 'image/jpeg'
};
exports.handler = async (event, context) => {
var body = JSON.parse(event.body);
var data = body.mediaString;
let mediaData = new Buffer(data, 'base64');
var mediaKey = body.mediaKey;
try {
s3Params = {
Bucket: 'bucketID',
Key: mediaKey,
Body: mediaData
};
try {
const stored = await s3.upload(s3Params).promise();
console.log("stored successfully");
return { body: JSON.stringify(data) };
} catch (err) {
console.log("error storing");
console.log(err);
return { error: err };
}
} catch (err) {
return { error: err };
}
};
Runtime env is Node 14 (AWS Lambda function). The S3 bucket and Lambda function are in the same region, and the I have confirmed that the Lambda function is able to get the object from S3 (i.e. permissions does not seem to be an issue). The Lambda triggers upon PUT of an object (a very simple CSV file) into the S3 bucket. No errors or exceptions appear in the CloudWatch log stream.
package.json
{
"dependencies": {
"#fast-csv/parse": "4.3.6"
}
}
index.js
const aws = require('aws-sdk');
const s3 = new aws.S3({region: 'us-east-2'});
const fs = require('fs');
const csv = require('#fast-csv/parse');
exports.handler = async (event, context) => {
const bucket = event.Records[0].s3.bucket.name;
const key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
const params = {
Bucket: bucket,
Key: key,
};
const file = await s3.getObject(params).promise();
fs.createReadStream(file.Body).pipe(csv.parse())
.on('error', (error) => console.error(error))
.on('data', (row) => console.log(row))
.on('end', (rowCount) => console.log(`Parsed ${rowCount} rows`));
};
I also tried the following variation, which had the same outcome:
index.js (variant)
const aws = require('aws-sdk');
const s3 = new aws.S3({region: 'us-east-2'});
const fs = require('fs');
const csv = require('fast-csv');
exports.handler = async (event, context) => {
const bucket = event.Records[0].s3.bucket.name;
const key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
const params = {
Bucket: bucket,
Key: key,
};
const file = await s3.getObject(params).promise();
const stream = fs.createReadStream(file.Body);
csv.parseStream(stream)
.on('data', (data) => {
console.info('Data: ' + JSON.stringify(data));
})
.on('data-invalid', (data) => {
console.error('Invalid batch row ' + data);
})
.on('end', () => {
console.info('End of Stream');
})
.on('error', (error) => {
let message = "Error in csv stream processing";
console.error(message, ":", error);
}
);
};
Note: I already tried simply doing await s3.getObject(params).createReadStream(), but that results in undefined whereas getting the object via promise() first gets the object data.
I've been wrestling with this for hours, so any help is appreciated. Thanks!
Kindly excuse my knowledge with NodeJs, as I've just started with it. I've following lambda function which isn't fetching list of objects (more than 1000) in S3 and stuck in infinite loop, resulting lambda timimg out. Not sure what's wrong here
Code:
console.log('Loading');
const AWS = require('aws-sdk');
var request=true;
const awsOptions = {
region: "us-east-1"
};
const s3 = new AWS.S3(awsOptions);
var list = [];
exports.handler = async (event, context, callback) => {
const SrcBucket = event.Records[0].s3.bucket.name;
const trigger_file = event.Records[0].s3.object.key;
var bucketParams = {
Bucket: SrcBucket,
Prefix: 'Test/'
};
do
{
s3.listObjects(bucketParams, (err, data) => {
if (err)
console.log("Error", err);
else
{
list.push(data.Contents);
if (data.IsTruncated)
bucketParams.Marker = data.NextMarker;
else
request = false;
}
});
} while (request);
callback(null, {
listLen: list.length
});
I've following lambda function for fetching list of all the keys from S3, however, it ends up in error message as s3.listObjectsV2 is not a function. Not sure what's wrong. The code is taken from another SO post, however it doesn't seem to be working.
Code:
const AWS = require('aws-sdk');
var request=true;
const awsOptions = {
region: "us-east-1"
};
const s3 = new AWS.S3(awsOptions);
exports.handler = async (event, context, callback) => {
const SrcBucket = event.Records[0].s3.bucket.name;
const trigger_file = event.Records[0].s3.object.key;
var bucketParams = {
Bucket: SrcBucket,
Prefix: 'Temp/',
};
var allKeys = [];
listAllKeys();
function listAllKeys()
{
s3.listObjectsV2(bucketParams, function (err, data)
{
if (err)
{
console.log(err, err.stack); // an error occurred
}
else
{
var contents = data.Contents;
contents.forEach(function (content) {
allKeys.push(content.Key);
});
if (data.IsTruncated) {
bucketParams.ContinuationToken = data.NextContinuationToken;
console.log("get further list...");
listAllKeys();
}
}
});
}
console.log(allKeys.length);
}
I have the following code below, which is a lambda function to get content from a s3Object zip file. I know for a fact that I am not resolving the list of promises and need a little direction on how to resolve. I have read several codes on here but having a hard time applying it to my code. Any assistance would be greatly appreciated.
// dependencies
const AWS = require('aws-sdk');
var JSZip = require('jszip');
// get reference to S3 client
const s3 = new AWS.S3();
exports.handler = async (event, context, callback) => {
// Read options from the event parameter.
const srcBucket = event.Records[0].s3.bucket.name;
// Object key may have spaces or unicode non-ASCII characters.
const srcKey = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " "));
// Download the file from the S3 source bucket.
try {
const params = {
Bucket: srcBucket,
Key: srcKey
};
const data = await s3.getObject(params).promise();
var zip = JSZip.loadAsync(data.Body).then(function (content){
return content;
});
zip.then(function(result){
var entries = Object.keys(result.files).map(function (name) {
if(name.indexOf("TestStatus") != -1){
return name;
}
}).filter(notUndefined => notUndefined !== undefined);
var listOfPromises = entries.map(function(entry) {
return result.file(entry).async("text").then(function(fileContent){
return fileContent;
});
});
Promise.all(listOfPromises).then((values) =>{
values.forEach(function(value){
console.log(value);
});
});
});
} catch (error) {
context.fail(error);
return;
}
};
Modified/Corrected code
// dependencies
const AWS = require('aws-sdk');
var JSZip = require('jszip');
// get reference to S3 client
const s3 = new AWS.S3();
exports.handler = async (event, context, callback) => {
// Read options from the event parameter.
const srcBucket = event.Records[0].s3.bucket.name;
// Object key may have spaces or unicode non-ASCII characters.
const srcKey = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " "));
// Download the file from the S3 source bucket.
try {
const params = {
Bucket: srcBucket,
Key: srcKey
};
const data = await s3.getObject(params).promise();
var zip = JSZip.loadAsync(data.Body);
return zip.then(function(result){
var entries = Object.keys(result.files).map((name) =>{
if(name.indexOf("TestStatus") != -1){
return result.files[name];
}
}).filter(notUndefined => notUndefined !== undefined);
var listOfPromises = entries.map((entry) => {
return entry.async("text")
.then((u8) => {
return [entry.name, u8];
}).catch(error => console.error(error));
});
var promiseOfList = Promise.all(listOfPromises);
promiseOfList.then(function (list) {
console.log(list.toString());
});
});
} catch (error) {
context.fail(error);
return;
}
};
If you look closely you are not retuning anything it is why it stays on Pending
const AWS = require('aws-sdk');
var JSZip = require('jszip');
// get reference to S3 client
const s3 = new AWS.S3();
exports.handler = async (event, context, callback) => {
// Read options from the event parameter.
const srcBucket = event.Records[0].s3.bucket.name;
// Object key may have spaces or unicode non-ASCII characters.
const srcKey = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " "));
// Download the file from the S3 source bucket.
try {
const params = {
Bucket: srcBucket,
Key: srcKey
};
const data = await s3.getObject(params).promise();
// here is the problem
// var zip = JSZip.loadAsync(data.Body).then(function (content){
// return content;
// }
var zip = await JSZip.loadAsync(data.body)
return zip.then(function(result){
var entries = Object.keys(result.files).map(function (name) {
if(name.indexOf("TestStatus") != -1){
return name;
}
}).filter(notUndefined => notUndefined !== undefined);
var listOfPromises = entries.map(function(entry) {
return result.file(entry).async("text").then(function(fileContent){
return fileContent;
});
});
console.log("Helo");
Promise.all(listOfPromises).then((values) =>{
values.forEach(function(value){
console.log(value);
});
});
});
} catch (error) {
context.fail(error);
return;
}
};
```