I have built a AWS Lambda function with custom container image. I am trying to convert an excel file to pdf with Libreoffice - getting the file from S3 and saving it to a file and converting it to pdf and then uploading it back to S3.
Here the code.
const fs = require('fs');
const getStream = require('get-stream');
const { Readable } = require('stream')
const { S3Client, GetObjectCommand, PutObjectCommand } = require("#aws-sdk/client-s3");
const libre = require('libreoffice-convert');
const path = require('path');
exports.handler = async (event) => {
const bucket = event.queryStringParameters.bucket;
const file = event.queryStringParameters.file;
const convertedFile = event.queryStringParameters.convertedFile;
if (event.queryStringParameters['warmup'] !== undefined) {
return {
result: true,
message: 'warmed up'
}
}
const client = new S3Client({ region: "ap-south-1" });
const command = new GetObjectCommand({ Bucket: bucket, Key: file });
const response = await client.send(command);
const objectData = response.Body;
const writeStream = fs.createWriteStream("/tmp/sample.xlsx");
objectData.pipe(writeStream);
var end = new Promise((resolve, reject) => {
objectData.on('close', resolve(true));
objectData.on('end', resolve(true));
objectData.on('error', reject(false));
});
let completed = await end;
if (completed) {
const extend = '.pdf'
const outputPath = `/tmp/sample${extend}`;
const enterPath = '/tmp/sample.xlsx';
var readingFile = new Promise((resolve, reject) => {
fs.readFile(enterPath, (err, data)=>{
if (err) {
reject(false);
}
resolve(data);
});
});
var fileData = await readingFile;
var converting = new Promise((resolve, reject) => {
libre.convert(fileData, extend, undefined, (err, done) => {
if (err) {
reject(false)
}
fs.writeFileSync(outputPath, done);
resolve(true)
});
})
var converted = await converting;
if (converted) {
var convertedFileStream = fs.createReadStream(outputPath);
const uploadCommand = new PutObjectCommand({ Bucket: bucket, Key: convertedFile, Body: convertedFileStream });
const lastResponse = await client.send(uploadCommand);
const returnResponse = {
result: true,
message: 'success',
bucket: event.queryStringParameters.bucket,
file: event.queryStringParameters.file,
convertedFile: event.queryStringParameters.convertedFile
};
if (event.queryStringParameters['returnEvent'] !== undefined) {
returnResponse['returnEvent'] = event;
}
return returnResponse;
}
}
return completed;
};
However, I am getting this error at time. Sometimes, it is success, but, sometimes it throws this error.
{
"errorType": "Error",
"errorMessage": "false",
"stack": [
"Error: false",
" at _homogeneousError (/function/node_modules/aws-lambda-ric/lib/Runtime/CallbackContext.js:56:16)",
" at postError (/function/node_modules/aws-lambda-ric/lib/Runtime/CallbackContext.js:72:34)",
" at done (/function/node_modules/aws-lambda-ric/lib/Runtime/CallbackContext.js:99:13)",
" at fail (/function/node_modules/aws-lambda-ric/lib/Runtime/CallbackContext.js:113:13)",
" at /function/node_modules/aws-lambda-ric/lib/Runtime/CallbackContext.js:148:24",
" at processTicksAndRejections (internal/process/task_queues.js:97:5)"
]
}
I dont know Nodejs on a great deal so I think if the code is not written the correct way. Any ideas what I am doing wrong here ?
Like #hoangdv when I logged errors I came to know that the file saving to the disk was not correct. So, I changed the area of the code where it saves to like this and then it worked.
const client = new S3Client({ region: "ap-south-1" });
const command = new GetObjectCommand({ Bucket: bucket, Key: file });
const { Body } = await client.send(command);
await new Promise((resolve, reject) => {
Body.pipe(fs.createWriteStream(filePath))
.on('error', err => reject(err))
.on('close', () => resolve())
})
const excelFile = fs.readFileSync(filePath);
Related
I have a Lambda function in AWS which fetches an html file from S3 after a PUT event in the bucket. The Lambda function fetches it and saves it in /tmp/tml and then loads it again so as to be set as content in puppeteer and produce a pdf. My code is the following:
const chromium = require("#sparticuz/chrome-aws-lambda");
const AWS = require('aws-sdk');
const fs = require('fs');
const path = require('path');
const IAM_USER_KEY = "asdfadsfasd";
const IAM_USER_SECRET = "asdfdsafasdfds";
const s3bucket = new AWS.S3({
accessKeyId: IAM_USER_KEY,
secretAccessKey: IAM_USER_SECRET
});
const copyRecursiveSync = function (src, dest) {
const exists = fs.existsSync(src);
const stats = exists && fs.statSync(src);
const isDirectory = exists && stats.isDirectory();
if (isDirectory) {
if (!fs.existsSync(dest)) {
fs.mkdirSync(dest);
}
fs.readdirSync(src).forEach(function (childItemName) {
copyRecursiveSync(path.join(src, childItemName), path.join(dest, childItemName));
});
} else {
fs.copyFileSync(src, dest);
}
};
function uploadObjectToS3Bucket(objectName, objectData) {
const params = {
Bucket: 'asdfasdfsadf',
Key: objectName,
Body: objectData,
ContentType: 'application/pdf'
};
s3bucket.upload(params, function(err, data) {
if (err) throw err;
console.log('File uploaded successfully');
});
}
function downloadFromS3(bucket, key, location){
const params = {
Bucket: bucket,
Key: key,
};
const rs = s3bucket.getObject(params).createReadStream();
const ws = fs.createWriteStream(location);
rs.pipe(ws);
return true;
}
exports.handler = async (event, context, callback) => {
copyRecursiveSync('mylayerfiles/tml/', '/tmp/tml/');
console.log('Assets copied to /tmp/tml \n');
const bucket = event.Records[0].s3.bucket.name;
const key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
const html_file_name = key.split('/').pop();
console.log('Launching browser');
const browser = await chromium.puppeteer.launch({ headless: true, executablePath: await chromium.executablePath, args: ['--no-sandbox', '--disable-setuid-sandbox','--disable-web-security',
'--disable-dev-shm-usage',
'--single-process']});
console.log("Browser launched");
const page = await browser.newPage();
console.log(`Saving in /tmp/tml/${html_file_name}`);
downloadFromS3(bucket, key, `/tmp/tml/${html_file_name}`);
const bufferFile = async (relPath) => {
const data = fs.readFileSync(relPath, { encoding: 'utf8' }, function (err) {
if (err) {
console.log("readfile failed: " + err);
return 400;
} else {
console.log("readfile succeeded");
}
});
return data;};
const BUFFER = await bufferFile(`/tmp/tml/${html_file_name}`);
console.log('html file read from /tmp');
await page.setContent(content);
console.log('html set as content');
const pdfConfig = {
printBackground: true,
landscape: false,
width: "338.63mm",
height: "190.5mm"
};
await page.emulateMediaType('screen');
const pdf = await page.pdf(pdfConfig);
console.log('Uploading to S3 bucket');
const key_to_save = key.replace(/\.[^/.]+$/, ".pdf");
console.log(key_to_save);
uploadObjectToS3Bucket(key_to_save, pdf);
console.log('Uploaded to S3 bucket');
await browser.close();
console.log('Browser closed');
return 200;}
catch (err) {
console.log(err);
return 500}
};
However, the problems I'm facing are two:
The file sometimes is not written in /tmp/tml for some reason(!)
If written, it is not read correctly and subsequently, the setcontent() function does not product the proper pdf.
#TBA gave the solution.
The root cause is not by runtime. It came from SDK v3.
Point: Do not update the code with mixed things (like both of runtime & SDK version together 🥲)
Thanks again, TBA.
I was using Node.js 14.x version runtime Lambda to read some json file from S3.
Brief code is below
const AWS = require("aws-sdk");
const s3 = new AWS.S3();
exports.handler = (event) => {
const { bucketName, objKey } = event
const params = {
Bucket: bucketName,
Key: objKey
};
return new Promise((resolve) => {
s3.getObject(params, async (err, data) =>{
if (err) console.log(err, err.stack);
else {
const contents = JSON.parse(data.Body)
resolve(contents);
}
});
})
};
and it returned the json data as I expected.
And today I tried to create a new lambda with runtime Node.js 18.x but it returned null or some errors...
Q) Could you give me some advice to solve this 🥲 ?
+) I used same json file for each lambda
+) Not sure why, but in my case, data.Body.toString() didn't work (I saw some answers in stackoverflow provide that and tried but no lucks)
Thanks in advance!
Case A (returns null)
import { S3Client, GetObjectCommand } from "#aws-sdk/client-s3";
const s3Client = new S3Client({ region: "ap-northeast-2" });
export const handler = (event) => {
const { objKey, bucketName } = event;
const params={
Bucket: bucketName,
Key: objKey
};
const getObjCommand = new GetObjectCommand(params);
return new Promise((resolve) => {
s3Client.send(getObjCommand, async (err, data) =>{
if (err) console.log(err, err.stack);
else {
const list = JSON.parse(data.Body)
resolve(list);
}
});
})
};
Case B (returns "Unexpected token o in JSON at position 1")
export const handler = async (event) => {
const { objKey, bucketName } = event;
const params={
Bucket: bucketName,
Key: objKey
};
const getObjCommand = new GetObjectCommand(params);
const response = await s3Client.send(getObjCommand)
console.log("JSON.parse(response.Body)", JSON.parse(response.Body))
};
Case C (returns "TypeError: Converting circular structure to JSON")
export const handler = async (event) => {
const { objKey, bucketName } = event;
const params={
Bucket: bucketName,
Key: objKey
};
const getObjCommand = new GetObjectCommand(params);
try {
const response = await s3Client.send(getObjCommand)
return JSON.stringify(response.Body)
} catch(err) {
console.log("error", err)
return err
}
};
I am trying to upload files like docs ppts etc. I have a front end in react and my upload function looks like this:
const reader = new FileReader()
const toBase64 = (file) =>
new Promise((resolve, reject) => {
reader.readAsDataURL(file);
reader.onload = () => resolve(reader.result);
reader.onerror = (error) => reject(error);
});
` const UploadMinues = async (event: any) => {
console.log(event, 'envent from forntend');
if (event.target && event?.target?.files[0]) {
try {
await toBase64(event.target.files[0]);
console.log(reader.result, 'reader.result');
const res = (await API.graphql({
query: `mutation MyMutation {
updateMinutes(input: { projectId:"${event.target.id}", adviserId: "${user.username}", file: "${reader.result}", fileName: "${event.target.files[0].name}", fileType: "${event.target.files[0].type}"}) {
minutesKey
}
}`,
authMode: GRAPHQL_AUTH_MODE.AMAZON_COGNITO_USER_POOLS
})) as any;
} catch (e) {
console.log('UpdateImage_Error==>', e);
setMinutesErr(e.message);
setOpenAlert(true);
}
} else {
console.log('errorrrrrrrrrrr');
return;
}
};`
And on the back end which is in I have a lambda function like this:
const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient();
export async function updateMinutes(data: any) {
let { adviserId, projectId, file, fileName, fileType } = data;
console.log(data, "received from front end")
let s3bucket = new AWS.S3({ params: { Bucket: `${process.env.S3_BUCKET_NAME}` } });
try {
// const buf = Buffer.from(file.replace(/^data:application\/\w+;base64,/, ""), 'base64')
let params_upload = {
Key: `minutes/${adviserId}/${projectId}/${fileName}`,
Body: Buffer.from(file, "base64"),
ContentType: fileType,
CacheControl: 'max-age=86400'
};
const minutes_save = await s3bucket.upload(params_upload).promise()
const minutesKey = minutes_save.Key
let params = {
TableName: process.env.CONSULTATION_TABLE,
Key: {
adviserId: adviserId,
projectId: projectId,
},
UpdateExpression: `set minutes = :edu`,
ExpressionAttributeValues: {':edu' : [minutesKey]}
}
const data = await docClient.update(params).promise()
return {
minutesKey: minutesKey
}
} catch (err) {
console.log(err, "IMAGE_UPLOAD_ERROR")
}
}
The file get uploaded to s3 bucket but when I open it it is in some symbols format. Could someone please explain what i am doing wrong here. because same approach is working fine when I try to upload pdf or image but not with docs or excels files.
My input look like this:
<Input
id={data.projectId}
name={data.projectId}
onChange={UploadMinues}
accept="application/*"
multiple
type="file"
/>
I am trying to upload image to AWS S3 bucket using NodeJS. The issue I am facing it is while the image is getting saved but the API is returning 404(Not Found). Here is my controller code:
async UploadProfileImage(ctx) {
try {
var file = ctx.request.files.profileImage;
if (file) {
fs.readFile(file.path, (err, fileData) => {
var resp = s3Utility.UploadProfileImageToS3(file.name, fileData);
//Not reaching here. Although E3 tag printing in console.
console.log(resp);
ctx.status = 200;
ctx.body = { response: 'file Uploaded!' };
});
}
else {
ctx.status = 400;
ctx.body = { response: 'File not found!' };
}
} catch (error) {
ctx.status = 500;
ctx.body = { response: 'There was an error. Please try again later!' };
}
}
Utility Class I am using is:
const AWS = require('aws-sdk');
const crypto = require("crypto");
var fs = require('fs');
const mime = require('mime-types');
export class S3Utility {
constructor() { }
async UploadProfileImageToS3(fileName, data) {
let randomId = crypto.randomBytes(16).toString("hex");
AWS.config.update({ region: "Region", accessKeyId: "KeyID", secretAccessKey: "SecretAccessKey" });
var s3 = new AWS.S3();
var imageName = randomId + fileName;
var params = {
Bucket: "BucketName"
Key: imageName,
Body: data,
ContentType: mime.lookup(fileName)
};
return new Promise((resolve, reject) => {
s3.putObject(params, function (err, data) {
if (err) {
console.log('Error: ', err);
reject(new Error(err.message));
} else {
console.log(data);
resolve({
response: data,
uploadedFileName: imageName
});
}
});
});
}
}
const s3Utility: S3Utility = new S3Utility();
export default s3Utility;
The code is uploading file on S3 but it is not returning proper response. Upon testing this endpoint on postman, I get "Not Found" message. However, I can see E Tag getting logged in console. I don't know what is going wrong here. I am sure it has something to do with promise. Can someone please point out or fix the mistake?
Edit:
Using async fs.readFile does the trick.
const fs = require('fs').promises;
const fileData = await fs.readFile(file.path, "binary");
var resp = await s3Utility.UploadProfileImageToS3(file.name, fileData);
I need to upload a lot of files (about 65.000) splitted in subdirectory.
I tried to iterate and load every single file like this:
const fs = require("fs");
const path = require("path");
const async = require("async");
const AWS = require("aws-sdk");
const readdir = require("recursive-readdir");
const slash = require("slash");
const { BUCKET, KEY, SECRET } = process.env;
const rootFolder = path.resolve(__dirname, "./");
const uploadFolder = "./test_files/15";
const s3 = new AWS.S3({
signatureVersion: "v4",
accessKeyId: KEY,
secretAccessKey: SECRET,
});
function getFiles(dirPath) {
return fs.existsSync(dirPath) ? readdir(dirPath) : [];
}
async function deploy(upload) {
if (!BUCKET || !KEY || !SECRET) {
throw new Error("you must provide env. variables: [BUCKET, KEY, SECRET]");
}
const filesToUpload = await getFiles(path.resolve(__dirname, upload));
return new Promise((resolve, reject) => {
async.eachOfLimit(
filesToUpload,
10,
async.asyncify(async (file) => {
const Key = file.replace(rootFolder + path.sep, "");
console.log(`uploading: [${slash(Key)}]`);
var options = { partSize: 5 * 1024 * 1024, queueSize: 4 };
return new Promise((res, rej) => {
s3.upload(
{
Key: slash(Key),
Bucket: BUCKET,
Body: fs.readFileSync(file),
},
(err) => {
if (err) {
return rej(new Error(err));
}
res({ result: true });
}
);
});
}),
(err) => {
if (err) {
return reject(new Error(err));
}
resolve({ result: true });
}
);
});
}
deploy(uploadFolder)
.then(() => {
console.log("task complete");
process.exit(0);
})
.catch((err) => {
console.error(err);
process.exit(1);
});
but after a considerable number of uploads i have this:
Error: Error: NetworkingError: connect ETIMEDOUT IP_S3_AWS
I need to upload this set of files from ec2 instance (because its a result of a image processing). I have this behavior from my pc, i don't know if from ec2 have the same problem.
I have considered the way of zip all and upload but i need to keep the original directory structure.
I accept also new way to resolve the problem.
Sorry for my bad english.
It would probably be much simpler to use the AWS CLI aws s3 sync command instead of building this yourself.