Save html file from S3 and set is as content to puppeteer - node.js

I have a Lambda function in AWS which fetches an html file from S3 after a PUT event in the bucket. The Lambda function fetches it and saves it in /tmp/tml and then loads it again so as to be set as content in puppeteer and produce a pdf. My code is the following:
const chromium = require("#sparticuz/chrome-aws-lambda");
const AWS = require('aws-sdk');
const fs = require('fs');
const path = require('path');
const IAM_USER_KEY = "asdfadsfasd";
const IAM_USER_SECRET = "asdfdsafasdfds";
const s3bucket = new AWS.S3({
accessKeyId: IAM_USER_KEY,
secretAccessKey: IAM_USER_SECRET
});
const copyRecursiveSync = function (src, dest) {
const exists = fs.existsSync(src);
const stats = exists && fs.statSync(src);
const isDirectory = exists && stats.isDirectory();
if (isDirectory) {
if (!fs.existsSync(dest)) {
fs.mkdirSync(dest);
}
fs.readdirSync(src).forEach(function (childItemName) {
copyRecursiveSync(path.join(src, childItemName), path.join(dest, childItemName));
});
} else {
fs.copyFileSync(src, dest);
}
};
function uploadObjectToS3Bucket(objectName, objectData) {
const params = {
Bucket: 'asdfasdfsadf',
Key: objectName,
Body: objectData,
ContentType: 'application/pdf'
};
s3bucket.upload(params, function(err, data) {
if (err) throw err;
console.log('File uploaded successfully');
});
}
function downloadFromS3(bucket, key, location){
const params = {
Bucket: bucket,
Key: key,
};
const rs = s3bucket.getObject(params).createReadStream();
const ws = fs.createWriteStream(location);
rs.pipe(ws);
return true;
}
exports.handler = async (event, context, callback) => {
copyRecursiveSync('mylayerfiles/tml/', '/tmp/tml/');
console.log('Assets copied to /tmp/tml \n');
const bucket = event.Records[0].s3.bucket.name;
const key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
const html_file_name = key.split('/').pop();
console.log('Launching browser');
const browser = await chromium.puppeteer.launch({ headless: true, executablePath: await chromium.executablePath, args: ['--no-sandbox', '--disable-setuid-sandbox','--disable-web-security',
'--disable-dev-shm-usage',
'--single-process']});
console.log("Browser launched");
const page = await browser.newPage();
console.log(`Saving in /tmp/tml/${html_file_name}`);
downloadFromS3(bucket, key, `/tmp/tml/${html_file_name}`);
const bufferFile = async (relPath) => {
const data = fs.readFileSync(relPath, { encoding: 'utf8' }, function (err) {
if (err) {
console.log("readfile failed: " + err);
return 400;
} else {
console.log("readfile succeeded");
}
});
return data;};
const BUFFER = await bufferFile(`/tmp/tml/${html_file_name}`);
console.log('html file read from /tmp');
await page.setContent(content);
console.log('html set as content');
const pdfConfig = {
printBackground: true,
landscape: false,
width: "338.63mm",
height: "190.5mm"
};
await page.emulateMediaType('screen');
const pdf = await page.pdf(pdfConfig);
console.log('Uploading to S3 bucket');
const key_to_save = key.replace(/\.[^/.]+$/, ".pdf");
console.log(key_to_save);
uploadObjectToS3Bucket(key_to_save, pdf);
console.log('Uploaded to S3 bucket');
await browser.close();
console.log('Browser closed');
return 200;}
catch (err) {
console.log(err);
return 500}
};
However, the problems I'm facing are two:
The file sometimes is not written in /tmp/tml for some reason(!)
If written, it is not read correctly and subsequently, the setcontent() function does not product the proper pdf.

Related

Nodejs S3 - Delete Multiple Images

My code it's responding as successful but the images are still in my bucket.
I'm trying to delete images from my s3Bucket (upload works perfect) but the Images delete process it's not working properly.
Here's my code:
myRouter
router.delete("/:id", auth, async (req, res) => {
const productSelected = await Product.findById(req.params.id);
const result = await drop(productSelected);
console.log('>>>>>>> ', result);
res.send(result);
});
myS3Class
const aws = require('aws-sdk');
const multer = require('multer');
const multerS3 = require('multer-s3');
const AWS_CONSTANTS = require('../constants');
const config = require('config');
aws.config.update({
secretAccessKey: config.get('AWS_SECRET_ACCESS_KEY'),
accessKeyId: config.get('AWS_ACCESS_KEY_ID'),
region: AWS_CONSTANTS.region
})
const s3 = new aws.S3();
const drop = async (data) => {
let obj = [];
let result = "";
data.images.map((image) => {
obj.push({ Key: `${AWS_CONSTANTS.bucket}/${image}` }); // This is the full path
});
const options = {
Bucket: AWS_CONSTANTS.bucket,
Delete: {
Objects: obj,
Quiet: false,
},
}
try {
await s3.deleteObjects(
options,
function (err, data) {
if (err) console.log('err ==>', err);
console.log('delete successfully', data);
result = data;
}
).promise();
} catch(error) {
return { success: false, data: null }
}
return result;
}
module.exports.drop = drop;
This is the response from my code:
>>>>>>> delete successfully {
Deleted: [ { Key: 'my-bucketfolder/1655743085375Python.png' } ],
Errors: []
}

How to upload any type of file to aws s3 bucket?

I am trying to upload files like docs ppts etc. I have a front end in react and my upload function looks like this:
const reader = new FileReader()
const toBase64 = (file) =>
new Promise((resolve, reject) => {
reader.readAsDataURL(file);
reader.onload = () => resolve(reader.result);
reader.onerror = (error) => reject(error);
});
` const UploadMinues = async (event: any) => {
console.log(event, 'envent from forntend');
if (event.target && event?.target?.files[0]) {
try {
await toBase64(event.target.files[0]);
console.log(reader.result, 'reader.result');
const res = (await API.graphql({
query: `mutation MyMutation {
updateMinutes(input: { projectId:"${event.target.id}", adviserId: "${user.username}", file: "${reader.result}", fileName: "${event.target.files[0].name}", fileType: "${event.target.files[0].type}"}) {
minutesKey
}
}`,
authMode: GRAPHQL_AUTH_MODE.AMAZON_COGNITO_USER_POOLS
})) as any;
} catch (e) {
console.log('UpdateImage_Error==>', e);
setMinutesErr(e.message);
setOpenAlert(true);
}
} else {
console.log('errorrrrrrrrrrr');
return;
}
};`
And on the back end which is in I have a lambda function like this:
const AWS = require('aws-sdk');
const docClient = new AWS.DynamoDB.DocumentClient();
export async function updateMinutes(data: any) {
let { adviserId, projectId, file, fileName, fileType } = data;
console.log(data, "received from front end")
let s3bucket = new AWS.S3({ params: { Bucket: `${process.env.S3_BUCKET_NAME}` } });
try {
// const buf = Buffer.from(file.replace(/^data:application\/\w+;base64,/, ""), 'base64')
let params_upload = {
Key: `minutes/${adviserId}/${projectId}/${fileName}`,
Body: Buffer.from(file, "base64"),
ContentType: fileType,
CacheControl: 'max-age=86400'
};
const minutes_save = await s3bucket.upload(params_upload).promise()
const minutesKey = minutes_save.Key
let params = {
TableName: process.env.CONSULTATION_TABLE,
Key: {
adviserId: adviserId,
projectId: projectId,
},
UpdateExpression: `set minutes = :edu`,
ExpressionAttributeValues: {':edu' : [minutesKey]}
}
const data = await docClient.update(params).promise()
return {
minutesKey: minutesKey
}
} catch (err) {
console.log(err, "IMAGE_UPLOAD_ERROR")
}
}
The file get uploaded to s3 bucket but when I open it it is in some symbols format. Could someone please explain what i am doing wrong here. because same approach is working fine when I try to upload pdf or image but not with docs or excels files.
My input look like this:
<Input
id={data.projectId}
name={data.projectId}
onChange={UploadMinues}
accept="application/*"
multiple
type="file"
/>

How can I upload multiple images to an s3 bucket in a lambda function using node.js?

I am not very familiar with node and trying to upload an array of media objects to an s3 bucket using an AWS Lambda node function.
the payload has an album which is an array of key/data dictionaries. My code is as below but I'm certain this is wrong.
const awsServerlessExpress = require('aws-serverless-express');
const app = require('./app');
const server = awsServerlessExpress.createServer(app);
const AWS = require("aws-sdk");
const docClient = new AWS.DynamoDB.DocumentClient();
var s3 = new AWS.S3();
var s3Params = {
Bucket: 'bucketid',
ContentEncoding: 'base64',
ContentType: 'image/jpeg'
};
exports.handler = async (event, context) => {
console.log(event);
var body = JSON.parse(event.body);
if (typeof body.album !== 'undefined' && body.album) {
body.album.forEach(function (value) {
var data = body.album.mediaString;
let mediaData = new Buffer(data, 'base64');
var mediaKey = body.album.mediaKey;
try {
s3Params = {
Bucket: 'bucketID',
Key: mediaKey,
Body: mediaData
};
try {
const stored = await s3.upload(s3Params).promise();
console.log("stored successfully");
return { body: JSON.stringify(data) };
} catch (err) {
console.log("error storing");
console.log(err);
return { error: err };
}
} catch (err) {
return { error: err };
}
});
return { body: JSON.stringify(data) };
} else {
return { error: 'error'};
}
};
I have an error that s3 not found. Just wondering if I'm going about this all wrong.
When I only upload one image with the following code everything works fine:
const awsServerlessExpress = require('aws-serverless-express');
const app = require('./app');
const server = awsServerlessExpress.createServer(app);
const AWS = require("aws-sdk");
const docClient = new AWS.DynamoDB.DocumentClient();
var s3 = new AWS.S3();
var s3Params = {
Bucket: 'bucketID',
ContentEncoding: 'base64',
ContentType: 'image/jpeg'
};
exports.handler = async (event, context) => {
var body = JSON.parse(event.body);
var data = body.mediaString;
let mediaData = new Buffer(data, 'base64');
var mediaKey = body.mediaKey;
try {
s3Params = {
Bucket: 'bucketID',
Key: mediaKey,
Body: mediaData
};
try {
const stored = await s3.upload(s3Params).promise();
console.log("stored successfully");
return { body: JSON.stringify(data) };
} catch (err) {
console.log("error storing");
console.log(err);
return { error: err };
}
} catch (err) {
return { error: err };
}
};

AWS Lambda Custom Nodejs Container Shows Runtime Error

I have built a AWS Lambda function with custom container image. I am trying to convert an excel file to pdf with Libreoffice - getting the file from S3 and saving it to a file and converting it to pdf and then uploading it back to S3.
Here the code.
const fs = require('fs');
const getStream = require('get-stream');
const { Readable } = require('stream')
const { S3Client, GetObjectCommand, PutObjectCommand } = require("#aws-sdk/client-s3");
const libre = require('libreoffice-convert');
const path = require('path');
exports.handler = async (event) => {
const bucket = event.queryStringParameters.bucket;
const file = event.queryStringParameters.file;
const convertedFile = event.queryStringParameters.convertedFile;
if (event.queryStringParameters['warmup'] !== undefined) {
return {
result: true,
message: 'warmed up'
}
}
const client = new S3Client({ region: "ap-south-1" });
const command = new GetObjectCommand({ Bucket: bucket, Key: file });
const response = await client.send(command);
const objectData = response.Body;
const writeStream = fs.createWriteStream("/tmp/sample.xlsx");
objectData.pipe(writeStream);
var end = new Promise((resolve, reject) => {
objectData.on('close', resolve(true));
objectData.on('end', resolve(true));
objectData.on('error', reject(false));
});
let completed = await end;
if (completed) {
const extend = '.pdf'
const outputPath = `/tmp/sample${extend}`;
const enterPath = '/tmp/sample.xlsx';
var readingFile = new Promise((resolve, reject) => {
fs.readFile(enterPath, (err, data)=>{
if (err) {
reject(false);
}
resolve(data);
});
});
var fileData = await readingFile;
var converting = new Promise((resolve, reject) => {
libre.convert(fileData, extend, undefined, (err, done) => {
if (err) {
reject(false)
}
fs.writeFileSync(outputPath, done);
resolve(true)
});
})
var converted = await converting;
if (converted) {
var convertedFileStream = fs.createReadStream(outputPath);
const uploadCommand = new PutObjectCommand({ Bucket: bucket, Key: convertedFile, Body: convertedFileStream });
const lastResponse = await client.send(uploadCommand);
const returnResponse = {
result: true,
message: 'success',
bucket: event.queryStringParameters.bucket,
file: event.queryStringParameters.file,
convertedFile: event.queryStringParameters.convertedFile
};
if (event.queryStringParameters['returnEvent'] !== undefined) {
returnResponse['returnEvent'] = event;
}
return returnResponse;
}
}
return completed;
};
However, I am getting this error at time. Sometimes, it is success, but, sometimes it throws this error.
{
"errorType": "Error",
"errorMessage": "false",
"stack": [
"Error: false",
" at _homogeneousError (/function/node_modules/aws-lambda-ric/lib/Runtime/CallbackContext.js:56:16)",
" at postError (/function/node_modules/aws-lambda-ric/lib/Runtime/CallbackContext.js:72:34)",
" at done (/function/node_modules/aws-lambda-ric/lib/Runtime/CallbackContext.js:99:13)",
" at fail (/function/node_modules/aws-lambda-ric/lib/Runtime/CallbackContext.js:113:13)",
" at /function/node_modules/aws-lambda-ric/lib/Runtime/CallbackContext.js:148:24",
" at processTicksAndRejections (internal/process/task_queues.js:97:5)"
]
}
I dont know Nodejs on a great deal so I think if the code is not written the correct way. Any ideas what I am doing wrong here ?
Like #hoangdv when I logged errors I came to know that the file saving to the disk was not correct. So, I changed the area of the code where it saves to like this and then it worked.
const client = new S3Client({ region: "ap-south-1" });
const command = new GetObjectCommand({ Bucket: bucket, Key: file });
const { Body } = await client.send(command);
await new Promise((resolve, reject) => {
Body.pipe(fs.createWriteStream(filePath))
.on('error', err => reject(err))
.on('close', () => resolve())
})
const excelFile = fs.readFileSync(filePath);

In NodeJS, how to download files from S3

In ExpressJS, I would like to download files previously uploaded to an Amazon S3 bucket.
Here is my current route:
const express = require('express');
const AWS = require('aws-sdk');
const mammoth = require('mammoth');
const fs = require('fs').promises
const path = require('path')
const router = express.Router();
router.put('/:id/download', async (req, res, next) => {
console.log('hitting download route')
var id = req.params.id;
let upload = await Upload.query().findById( id ).eager('user');
console.log("file to download is: ", upload.name)
AWS.config.update({
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
});
const s3 = new AWS.S3();
// var fileStream = fs.createWriteStream('/tmp/file.docx');
// var s3Stream = s3.getObject(params).createReadStream();
const downloadFromS3 = async () => {
const params = {
Bucket: process.env.AWS_BUCKET,
Key: upload.file_url.split("com/").reverse()[0]
};
const { Body } = await s3.getObject(params).promise()
await fs.writeFile(`${ __dirname }/download.docx`, Body)
return Body
}
// mammoth.convertToHtml({ path: '/Users/dariusgoore/Downloads/1585930968750.docx' })
// .then(async function(result) {
// await Upload.query().findById( id )
// .patch({
// html: result.value,
// conversion_messages: result.messages
// })
// res.json(result);
// })
// .done();
res.send(downloadFromS3)
});
I get no errors, but the file is not created, or if I manually create the file, it remains empty.
If I've understood you correctly the issue is that you're not waiting for the file to be written to the local file system, you're returning it in the response via express.
Give this code a go.
const express = require('express')
const AWS = require('aws-sdk')
const mammoth = require('mammoth')
const fs = require('fs').promises
const path = require('path')
const router = express.Router()
const s3 = new AWS.S3()
AWS.config.update({
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
})
const downloadFromS3 = async (key, location) => {
const params = {
Bucket: process.env.AWS_BUCKET,
Key: key,
}
const { Body } = await s3.getObject(params).promise()
await fs.writeFile(location, Body)
return true
}
router.put('/:id/download', async (req, res, next) => {
console.log('hitting download route')
const upload = await Upload.query()
.findById(req.params.id)
.eager('user')
console.log('file to download is: ', upload.name)
const key = upload.file_url.split('com/').reverse()[0]
const location = `${__dirname}/${key}.docx`
await downloadFromS3(key, location)
res.send({ key, location })
})
import { S3 } from 'aws-sdk';
import fs from 'fs';
export default class S3Service {
s3: S3;
constructor() {
this.s3 = new S3({
apiVersion: *****,
region: ********
});
}
//Download File
async download(bucketName: string, keyName: string, localDest?: string): Promise<any> {
if (typeof localDest == 'undefined') {
localDest = keyName;
}
const params = {
Bucket: bucketName,
Key: keyName
};
console.log("params: ", params);
let writeStream = fs.createWriteStream(localDest);
return new Promise<any>((resolve, reject) => {
const readStream = this.s3.getObject(params).createReadStream();
// Error handling in read stream
readStream.on("error", (e) => {
console.error(e);
reject(e);
});
// Resolve only if we are done writing
writeStream.once('finish', () => {
resolve(keyName);
});
// pipe will automatically finish the write stream once done
readStream.pipe(writeStream);
});
}
}

Resources