Nodejs S3 - Delete Multiple Images - node.js

My code it's responding as successful but the images are still in my bucket.
I'm trying to delete images from my s3Bucket (upload works perfect) but the Images delete process it's not working properly.
Here's my code:
myRouter
router.delete("/:id", auth, async (req, res) => {
const productSelected = await Product.findById(req.params.id);
const result = await drop(productSelected);
console.log('>>>>>>> ', result);
res.send(result);
});
myS3Class
const aws = require('aws-sdk');
const multer = require('multer');
const multerS3 = require('multer-s3');
const AWS_CONSTANTS = require('../constants');
const config = require('config');
aws.config.update({
secretAccessKey: config.get('AWS_SECRET_ACCESS_KEY'),
accessKeyId: config.get('AWS_ACCESS_KEY_ID'),
region: AWS_CONSTANTS.region
})
const s3 = new aws.S3();
const drop = async (data) => {
let obj = [];
let result = "";
data.images.map((image) => {
obj.push({ Key: `${AWS_CONSTANTS.bucket}/${image}` }); // This is the full path
});
const options = {
Bucket: AWS_CONSTANTS.bucket,
Delete: {
Objects: obj,
Quiet: false,
},
}
try {
await s3.deleteObjects(
options,
function (err, data) {
if (err) console.log('err ==>', err);
console.log('delete successfully', data);
result = data;
}
).promise();
} catch(error) {
return { success: false, data: null }
}
return result;
}
module.exports.drop = drop;
This is the response from my code:
>>>>>>> delete successfully {
Deleted: [ { Key: 'my-bucketfolder/1655743085375Python.png' } ],
Errors: []
}

Related

Save html file from S3 and set is as content to puppeteer

I have a Lambda function in AWS which fetches an html file from S3 after a PUT event in the bucket. The Lambda function fetches it and saves it in /tmp/tml and then loads it again so as to be set as content in puppeteer and produce a pdf. My code is the following:
const chromium = require("#sparticuz/chrome-aws-lambda");
const AWS = require('aws-sdk');
const fs = require('fs');
const path = require('path');
const IAM_USER_KEY = "asdfadsfasd";
const IAM_USER_SECRET = "asdfdsafasdfds";
const s3bucket = new AWS.S3({
accessKeyId: IAM_USER_KEY,
secretAccessKey: IAM_USER_SECRET
});
const copyRecursiveSync = function (src, dest) {
const exists = fs.existsSync(src);
const stats = exists && fs.statSync(src);
const isDirectory = exists && stats.isDirectory();
if (isDirectory) {
if (!fs.existsSync(dest)) {
fs.mkdirSync(dest);
}
fs.readdirSync(src).forEach(function (childItemName) {
copyRecursiveSync(path.join(src, childItemName), path.join(dest, childItemName));
});
} else {
fs.copyFileSync(src, dest);
}
};
function uploadObjectToS3Bucket(objectName, objectData) {
const params = {
Bucket: 'asdfasdfsadf',
Key: objectName,
Body: objectData,
ContentType: 'application/pdf'
};
s3bucket.upload(params, function(err, data) {
if (err) throw err;
console.log('File uploaded successfully');
});
}
function downloadFromS3(bucket, key, location){
const params = {
Bucket: bucket,
Key: key,
};
const rs = s3bucket.getObject(params).createReadStream();
const ws = fs.createWriteStream(location);
rs.pipe(ws);
return true;
}
exports.handler = async (event, context, callback) => {
copyRecursiveSync('mylayerfiles/tml/', '/tmp/tml/');
console.log('Assets copied to /tmp/tml \n');
const bucket = event.Records[0].s3.bucket.name;
const key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
const html_file_name = key.split('/').pop();
console.log('Launching browser');
const browser = await chromium.puppeteer.launch({ headless: true, executablePath: await chromium.executablePath, args: ['--no-sandbox', '--disable-setuid-sandbox','--disable-web-security',
'--disable-dev-shm-usage',
'--single-process']});
console.log("Browser launched");
const page = await browser.newPage();
console.log(`Saving in /tmp/tml/${html_file_name}`);
downloadFromS3(bucket, key, `/tmp/tml/${html_file_name}`);
const bufferFile = async (relPath) => {
const data = fs.readFileSync(relPath, { encoding: 'utf8' }, function (err) {
if (err) {
console.log("readfile failed: " + err);
return 400;
} else {
console.log("readfile succeeded");
}
});
return data;};
const BUFFER = await bufferFile(`/tmp/tml/${html_file_name}`);
console.log('html file read from /tmp');
await page.setContent(content);
console.log('html set as content');
const pdfConfig = {
printBackground: true,
landscape: false,
width: "338.63mm",
height: "190.5mm"
};
await page.emulateMediaType('screen');
const pdf = await page.pdf(pdfConfig);
console.log('Uploading to S3 bucket');
const key_to_save = key.replace(/\.[^/.]+$/, ".pdf");
console.log(key_to_save);
uploadObjectToS3Bucket(key_to_save, pdf);
console.log('Uploaded to S3 bucket');
await browser.close();
console.log('Browser closed');
return 200;}
catch (err) {
console.log(err);
return 500}
};
However, the problems I'm facing are two:
The file sometimes is not written in /tmp/tml for some reason(!)
If written, it is not read correctly and subsequently, the setcontent() function does not product the proper pdf.

How can I upload multiple images to an s3 bucket in a lambda function using node.js?

I am not very familiar with node and trying to upload an array of media objects to an s3 bucket using an AWS Lambda node function.
the payload has an album which is an array of key/data dictionaries. My code is as below but I'm certain this is wrong.
const awsServerlessExpress = require('aws-serverless-express');
const app = require('./app');
const server = awsServerlessExpress.createServer(app);
const AWS = require("aws-sdk");
const docClient = new AWS.DynamoDB.DocumentClient();
var s3 = new AWS.S3();
var s3Params = {
Bucket: 'bucketid',
ContentEncoding: 'base64',
ContentType: 'image/jpeg'
};
exports.handler = async (event, context) => {
console.log(event);
var body = JSON.parse(event.body);
if (typeof body.album !== 'undefined' && body.album) {
body.album.forEach(function (value) {
var data = body.album.mediaString;
let mediaData = new Buffer(data, 'base64');
var mediaKey = body.album.mediaKey;
try {
s3Params = {
Bucket: 'bucketID',
Key: mediaKey,
Body: mediaData
};
try {
const stored = await s3.upload(s3Params).promise();
console.log("stored successfully");
return { body: JSON.stringify(data) };
} catch (err) {
console.log("error storing");
console.log(err);
return { error: err };
}
} catch (err) {
return { error: err };
}
});
return { body: JSON.stringify(data) };
} else {
return { error: 'error'};
}
};
I have an error that s3 not found. Just wondering if I'm going about this all wrong.
When I only upload one image with the following code everything works fine:
const awsServerlessExpress = require('aws-serverless-express');
const app = require('./app');
const server = awsServerlessExpress.createServer(app);
const AWS = require("aws-sdk");
const docClient = new AWS.DynamoDB.DocumentClient();
var s3 = new AWS.S3();
var s3Params = {
Bucket: 'bucketID',
ContentEncoding: 'base64',
ContentType: 'image/jpeg'
};
exports.handler = async (event, context) => {
var body = JSON.parse(event.body);
var data = body.mediaString;
let mediaData = new Buffer(data, 'base64');
var mediaKey = body.mediaKey;
try {
s3Params = {
Bucket: 'bucketID',
Key: mediaKey,
Body: mediaData
};
try {
const stored = await s3.upload(s3Params).promise();
console.log("stored successfully");
return { body: JSON.stringify(data) };
} catch (err) {
console.log("error storing");
console.log(err);
return { error: err };
}
} catch (err) {
return { error: err };
}
};

AWS Lambda Custom Nodejs Container Shows Runtime Error

I have built a AWS Lambda function with custom container image. I am trying to convert an excel file to pdf with Libreoffice - getting the file from S3 and saving it to a file and converting it to pdf and then uploading it back to S3.
Here the code.
const fs = require('fs');
const getStream = require('get-stream');
const { Readable } = require('stream')
const { S3Client, GetObjectCommand, PutObjectCommand } = require("#aws-sdk/client-s3");
const libre = require('libreoffice-convert');
const path = require('path');
exports.handler = async (event) => {
const bucket = event.queryStringParameters.bucket;
const file = event.queryStringParameters.file;
const convertedFile = event.queryStringParameters.convertedFile;
if (event.queryStringParameters['warmup'] !== undefined) {
return {
result: true,
message: 'warmed up'
}
}
const client = new S3Client({ region: "ap-south-1" });
const command = new GetObjectCommand({ Bucket: bucket, Key: file });
const response = await client.send(command);
const objectData = response.Body;
const writeStream = fs.createWriteStream("/tmp/sample.xlsx");
objectData.pipe(writeStream);
var end = new Promise((resolve, reject) => {
objectData.on('close', resolve(true));
objectData.on('end', resolve(true));
objectData.on('error', reject(false));
});
let completed = await end;
if (completed) {
const extend = '.pdf'
const outputPath = `/tmp/sample${extend}`;
const enterPath = '/tmp/sample.xlsx';
var readingFile = new Promise((resolve, reject) => {
fs.readFile(enterPath, (err, data)=>{
if (err) {
reject(false);
}
resolve(data);
});
});
var fileData = await readingFile;
var converting = new Promise((resolve, reject) => {
libre.convert(fileData, extend, undefined, (err, done) => {
if (err) {
reject(false)
}
fs.writeFileSync(outputPath, done);
resolve(true)
});
})
var converted = await converting;
if (converted) {
var convertedFileStream = fs.createReadStream(outputPath);
const uploadCommand = new PutObjectCommand({ Bucket: bucket, Key: convertedFile, Body: convertedFileStream });
const lastResponse = await client.send(uploadCommand);
const returnResponse = {
result: true,
message: 'success',
bucket: event.queryStringParameters.bucket,
file: event.queryStringParameters.file,
convertedFile: event.queryStringParameters.convertedFile
};
if (event.queryStringParameters['returnEvent'] !== undefined) {
returnResponse['returnEvent'] = event;
}
return returnResponse;
}
}
return completed;
};
However, I am getting this error at time. Sometimes, it is success, but, sometimes it throws this error.
{
"errorType": "Error",
"errorMessage": "false",
"stack": [
"Error: false",
" at _homogeneousError (/function/node_modules/aws-lambda-ric/lib/Runtime/CallbackContext.js:56:16)",
" at postError (/function/node_modules/aws-lambda-ric/lib/Runtime/CallbackContext.js:72:34)",
" at done (/function/node_modules/aws-lambda-ric/lib/Runtime/CallbackContext.js:99:13)",
" at fail (/function/node_modules/aws-lambda-ric/lib/Runtime/CallbackContext.js:113:13)",
" at /function/node_modules/aws-lambda-ric/lib/Runtime/CallbackContext.js:148:24",
" at processTicksAndRejections (internal/process/task_queues.js:97:5)"
]
}
I dont know Nodejs on a great deal so I think if the code is not written the correct way. Any ideas what I am doing wrong here ?
Like #hoangdv when I logged errors I came to know that the file saving to the disk was not correct. So, I changed the area of the code where it saves to like this and then it worked.
const client = new S3Client({ region: "ap-south-1" });
const command = new GetObjectCommand({ Bucket: bucket, Key: file });
const { Body } = await client.send(command);
await new Promise((resolve, reject) => {
Body.pipe(fs.createWriteStream(filePath))
.on('error', err => reject(err))
.on('close', () => resolve())
})
const excelFile = fs.readFileSync(filePath);

NodeJS API to upload image to S3 not returning response

I am trying to upload image to AWS S3 bucket using NodeJS. The issue I am facing it is while the image is getting saved but the API is returning 404(Not Found). Here is my controller code:
async UploadProfileImage(ctx) {
try {
var file = ctx.request.files.profileImage;
if (file) {
fs.readFile(file.path, (err, fileData) => {
var resp = s3Utility.UploadProfileImageToS3(file.name, fileData);
//Not reaching here. Although E3 tag printing in console.
console.log(resp);
ctx.status = 200;
ctx.body = { response: 'file Uploaded!' };
});
}
else {
ctx.status = 400;
ctx.body = { response: 'File not found!' };
}
} catch (error) {
ctx.status = 500;
ctx.body = { response: 'There was an error. Please try again later!' };
}
}
Utility Class I am using is:
const AWS = require('aws-sdk');
const crypto = require("crypto");
var fs = require('fs');
const mime = require('mime-types');
export class S3Utility {
constructor() { }
async UploadProfileImageToS3(fileName, data) {
let randomId = crypto.randomBytes(16).toString("hex");
AWS.config.update({ region: "Region", accessKeyId: "KeyID", secretAccessKey: "SecretAccessKey" });
var s3 = new AWS.S3();
var imageName = randomId + fileName;
var params = {
Bucket: "BucketName"
Key: imageName,
Body: data,
ContentType: mime.lookup(fileName)
};
return new Promise((resolve, reject) => {
s3.putObject(params, function (err, data) {
if (err) {
console.log('Error: ', err);
reject(new Error(err.message));
} else {
console.log(data);
resolve({
response: data,
uploadedFileName: imageName
});
}
});
});
}
}
const s3Utility: S3Utility = new S3Utility();
export default s3Utility;
The code is uploading file on S3 but it is not returning proper response. Upon testing this endpoint on postman, I get "Not Found" message. However, I can see E Tag getting logged in console. I don't know what is going wrong here. I am sure it has something to do with promise. Can someone please point out or fix the mistake?
Edit:
Using async fs.readFile does the trick.
const fs = require('fs').promises;
const fileData = await fs.readFile(file.path, "binary");
var resp = await s3Utility.UploadProfileImageToS3(file.name, fileData);

In NodeJS, how to download files from S3

In ExpressJS, I would like to download files previously uploaded to an Amazon S3 bucket.
Here is my current route:
const express = require('express');
const AWS = require('aws-sdk');
const mammoth = require('mammoth');
const fs = require('fs').promises
const path = require('path')
const router = express.Router();
router.put('/:id/download', async (req, res, next) => {
console.log('hitting download route')
var id = req.params.id;
let upload = await Upload.query().findById( id ).eager('user');
console.log("file to download is: ", upload.name)
AWS.config.update({
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
});
const s3 = new AWS.S3();
// var fileStream = fs.createWriteStream('/tmp/file.docx');
// var s3Stream = s3.getObject(params).createReadStream();
const downloadFromS3 = async () => {
const params = {
Bucket: process.env.AWS_BUCKET,
Key: upload.file_url.split("com/").reverse()[0]
};
const { Body } = await s3.getObject(params).promise()
await fs.writeFile(`${ __dirname }/download.docx`, Body)
return Body
}
// mammoth.convertToHtml({ path: '/Users/dariusgoore/Downloads/1585930968750.docx' })
// .then(async function(result) {
// await Upload.query().findById( id )
// .patch({
// html: result.value,
// conversion_messages: result.messages
// })
// res.json(result);
// })
// .done();
res.send(downloadFromS3)
});
I get no errors, but the file is not created, or if I manually create the file, it remains empty.
If I've understood you correctly the issue is that you're not waiting for the file to be written to the local file system, you're returning it in the response via express.
Give this code a go.
const express = require('express')
const AWS = require('aws-sdk')
const mammoth = require('mammoth')
const fs = require('fs').promises
const path = require('path')
const router = express.Router()
const s3 = new AWS.S3()
AWS.config.update({
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
})
const downloadFromS3 = async (key, location) => {
const params = {
Bucket: process.env.AWS_BUCKET,
Key: key,
}
const { Body } = await s3.getObject(params).promise()
await fs.writeFile(location, Body)
return true
}
router.put('/:id/download', async (req, res, next) => {
console.log('hitting download route')
const upload = await Upload.query()
.findById(req.params.id)
.eager('user')
console.log('file to download is: ', upload.name)
const key = upload.file_url.split('com/').reverse()[0]
const location = `${__dirname}/${key}.docx`
await downloadFromS3(key, location)
res.send({ key, location })
})
import { S3 } from 'aws-sdk';
import fs from 'fs';
export default class S3Service {
s3: S3;
constructor() {
this.s3 = new S3({
apiVersion: *****,
region: ********
});
}
//Download File
async download(bucketName: string, keyName: string, localDest?: string): Promise<any> {
if (typeof localDest == 'undefined') {
localDest = keyName;
}
const params = {
Bucket: bucketName,
Key: keyName
};
console.log("params: ", params);
let writeStream = fs.createWriteStream(localDest);
return new Promise<any>((resolve, reject) => {
const readStream = this.s3.getObject(params).createReadStream();
// Error handling in read stream
readStream.on("error", (e) => {
console.error(e);
reject(e);
});
// Resolve only if we are done writing
writeStream.once('finish', () => {
resolve(keyName);
});
// pipe will automatically finish the write stream once done
readStream.pipe(writeStream);
});
}
}

Resources