Amazon S3 bucket file upload response is not available - node.js

I'm using multer ,aws-sdk and multer-s3 packages along with express.
when users edit profile user may change profile picture /avatar or not.
I've passed multer object
multer({storage:
multer.memoryStorage()}).single('profileHeroImageEdit')
if a file with current request then I will upload the file to s3 bucket but I am not getting any response from upload_replace where req.file.location will provide the url of S3 bucket (file's location).
And Inside upload_replace I can get the file I am trying to upload(req.file) but I want the location of uploaded file to S3 bucket .
What I'm missing ? Help will be appreciated
router.put("/:id",multer({ storage:
multer.memoryStorage()}).single('profileHeroImageEdit'),
middleware.checkProfileOwnership,function(req, res){
if(isFileExists(req)==false){
delete req.body.profileHeroImage
}
else{
console.log('file has')
var upload_replace = multer({
limits:{
fileSize:MAX_FILE_SIZE,
files:1
},
storage: multerS3({
s3: photoBucket,
bucket: BucketName,
acl: 'public-read',
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function (req, file, cb) {
cb(null,Date.now().toString())
}
})
}).single('profileHeroImageEdit')
upload_replace(req, res, function (err,log) {
console.log('request log')
console.log(req.file.location)
console.log()
});
}
Profile.findByIdAndUpdate(req.params.id, req.body.profile, function(err, updatedProfile){
if (err){
res.redirect("/profiles");
} else {
res.redirect("/profiles/" + req.params.id);
}
});
});
function isFileExists(request){
if(request.file)
{
return true
}
else{
return false
}
}

I have whole code using multer and aws-sdk
include this files and npm install all
//aws s3 packages
const aws = require("aws-sdk");
const multerS3 = require("multer-s3");
const multer = require("multer");
const path = require("path");
then
//profile image upload start
const s3 = new aws.S3({
accessKeyId: "***",
secretAccessKey: "***",
Bucket: "***"
});
//Singe profile image upload
const profileImgUpload = multer({
storage: multerS3({
s3: s3,
bucket: "***",
acl: "public-read",
key: function(req, file, cb) {
cb(
null,
path.basename(file.originalname, path.extname(file.originalname)) +
"-" +
Date.now() +
path.extname(file.originalname)
);
}
}),
limits: { fileSize: 2000000 }, // In bytes: 2000000 bytes = 2 MB
fileFilter: function(req, file, cb) {
checkFileType(file, cb);
}
}).single("profileImage");
// getExtension of file by this function
function getExtension(filename) {
var parts = filename.split(".");
return parts[parts.length - 1];
}
//checkfile type of input function
function checkFileType(file, cb) {
const ext = getExtension(file.originalname);
switch (ext.toLowerCase()) {
case "jpeg":
case "jpg":
case "png":
case "gif":
return cb(null, true);
}
cb("Error: Images Only!");
}
router.post(
"/image",
passport.authenticate("jwt", { session: false }),
(req, res) => {
profileImgUpload(req, res, error => {
if (error) {
res.json({ error: error });
} else {
//here we can get req.body
const userDp = {};
//end of getting values
// If File not found then dont store anything
if (req.file !== undefined) userDp.dpUrl = req.file.location;
// Save the file name into database into profile model
User.findOne({ email: req.user.email }).then(user => {
if (user) {
// Update
User.findOneAndUpdate(
{ email: req.user.email },
{ $set: userDp },
{ new: true }
).then(user => res.json(user));
} else {
res.json({ msg: "not able not update data" });
}
});
}
});
}
);
3.need to send data from react frontend by using
const data = new Formdata();
data.append()
and by including headers also

Add a listener for when the OutgoingMessage to upload the photo to S3 is completed. The on-finished library is handy for this.
const onFinished = require('on-finished');
const print = process._rawDebug;
uploadReplace(req, null, function (err) {
onFinished(req, function () {
if (err) print(err);
print(req.file.location);
// Other things can be done here
})
});

Related

how to use sharp to resize image while uploading to s3 nodejs

I use multer to upload the images to s3. The user clicks on upload button in the frontend, and uploads the image, which is then uploaded to s3 with the helper multer backend.
But the images uploaded aren't optimised i.e if the user uploads a 4mb image, the image is uploaded with any compression. That indeed led to slow website frontend.
How do I optimise or compress the image using sharp ?
Code to upload to s3 (nodejs):
const s3 = new aws.S3({
accessKeyId: process.env.AWS_ACCESS_KEY,
secretAccessKey: process.env.AWS_SECERT_KEY,
});
//S3
const uploadS3 = multer({
storage: multerS3({
s3: s3,
bucket: "testbucket",
acl: "public-read",
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function (req, file, cb) {
cb(null, shortid.generate() + "-" + file.originalname);
},
}),
});
router.post("/poster/create", uploadS3.array("posterPictures"), createPoster);
CreatePoster is a function that returns the path of the image in s3:
exports.createPoster = (req, res) => {
let posterPictures = [];
if (req.files.length > 0) {
posterPictures = req.files.map((file) => {
return { img: file.location };
});
}
const { name, id } = req.body;
const poster = new Poster({
name: name,
slug: slugify(name),
id,
posterPictures,
});
poster.save((error, poster) => {
if (error) return res.status(400).json(error);
if (poster) {
return res.status(201).json({ poster, files: req.files });
}
});
};

NodeJs >Express > AWS > S3 > getting emplty res.send() intested of object probably a Promise delay

I am trying to work with, I am getting the object in console.log(data,"MongoData--->") but getting the empty object as a response in clint side
Promise {
{ _id: 5f44ed71ae6ebea2d14a9808,
name: 'https://rsboost.s3.amazonaws.com/sn3-1598352752273.png' } } 'MongoData--->'
RouteFile
router.post("/",async function(req, res, next){
ImgUpload(req, res,(error) => {
if (error) {
console.log("errors", error);
res.json({ error: error });
} else {
// If File not found
if (req.file === undefined) {
console.log("Error: No File Selected!");
res.json("Error: No File Selected");
} else {
// If Success
const imageLocation = req.file.location; // Save the file name into database into File model
const data = await CreateFile({name:imageLocation})
console.log(data,"MongoData--->")
res.json(data)
}
}
})
});
Aws Function returning correct data
const ImgUpload = multer({
storage: multerS3({
s3: s3,
bucket: "*****",
acl: "public-read",
key: function (req, file, cb) {
cb(
null,
path.basename(file.originalname, path.extname(file.originalname)) +
"-" +
Date.now() +
path.extname(file.originalname)
);
},
}),
limits: { fileSize: 20000000 }, // In bytes: 2000000 bytes = 20 MB
fileFilter: function (req, file, cb) {
checkFileType(file, cb);
},
}).single("fileImage");
Problem: getting the desired response with data but under-promise, Need help to write promise as I tried async-await but it's not working
Thanks in advance
solved it by myself > in route file > .then()=> is the solution for waiting for the promise, little thing I missed
const data = CreateFile({ name: imageLocation }).then((data) => {
res.send(data);
});

Anyway to unzip and upload array of files to AWS S3 using multer (multer s3) in Nodejs

Here's how I'm uploading files to AWS S3:
const config = (req) => {
limits: { fileSize: 20000000 }, // 20 MB
storage: multerS3({
acl: 'public-read',
bucket: `${Bucket}/${orderNumber}/orderFiles`,
key(request, file, cb) {
cb(null, `${file.originalname}`);
},
metadata(request, file, cb) {
cb(null, {});
},
s3,
}),
}
uploadFiles = async req => {
try {
const uploadOrder = multer(config(req)).array('files');
return new Promise((resolve, reject) => {
uploadOrder(req, {}, error => {
const fileArray = req.files;
let fileLocation;
fileArray.forEach(file => {
fileLocation = file.location;
});
resolve(fileArray[0].bucket);
});
});
} catch (error) {
return error;
}
};
And then:
router.post('/addFiles', auth.admin, async (req, res, next) => {
try {
await uploadFiles(req);
// handle rest of the things...
res.send();
} catch (error) {
next(error);
}
});
Is there any way I can check if the file is zip and then I can unzip and upload each individual file of the zip to s3?
I have wrapped my mind around the internet and could not get it working.

Best way to upload file with text inputs

I am trying to create an item where users will be able to upload a file image, that uploads the file to amazon s3, and returns a string for the image address that should be passed into mongoDB as a single document along with text inputs such as name of item, description, dates etc.
So far I have created a simple mongoDB model with a name and an image to test uploading a file with one text input, but when I try to test it in postman using form-urlencoded, there is no option to select a file, and if i try to test it in form-data, I get an empty string for the name input, but my code does read the file and return a string for the file, just not both together.
My code is:
setup file for amazon s3
const multer = require("multer");
const multerS3 = require("multer-s3");
const { secret_key, access_key, bucket_name } = require("../config/config");
aws.config.update({
secretAccessKey: secret_key,
accessKeyId: access_key,
region: "us-east-2"
});
const s3 = new aws.S3();
const fileFilter = (req, file, cb) => {
if (file.mimetype === "image/jpeg" || file.mimetype === "image/png") {
cb(null, true);
} else {
cb(new Error("Invalid Mime Type, only JPEG or PNG"), false);
}
};
const upload = multer({
fileFilter: fileFilter,
storage: multerS3({
s3: s3,
bucket: bucket_name,
acl: "public-read",
metadata: function(req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function(req, file, cb) {
cb(null, Date.now().toString());
}
})
});
module.exports = upload;
controller file for the upload at the target route
const upload = require("../services/file-upload");
const singleUpload = upload.single("img");
let uploadArtefact = (req, res) => {
const { name } = req.body;
singleUpload(req, res, err => {
if (err) {
return res.status(422).send({
errors: [{ title: "File Upload Error", detail: err.message }]
});
}
let artefact = new Artefact({
name: name,
img: req.file.location
});
if (!name) {
return res.json({
success: false,
error: "Invalid Inputs"
});
}
artefact.save(err => {
if (err) {
return res.json({ sucess: false, error: err });
}
return res.json({ success: true });
});
});
};
module.exports = uploadArtefact;
So I was wondering what would be the best approach to this? Is there a way to submit files with text in one request? I would preferably want to be able to find a way to send them both together.
Nevermind, I realised that multer can parse the text but i just needed to put the const { name } = req.body; inside the singleUpload{} section.

upload image to s3 from node js

I'm trying to upload a file to s3 but it not happening what I expected.
I create a file-helper.js in middleware, that is looks like below
const aws = require('aws-sdk');
const multer = require('multer');
const multerS3 = require('multer-s3');
aws.config.update({
accessKeyID:'XXXXXXXXXXXXXX',
SecretAccessKey:'XXXXXXXXXXXXXXXXXXXXXXXXXXX',
region:'ap-south-1'
});
const s3 = new aws.S3();
const fileFilter = (req, file, cb) => {
if (file.mimetype === 'image/jpeg' || file.mimetype === 'image/png') {
cb(null, true);
} else {
cb(new Error('Invalid file type, only JPEG and PNG is allowed!'), false);
}
}
const upload = multer({
fileFilter,
storage: multerS3({
s3,
bucket: 'demo.moveies.com',
acl: 'public-read',
metadata: function (req, file, cb) {
cb(null, {fieldName: file.fieldname});
},
key: function (req, file, cb) {
cb(null, Date.now().toString())
}
})
})
module.exports = upload;
and my controller file like below
const upload = require('../middleware/file-helper');
const imageUpload = upload.single('image');
exports.fileUpload = async(req,res)=>{
imageUpload(req, res, function(err, some) {
if (err) {
return res.status(422).send({errors: [{title: 'Image Upload Error', detail: err.message}] });
}
return res.json({'imageUrl': req.file.location});
});
}
when hit the API end point it is giving error
{
"errors": [
{
"title": "Image Upload Error",
"detail": "Missing credentials in config"
}
]
}
I'm not able to figure out where I went in wrong in my code. can one help me in this situation
There are typos in your config details. It should be accessKeyId not accessKeyID and secretAccessKey and not SecretAccessKey.
You have used the wrong key SecretAccessKey and accessKeyID, try changing it to secretAccessKey and accessKeyId.
aws.config.update({
accessKeyId:'XXXXXXXXXXXXXX',
secretAccessKey:'XXXXXXXXXXXXXXXXXXXXXXXXXXX',
region:'ap-south-1'
});

Resources