Im uploading image files to my S3 bucket with multer S3. Everthing goes smooth, file does get uploaded but bucket name gets duplicated in returned FILE LOCATION:
Here's the file location is s3: CORRECT in S3
https://MYBUCKET.s3.eu-west-3.amazonaws.com/1669200254736-img-intro-bg.jpg
Here's what I get as file.location in my node app: Bucket name is doubled:
https://MYBUCKET.MYBUCKET.s3.eu-west-3.amazonaws.com/1669200254736-img-intro-bg.jpg'
HERE's MY APP CODE;
const { S3Client } = require('#aws-sdk/client-s3');
const multer = require('multer');
const multerS3 = require('multer-s3');
require('dotenv').config();
const credentials = {
region: process.env.region,
credentials: {
accessKeyId: process.env.accessKeyId,
secretAccessKey: process.env.secretAccessKey,
},
};
const s3 = new S3Client(credentials);
const imageFilter = (req, file, cb) => {
if (file.mimetype.startsWith('image')) {
cb(null, true);
} else {
cb('Please upload only images.', false);
}
};
const uploadFile = multer({
storage: multerS3({
s3: s3,
bucket: 'MYBUCKET',
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function (req, file, cb) {
cb(null, `${Date.now()}-img-${file.originalname}`);
},
}),
fileFilter: imageFilter,
limits: {
fieldSize: '50mb',
},
});
module.exports = uploadFile;
Thank you in advance for your support.
I've been searching online for a few days but of no avail.
Apparently the problem stems form underlying Amazon S3 sdk. The most recent s3 client sdk still has the issue.
Downgrade Amazon sdk to < 3.180.0, untill issue is fixed by AWS.
npm i #aws-sdk/client-s3#3.180.0
Link to Original Issue at Github
Related
I am getting this error while working on a node backend using typescript and this is the function for file upload to aws S3. Im new to using typescript so can anyone help me on this.
import AWS from "aws-sdk";
import multer from "multer";
import multerS3 from "multer-s3";
let S3 = new AWS.S3({
accessKeyId: process.env.AWS_KEY,
secretAccessKey: process.env.AWS_SECRET
})
const upload = multer({
storage: multerS3({
s3:S3, //error here
bucket: 'bucket-name',
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function (req, file, cb) {
cb(null, Date.now().toString())
}
})
})
export { upload }
So the problem is that you are creating an s3 client using the version 2 of the s3 ask instead of the v3 sdk: #aws-sdk/client-s3. The aws-sdk provides the version 2 client, #aws-sdk/client-s3 is part of the V3 javascript SDK.
Make sure to install npm i #aws-sdk/client-s3.
You can read about the client s3 sdk here
import AWS from "aws-sdk";
import { S3Client } from '#aws-sdk/client-s3';
import multer from "multer";
import multerS3 from "multer-s3";
const s3Config = new S3Client({
region: 'us-west-1',
credentials:{
accessKeyId:'',
secretAccessKey:''
}
})
const upload = multer({
storage: multerS3({
s3: s3Config,
bucket: 'bucket-name',
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function (req, file, cb) {
cb(null, Date.now().toString())
}
})
})
export { upload }
Need to upload my images into local storage and s3
My code:
const fileStorage = multer.diskStorage({
destination: function(req, file, cb) {
cb(null, "./public/uploads");
},
filename: function(req, file, cb) {
cb(null, file.originalname);
}
});
/** AWS catalog */
aws.config.update({
secretAccessKey: process.env.SECRET_KEY,
accessKeyId: process.env.ACCESS_KEY,
region: "us-east-1"
});
const s3 = new aws.S3();
const awsStorage = multerS3({
s3: s3,
bucket: process.env.BUCKET_NAME,
key: function(req, file, cb) {
console.log(file);
cb(null, file.originalname);
}
});
var upload = multer({ storage: awsStorage}).array('userPhoto',10);
router.post('/postimages',function(req,res)
{
upload(req,res,function(err) {
});
});
In this case, I can upload to either local storage or S3. I am unable to upload to both places. Please help to solve this issue.
I would suggest you try multer-s3 which will upload the file directly to s3 without saving to your local. if you want to save files to your Local you can do it as well using the same.
I have uploaded files to s3 upto 18Gb without any issues.
Here is the helpful link - https://www.npmjs.com/package/multer-s3
I have spent days trying to figure out how to post photos to my AWS bucket and have read a lot of similar questions on SO but nothing seems to be working and I almost always get an "unexpected field" error.
I have confirmed the libraries are the correct versions; added bodyParser; added region / signatureVersion; and have tried changing 'upload.single('image') a variety of ways.
In my Model I am trying to save the image url to a field name 'image-field' but using it doesn't change the error I am getting. I am only trying to have this work on my backend before attempting the FE and am using with PostMan.
Please help, I am losing my mind over here.
import express from 'express';
import aws from 'aws-sdk';
import multer from 'multer';
import multerS3 from 'multer-s3';
import bodyParser from 'body-parser';
const router = express.Router();
router.use(bodyParser.json())
aws.config.update({
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
region: 'us-west-2',
signatureVersion: 'v4'
})
const s3 = new aws.S3()
const upload = multer({
storage: multerS3({
s3: s3,
bucket: 'bucket-name',
acl: "public-read",
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function (req, file, cb) {
cb(null, Date.now().toString() + '-' + file.originalname);
},
}),
})
router.post('/', upload.single('image'), (req, res, next) => {
console.log('Uploaded!')
res.send(req.file)
})
export default router;
PostMan
enter image description here
Note: If it helps, I am including old code I was using successfully for my FE and BE to save images locally and then the urls on Mongo.
import path from 'path';
import express from 'express';
import multer from 'multer';
const router = express.Router();
const storage = multer.diskStorage({
destination(req, file, cb) {
cb(null, 'uploads/')
},
filename(req, file, cb) {
cb(null, `${file.fieldname}-${Date.now()}${path.extname(file.originalname)}`)
}
})
function checkFileType(file, cb) {
const filetypes = /jpg|jpeg|png/
const extname = filetypes.test(path.extname(file.originalname).toLowerCase())
const mimetype = filetypes.test(file.mimetype)
if(extname && mimetype) {
return cb(null, true)
} else {
cb('Images only!')
}
}
const upload = multer({
storage,
fileFilter: function(req, file, cb) {
checkFileType(file, cb)
}
})
router.post('/', upload.single('image'), (req, res) => {
res.send(`/${req.file.path}`)
})
export default router;
We did it! See below code below, but the update really pertains to changing the POST API from .single() to .any() and then setting fileFilter to remove any files you do not want to include. I will note that PostMan gives me an error on my route when I do post the file, but the image does successfully upload to the s3 bucket.
Said another way, .any() "Accepts all files that comes over the wire. An array of files will be stored in req.files". Since we know we are taking any file, we must use fileFilter to define which files we want to be able to post. Essentially, if you don't use fileFilter, you can post any file you want.
import express from 'express';
import aws from 'aws-sdk';
import multer from 'multer';
import multerS3 from 'multer-s3';
import path from 'path';
const router = express.Router();
aws.config.update({
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
region: 'us-west-2',
signatureVersion: 'v4'
})
const s3 = new aws.S3()
function checkFileType(file, cb) {
const filetypes = /jpg|jpeg|png/
const extname = filetypes.test(path.extname(file.originalname).toLowerCase())
const mimetype = filetypes.test(file.mimetype)
if(extname && mimetype) {
return cb(null, true)
} else {
cb('Images only!')
}
}
const upload = multer({
limits: { fileSize: 2000000 },
fileFilter: function(req, file, cb) {
checkFileType(file, cb)
},
storage: multerS3({
s3: s3,
bucket: 'recipebook-recipe-cover-images',
acl: "public-read",
contentType: multerS3.AUTO_CONTENT_TYPE,
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function (req, file, cb) {
cb(null, Date.now().toString() + '-' + file.originalname);
},
}),
})
router.post('/', upload.any())
export default router;
Got to do a crc32 checksum for a uploaded file before saving to s3 disk.
I am using multer and multer-s3 to upload a file to s3 which was easy to achive.
Here In middle trying to do a crc32 check during file upload. Which was partially completed The below code does the job of crc32 check perfectly but the uploaded file is not having all the chunks.
var AWS = require("aws-sdk");
const multer = require("multer");
const multerS3 = require("multer-s3");
const { crc32 } = require('crc');
AWS.config.update({
secretAccessKey: AWS_XXXXXX_SecretAccessKey,
accessKeyId: AWS_XXXXXX_AccessKeyID,
//region: 'us-east-1'
});
const s3 = new AWS.S3();
var upload = multer({
limits: { fileSize: 100000 },
storage: multerS3({
s3: s3,
bucket: constants.AWS_Cred.bucketName,
/* metadata: function (req, file, cb) {
cb(null, {fieldName: file.fieldname});
//tried moving stream logic here still no success
}, */
key: function (req, file, cb) {
console.log("KEY");
var buffer = new Buffer.alloc(0);
// const b = file.stream.pipe( new PassThrough()); -- tried to clone a stream here still failed
file.stream.on("data", function(data) {
buffer = Buffer.concat([buffer, data]);
});
file.stream.on('end', function() {
let fileCRCCheckSum = crc32(buffer).toString(16);
console.log(fileCRCCheckSum);
});
cb(null, constants.Firmware_Saved_Location + file.originalname);
},
}),
});
Router
router.post(
"/uploadSong",
upload.single("files"),
uploadController.DoesSomeDBEntriesHere
);
Will Take care of Controller which just saves other Entries to Db.
Happy to Hear Any Suggestions.
I have a problem with using multer and trying to upload an image to S3 (aws)
I watched a few tutorials, read docs but I cannot find the mistake.
const multer = require("multer");
const aws = require("aws-sdk");
const multerS3 = require("multer-s3");
aws.config.update({
secretAccessKey: process.env.AWS_SECRET_ACCESS,
accessKeyId: process.env.AWS_KEY_ID,
region: process.env.AWS_REGION
});
const awsS3 = new aws.S3();
const upload = multer({
storage: multerS3({
s3: awsS3,
bucket: "app-library-jaksa",
acl: "public-read",
metadata: function(req, file, cb) {
cb(null, {
fieldName: file.fieldname
});
},
key: function(req, file, cb) {
cb(null, Date.now().toString());
}
})
});
module.exports = upload;
And then I call:
const multer = require("../middleware/multer");
router.post("/uploadTest", multer.single("image"), (req, res) => {
res.status(200).send("Successful", req.file);
});
I expect to upload the image to S3 (of course, I have a bucket and access key), but instead I get
"Error: connect ETIMEDOUT (ip address)
at TCPConnectWrap.afterConnect [as oncomplete]"