Need to upload my images into local storage and s3
My code:
const fileStorage = multer.diskStorage({
destination: function(req, file, cb) {
cb(null, "./public/uploads");
},
filename: function(req, file, cb) {
cb(null, file.originalname);
}
});
/** AWS catalog */
aws.config.update({
secretAccessKey: process.env.SECRET_KEY,
accessKeyId: process.env.ACCESS_KEY,
region: "us-east-1"
});
const s3 = new aws.S3();
const awsStorage = multerS3({
s3: s3,
bucket: process.env.BUCKET_NAME,
key: function(req, file, cb) {
console.log(file);
cb(null, file.originalname);
}
});
var upload = multer({ storage: awsStorage}).array('userPhoto',10);
router.post('/postimages',function(req,res)
{
upload(req,res,function(err) {
});
});
In this case, I can upload to either local storage or S3. I am unable to upload to both places. Please help to solve this issue.
I would suggest you try multer-s3 which will upload the file directly to s3 without saving to your local. if you want to save files to your Local you can do it as well using the same.
I have uploaded files to s3 upto 18Gb without any issues.
Here is the helpful link - https://www.npmjs.com/package/multer-s3
Related
Im uploading image files to my S3 bucket with multer S3. Everthing goes smooth, file does get uploaded but bucket name gets duplicated in returned FILE LOCATION:
Here's the file location is s3: CORRECT in S3
https://MYBUCKET.s3.eu-west-3.amazonaws.com/1669200254736-img-intro-bg.jpg
Here's what I get as file.location in my node app: Bucket name is doubled:
https://MYBUCKET.MYBUCKET.s3.eu-west-3.amazonaws.com/1669200254736-img-intro-bg.jpg'
HERE's MY APP CODE;
const { S3Client } = require('#aws-sdk/client-s3');
const multer = require('multer');
const multerS3 = require('multer-s3');
require('dotenv').config();
const credentials = {
region: process.env.region,
credentials: {
accessKeyId: process.env.accessKeyId,
secretAccessKey: process.env.secretAccessKey,
},
};
const s3 = new S3Client(credentials);
const imageFilter = (req, file, cb) => {
if (file.mimetype.startsWith('image')) {
cb(null, true);
} else {
cb('Please upload only images.', false);
}
};
const uploadFile = multer({
storage: multerS3({
s3: s3,
bucket: 'MYBUCKET',
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function (req, file, cb) {
cb(null, `${Date.now()}-img-${file.originalname}`);
},
}),
fileFilter: imageFilter,
limits: {
fieldSize: '50mb',
},
});
module.exports = uploadFile;
Thank you in advance for your support.
I've been searching online for a few days but of no avail.
Apparently the problem stems form underlying Amazon S3 sdk. The most recent s3 client sdk still has the issue.
Downgrade Amazon sdk to < 3.180.0, untill issue is fixed by AWS.
npm i #aws-sdk/client-s3#3.180.0
Link to Original Issue at Github
I am getting this error while working on a node backend using typescript and this is the function for file upload to aws S3. Im new to using typescript so can anyone help me on this.
import AWS from "aws-sdk";
import multer from "multer";
import multerS3 from "multer-s3";
let S3 = new AWS.S3({
accessKeyId: process.env.AWS_KEY,
secretAccessKey: process.env.AWS_SECRET
})
const upload = multer({
storage: multerS3({
s3:S3, //error here
bucket: 'bucket-name',
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function (req, file, cb) {
cb(null, Date.now().toString())
}
})
})
export { upload }
So the problem is that you are creating an s3 client using the version 2 of the s3 ask instead of the v3 sdk: #aws-sdk/client-s3. The aws-sdk provides the version 2 client, #aws-sdk/client-s3 is part of the V3 javascript SDK.
Make sure to install npm i #aws-sdk/client-s3.
You can read about the client s3 sdk here
import AWS from "aws-sdk";
import { S3Client } from '#aws-sdk/client-s3';
import multer from "multer";
import multerS3 from "multer-s3";
const s3Config = new S3Client({
region: 'us-west-1',
credentials:{
accessKeyId:'',
secretAccessKey:''
}
})
const upload = multer({
storage: multerS3({
s3: s3Config,
bucket: 'bucket-name',
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function (req, file, cb) {
cb(null, Date.now().toString())
}
})
})
export { upload }
I want to upload multiple images to my S3 bucket. What I wish to do is User can upload zip file of multiple Images. And from the back-end I want to Unzip the folder and place all the images into specific location in S3 bucket.
Currently I'm uploading single images using the following code.
let S3Instance = new S3({
accessKeyId: process.env.AWS_ID,
secretAccessKey: process.env.AWS_SECRET,
});
let Upload = multer({
storage: multers3({
s3: S3Instance,
bucket: process.env.BUCKET || "test-bucket",
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function (req, file, cb) {
cb(null, uuid() + path.extname(file.originalname));
},
}),
});
I want to achieve this using node.
Got to do a crc32 checksum for a uploaded file before saving to s3 disk.
I am using multer and multer-s3 to upload a file to s3 which was easy to achive.
Here In middle trying to do a crc32 check during file upload. Which was partially completed The below code does the job of crc32 check perfectly but the uploaded file is not having all the chunks.
var AWS = require("aws-sdk");
const multer = require("multer");
const multerS3 = require("multer-s3");
const { crc32 } = require('crc');
AWS.config.update({
secretAccessKey: AWS_XXXXXX_SecretAccessKey,
accessKeyId: AWS_XXXXXX_AccessKeyID,
//region: 'us-east-1'
});
const s3 = new AWS.S3();
var upload = multer({
limits: { fileSize: 100000 },
storage: multerS3({
s3: s3,
bucket: constants.AWS_Cred.bucketName,
/* metadata: function (req, file, cb) {
cb(null, {fieldName: file.fieldname});
//tried moving stream logic here still no success
}, */
key: function (req, file, cb) {
console.log("KEY");
var buffer = new Buffer.alloc(0);
// const b = file.stream.pipe( new PassThrough()); -- tried to clone a stream here still failed
file.stream.on("data", function(data) {
buffer = Buffer.concat([buffer, data]);
});
file.stream.on('end', function() {
let fileCRCCheckSum = crc32(buffer).toString(16);
console.log(fileCRCCheckSum);
});
cb(null, constants.Firmware_Saved_Location + file.originalname);
},
}),
});
Router
router.post(
"/uploadSong",
upload.single("files"),
uploadController.DoesSomeDBEntriesHere
);
Will Take care of Controller which just saves other Entries to Db.
Happy to Hear Any Suggestions.
What I want to achieve.
I want to compress my images that a user uploads to my server using TinyPng Api and then Upload them to my s3 Bucket.
Code
Here is my code to upload to s3 Bucket.
var upload = multer({
storage: multerS3({
s3: s3,
bucket: 'thecuratorsbucket',
metadata: function (req, file, cb) {
cb(null, {fieldName: file.fieldname});
},
key: function (req, file, cb) {
const name ="images/" + file.originalname.toLowerCase().split(' ').join('-');
const ext = MIME_TYPE_MAP[file.mimetype];
cb(null, name + '-' + Date.now()+ '.' + ext);
// cb(null, Date.now().toString())
}
})
})
router.post('/add-product',checkAuth,upload.array("images[]", 12), adminController.postAddProduct);
TinyPng Code
const source = tinify.fromFile("large.jpg");
source.store({
service: "s3",
aws_access_key_id: "AKIAIOSFODNN7EXAMPLE",
aws_secret_access_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
region: "us-west-1",
headers: {
Cache-Control: "public, max-age=31536000"
},
path: "example-bucket/my-images/optimized.jpg"
});
link to api reference
https://tinypng.com/developers/reference/nodejs
Problem
I do not understand how to use the tinyPng code as middelware and upload to my s3 bucket.