I am uploading video file using s3 bucket in Nodejs.
when I upload file on s3 bucket, it is also uploaded in temp/ folder. So after few video upload temp folder is become full due to this I can not upload more video.
How to remove this video from temp folder after file successfully or not upload on s3 bucket in Nodejs code?
I have used below package for s3 bucket.
https://www.npmjs.com/package/s3
var s3 = new AWS.S3();
s3.abortMultipartUpload(params, function (err, data) {
if (err) console.log(err, err.stack); // USE BELOW CODE HERE
else console.log(data); // ALSO HERE
});
Here is example of how can you remove/delete file from system using NodeJS
// include node fs module
var fs = require('fs');
// delete file named 'sample.txt'
fs.unlink('sample.txt', function (err) {
if (err) throw err;
// if no error, file has been deleted successfully
console.log('File deleted!');
});
const multer = require("multer");
require("dotenv").config();
const aws = require("aws-sdk");
const multerS3 = require("multer-s3");
aws.config.update({
region: process.env.AWS_BUCKET_REGION,
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_KEY,
});
const BUCKET = process.env.AWS_BUCKET_NAME;
const s3 = new aws.S3();
// Multer config
module.exports = multer({
storage: multerS3({
bucket: BUCKET,
s3: s3,
contentType: multerS3.AUTO_CONTENT_TYPE,
acl: "public-read",
key: function (req, file, cb) {
cb(null, Date.now() + file.originalname); //use Date.now() for unique file keys
},
}),
});
Related
Im uploading image files to my S3 bucket with multer S3. Everthing goes smooth, file does get uploaded but bucket name gets duplicated in returned FILE LOCATION:
Here's the file location is s3: CORRECT in S3
https://MYBUCKET.s3.eu-west-3.amazonaws.com/1669200254736-img-intro-bg.jpg
Here's what I get as file.location in my node app: Bucket name is doubled:
https://MYBUCKET.MYBUCKET.s3.eu-west-3.amazonaws.com/1669200254736-img-intro-bg.jpg'
HERE's MY APP CODE;
const { S3Client } = require('#aws-sdk/client-s3');
const multer = require('multer');
const multerS3 = require('multer-s3');
require('dotenv').config();
const credentials = {
region: process.env.region,
credentials: {
accessKeyId: process.env.accessKeyId,
secretAccessKey: process.env.secretAccessKey,
},
};
const s3 = new S3Client(credentials);
const imageFilter = (req, file, cb) => {
if (file.mimetype.startsWith('image')) {
cb(null, true);
} else {
cb('Please upload only images.', false);
}
};
const uploadFile = multer({
storage: multerS3({
s3: s3,
bucket: 'MYBUCKET',
metadata: function (req, file, cb) {
cb(null, { fieldName: file.fieldname });
},
key: function (req, file, cb) {
cb(null, `${Date.now()}-img-${file.originalname}`);
},
}),
fileFilter: imageFilter,
limits: {
fieldSize: '50mb',
},
});
module.exports = uploadFile;
Thank you in advance for your support.
I've been searching online for a few days but of no avail.
Apparently the problem stems form underlying Amazon S3 sdk. The most recent s3 client sdk still has the issue.
Downgrade Amazon sdk to < 3.180.0, untill issue is fixed by AWS.
npm i #aws-sdk/client-s3#3.180.0
Link to Original Issue at Github
I upload videos and images using this code it was succesfull.
aws.config.update({
region: process.env.AWS_BUCKET_REGION,
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_KEY,
});
const BUCKET = process.env.AWS_BUCKET_NAME;
const s3 = new aws.S3();
// Multer config
module.exports = multer({
storage: multerS3({
bucket: BUCKET,
s3: s3,
contentType: multerS3.AUTO_CONTENT_TYPE,
acl: "public-read",
key: function (req, file, cb) {
cb(null, Date.now() + file.originalname); //use Date.now() for unique file keys
},
}),
});
and this code
const file =req.file
console.log(file)
let user = new Image({
name: req.body.name,
thumbnail:req.file.location,
});
now I want to upload my video and then the video screenshot is saved bucket
Got to do a crc32 checksum for a uploaded file before saving to s3 disk.
I am using multer and multer-s3 to upload a file to s3 which was easy to achive.
Here In middle trying to do a crc32 check during file upload. Which was partially completed The below code does the job of crc32 check perfectly but the uploaded file is not having all the chunks.
var AWS = require("aws-sdk");
const multer = require("multer");
const multerS3 = require("multer-s3");
const { crc32 } = require('crc');
AWS.config.update({
secretAccessKey: AWS_XXXXXX_SecretAccessKey,
accessKeyId: AWS_XXXXXX_AccessKeyID,
//region: 'us-east-1'
});
const s3 = new AWS.S3();
var upload = multer({
limits: { fileSize: 100000 },
storage: multerS3({
s3: s3,
bucket: constants.AWS_Cred.bucketName,
/* metadata: function (req, file, cb) {
cb(null, {fieldName: file.fieldname});
//tried moving stream logic here still no success
}, */
key: function (req, file, cb) {
console.log("KEY");
var buffer = new Buffer.alloc(0);
// const b = file.stream.pipe( new PassThrough()); -- tried to clone a stream here still failed
file.stream.on("data", function(data) {
buffer = Buffer.concat([buffer, data]);
});
file.stream.on('end', function() {
let fileCRCCheckSum = crc32(buffer).toString(16);
console.log(fileCRCCheckSum);
});
cb(null, constants.Firmware_Saved_Location + file.originalname);
},
}),
});
Router
router.post(
"/uploadSong",
upload.single("files"),
uploadController.DoesSomeDBEntriesHere
);
Will Take care of Controller which just saves other Entries to Db.
Happy to Hear Any Suggestions.
I have a problem with using multer and trying to upload an image to S3 (aws)
I watched a few tutorials, read docs but I cannot find the mistake.
const multer = require("multer");
const aws = require("aws-sdk");
const multerS3 = require("multer-s3");
aws.config.update({
secretAccessKey: process.env.AWS_SECRET_ACCESS,
accessKeyId: process.env.AWS_KEY_ID,
region: process.env.AWS_REGION
});
const awsS3 = new aws.S3();
const upload = multer({
storage: multerS3({
s3: awsS3,
bucket: "app-library-jaksa",
acl: "public-read",
metadata: function(req, file, cb) {
cb(null, {
fieldName: file.fieldname
});
},
key: function(req, file, cb) {
cb(null, Date.now().toString());
}
})
});
module.exports = upload;
And then I call:
const multer = require("../middleware/multer");
router.post("/uploadTest", multer.single("image"), (req, res) => {
res.status(200).send("Successful", req.file);
});
I expect to upload the image to S3 (of course, I have a bucket and access key), but instead I get
"Error: connect ETIMEDOUT (ip address)
at TCPConnectWrap.afterConnect [as oncomplete]"
When I try to use aws.s3(), everything works fine, my bucket is created with the appropriate key.
var s3 = new AWS.S3();
// Create a bucket and upload something into it
var bucketName = 'node-sdk-sample-' + uuid.v4();
var keyName = 'hello';
s3.createBucket({Bucket: bucketName}, function() {
var params = {Bucket: bucketName, Key: keyName,ACL: 'public-read'};
s3.putObject(params, function(err, data) {
if (err)
console.log(err)
else
console.log("Successfully uploaded data to " + bucketName + "/" + keyName);
});
});
But When I try to upload a multipart data with multer-3s , I get the error: Access Denied.
var aws = require('aws-sdk')
var uuid = require('node-uuid')
var multer= require('multer')
var multerS3 = require('multer-s3')
var s3 = new aws.S3()
var bucket = 'BucketWAR' + uuid.v4()
var upload = multer({
storage: multerS3({
s3: s3,
bucket: 'some-bucket',
acl: 'public-read',
key: function (req, file, cb) {
cb(null, Date.now().toString())
}
})
})
router.get('/index', indexController.idex)
router.post('/uploadFile', upload.single('file'), function (req, res, next) {
res.send("Uploaded!");
});
I can't fix this. Do I need Policy? But I didn't need it for putObject.
What is the mean difference?
Everything is correct except that I didn't use the right variable, silly problem, after defining bucket I didn't use it:
bucket: 'some-bucket'
I must put:
bucket: bucket
that I have defined by bucket = 'BucketWAR' + uuid.v4()