Multer S3 Ajax upload - node.js

I am having trouble adjusting my code to allow ajax uploads. I have the direct upload to S3 working but would like to use ajax to show upload progress.
js
$('#upload-input').on('change', function(){
var files = $(this).get(0).files;
if (files.length > 0){
var formData = new FormData();
for (var i = 0; i < files.length; i++) {
var file = files[i];
formData.append('uploads[]', file, file.name);
}
$.ajax({
type: 'POST',
processData: false,
contentType: false,
data: formData,
url: '/upload',
success : function(data){
getDetail();
console.log('picture uploaded ', data);
},
xhr: function() {
var xhr = new XMLHttpRequest();
xhr.upload.addEventListener('progress', function(evt) {
if (evt.lengthComputable) {
var percentComplete = evt.loaded / evt.total;
percentComplete = parseInt(percentComplete * 100);
$('.progress-bar').text(percentComplete + '%');
$('.progress-bar').width(percentComplete + '%');
if (percentComplete === 100) {
$('.progress-bar').html('Done');
}
}
}, false);
return xhr;
}
});
}
});
backend
setting up the variables
var aws = require('aws-sdk');
aws.config.loadFromPath('./data/s3config.json');
var multer = require('multer');
var multerS3 = require('multer-s3');
var s3 = new aws.S3({});
var upload = multer({
storage: multerS3({
s3: s3,
bucket: 'roe.pictures',
acl: 'public-read',
metadata: function (req, file, cb) {
cb(null, {fieldName: file.fieldname});
},
key: function (req, file, cb) {
cb(null, Date.now().toString())
}
})
});
Setting up post call
The issue is that I don't know how to add the files to the 'upload.array' parameter. If I load the files directly from the html form, I can just use the name of the input field. Since there is no 'req' object, I can't use req.body. Do I have to wrap this into another function?
Thanks!!
app.post('/upload', upload.array(this.body, 3), function(req, res, next) {
var newPics = [];
if(req.files) {
req.files.forEach(function(p) {
newPics.push({key : p.key, originalName : p.originalname, mimeType : p.mimetype, size : p.size})
});
Room.update({_id : req.signedCookies.prop},
{ $push: { pPics : { $each : newPics }}})
.exec(function(err, r) {
if(err) return err;
console.log('Successfully uploaded ' + req.files.length + ' files! with result ' + r);
res.redirect(req.get('referer'));
});
} else {
console.log('nothing to upload');
res.redirect(req.get('referer'));
}
});

Related

How to passing params whit file upload and get it in node js Ionic 3

Server Node JS file upload whit multer :
var multer = require('multer');
var storage = multer.diskStorage({
destination: function (req, file, callBack) {
console.log(req.body);
callBack(null, 'public/images/');
}, filename: function (req, file, callBack) {
callBack(null, 'someName');
}
});
var upload = multer({ storage: storage }).single('file');
outer.post('/uploads',uploads.single('file'),
function(req, res) {
//console.log(JSON.stringify(req.files.file));
console.log('/////////////////////////////////');
console.log(JSON.stringify(req.body));
console.log(JSON.stringify(req.params))
console.log(req.files.file)
var base64Data = req.files.file.data.toString('base64').replace(/^data:image\/jpeg;base64,/, "");
//console.log(base64Data);
var time = Date.now().toString()+"out.jpeg" ;
fs.writeFile('public/images'+"/"+time, base64Data, 'base64', function(err) {
console.log(err +' §§§§§§§§§§§' );
});
res.status(204).end();
}
);
Frontend Ionic 3 :
var name = "upload";
var par = {
token : this.myToken ,
contenu_titre : this.contenu_titre ,
contenu_text : this.contenu_text ,
contenu_type : 'img' ,
} ;
let option: FileUploadOptions = {
params : par,
fileKey:'file',
mimeType:'image/jpeg',
httpMethod:'POST',
fileName:'user_step4#'+name
};
let loader = this.loadingCtrl.create({
content: "Uploading...."
});
loader.present();
const fileTransfer:FileTransferObject = this.transfer.create();
// console.log('filename'+this.curfilename);
fileTransfer.upload(this.photo ,encodeURI(this.linkPic+"/publication/uploads"),option).then((result)=>
{
alert('uploaded')
console.log('success');
console.log(result);
loader.dismiss();
}).catch(error=>{
loader.dismiss();
alert(error);
console.log('uploaderror');
console.log(error.message);
});
}
I want to get the Params values in the server ,
every time is undefined or null
i want to get some params whit my file to save the params in my database.
the upload is working fine but in not getting the params
req.body return {}
and the req.file return onlu the filename and the file
You can use params with FileTransferOptions to send parameters along with your file.
Use mimeType: "multipart/form-data" to specify that this request contains multiple type of data
var options =
{
fileKey: "file",
fileName: filename,
chunkedMode: false,
mimeType: "multipart/form-data",
params: {
"full_name": this.userForm.value.u_name,
"email": this.userForm.value.u_email,
"country": this.userForm.value.u_country,
"state": this.userForm.value.u_state,
"city": this.userForm.value.u_city,
"pincode": this.userForm.value.u_pincode,
"fax": this.userForm.value.u_fax,
"address": this.userForm.value.u_address,
}
};
transfer.upload(this.photo, this.yourURL, options).then(data =>
{
//Handle the response from the server
}

Upload photo into a folder in nodejs

I want to upload an image in a folder using nodejs but i don't know how to do it
Here is my insert in my ImageDao
exports.insert = function(data, callback){
console.log("in imagesDao insert");
var query = " insert into " + tableName + " (url,ajoute_par)";
query = query + " values(?,?);";
var values = [data.url , data.ajoute_par];
// var values = [encodeURIComponent(data.url) , data.ajoute_par];
database.execute(query, values, function(){
callback();
});
}
And here is my image controller
// insert
exports.write = function(request, response){
console.log("in images write");
// Get the data.
var postData = "";
request.on('data', function(data){ // request.on is a listener. Call when data can be read
postData = postData + data;
});
request.on('end', function(){ // Called when data has been read
var dataObj = JSON.parse(postData);
dao.insert(dataObj, function(){
send(response, '{"write result" : "Inserted successfuly"}');
});
});
}
To upload files you can use multer module of nodejs. https://github.com/expressjs/multer
images_storage: function () {
return multer.diskStorage({
destination: function (req, file, cb) {
mkdirp(Config.upload_images_path, function (err) {
});
cb(null, Config.upload_images_path)
}
,
filename: function (req, file, cb) {
var getFileExt = function (fileName) {
var fileExt = fileName.split(".");
if (fileExt.length === 1 || (fileExt[0] === "" && fileExt.length === 2)) {
return "";
}
return fileExt.pop();
}
cb(null, Date.now() + '.' + getFileExt(file.originalname))
}
});
},
// Image uploading
const fs = require('fs');
const multer = require('multer');
const Uploads = multer({
storage: utility.images_storage(),
fileFilter: function (req, file, cb) {
if (Config.image_format_arr.indexOf(file.mimetype))
cb(null, true);
else
cb(null, false);
}
});
//And in your route you can use the upload function
router.post('/upload-logo', Uploads.single('school_logo'), function (req, res, next) {
var school_id = req.body.school_id;
var result = {flag: false, message: "Error Occurred! in saving school logo."};
console.log("REQUEST FILES " + req.file);
// Save School Logo
if (typeof req.file != 'undefined' && req.file.size > 0) {
School.findByIdAndUpdate(school_id, {
$set: {
school_logo: req.file.filename
}
}, {'new': true}, function (err, school_details) {
console.log("school_details " + school_details);
if (!err && school_details) {
result.flag = true;
result.message = "School logo has been successfully updated";
result.path = '/uploads/images/' + school_details.school_logo;
//req.session.school_details = school_details;
utility.upgradeSchoolLogoSessionValue(school_details, false, function (updated_school_details) {
console.log("BEFOR SESSION IS UPDATED" + JSON.stringify(req.session.school_details));
req.session.school_details = updated_school_details;
console.log("SESSION IS UPDATED" + JSON.stringify(req.session.school_details));
});
console.log("FILE NAME IS THIS " + req.file.filename);
}
res.json(JSON.stringify(result));
});
}
else {
res.json(JSON.stringify(result));
}
});

Video Upload using formidable in Nodejs, Error found: post 404 / 502

I'm uploading video file from local to server and then I'll be uploading it to cdn,
the issue i'm facing is my code is running well on local but its not working when i patch it to server.
Here is my code
commonJs
$("#uploadVideo").click(function (e) {
var reader = new FileReader();
var fileInput = document.getElementById('Videofile');
var previewUrl = window.URL.createObjectURL(fileInput.files[0]);
$(".video").attr("src", previewUrl);
var videotype = "video/mp4";
var file_data = $("#Videofile").prop("files")[0];
if (!file_data.type.match(videotype)) {
return "alert('Please upload mp4 files')"
} else {
var metadata = {
'content-type': 'video/mp4',
'size': file_data.size,
'uploaded': new Date(),
}
reader.onload = function (e) {
$("file_data").text("File Content: " + reader.result); // Show the file content
}
reader.readAsBinaryString(file_data);
file_data.onloadedmetadata = function () {
alert("Meta data for audio loaded");
};
};
var form_data = new FormData();
form_data.append("file", file_data)
form_data.append("metdata", metadata)
for (var key of form_data.entries()) {
console.log(key[0] + ', ' + key[1]);
}
if (form_data != undefined) {
$.ajax({
type: "post",
contentType: false,
processData: false,
url: "/api/recordvideo",
data: form_data,
dataType: 'json',
success: function (result) {
if (result) {
$(".video").attr("src", result.videolink);
alert("Successfully Uploaded Video");
console.log("Successfully Uploaded Video");
} else {
alert("Error on Uploading Video");
console.log("Error on Uploading Video");
}
},
error: function (err) {
console.log("error");
}
});
}
e.preventDefault();
e.stopPropagation();
});
ServerSide
app.post('/api/recordvideo',Api.recordvideo);
var Upload = require('gcs-resumable-upload');
ApiService.recordvideo = function (req, res) {
var db = req.db;
console.log("came in cloudupload");
var form = new formidable.IncomingForm();
var filesdata;
form.keepExtensions = true;
form.multiples = false;
form.on('fileBegin', function (name, file){
file.path = 'public/demo/' + file.name;
console.log("fileBegin: " + JSON.stringify(file));
});
form.on('file', function (name, file){
console.log('Uploaded ' + JSON.stringify(file));
var path = file.path;
console.log("came in cloud3 :" + JSON.stringify(path));
});
form.parse(req, function (err, fields, files) {
console.log("came in cloud0" + JSON.stringify(files));
filesdata = files;
});
console.log("came in cloud2");
form.on('end', function (fields, files) {
var userid = appconfig.ObjectID(appconfig.decrypt(req.signedCookies['gid']));
var path = this.openedFiles[0].path;
console.log("came in cloud3 :" + JSON.stringify(path));
fs.createReadStream(path)
.pipe(Upload.upload({ bucket: '******', file: path, metadata: { contentType: this.openedFiles[0].type } }))
.on('finish', function (response) {
console.log("Successfully Uploaded Video :" + JSON.stringify(response));
res.send({ "status": false, "videolink": "https://****/****/" + filesdata.file.name });
});
});
//res.send({ "status": false, "err": null });
}
At start atleast it was uploading to server folder & then in chrome developers tool it used to give response: {readystate : 4, . . . }
And now, I made some changes then it doesnt even hit my api, After few seconds it gives error in chrome developer tools 404() / 502 ()
Well, I got the solution, Previously I was using gcs-resumable-upload module to upload, but now I tried with '#google-cloud/storage' module through which I was able to upload upto 9mb.
const Storage = require('#google-cloud/storage');
var db = req.db;
console.log("came in cloudupload");
var form = new formidable.IncomingForm();
var filesdata;
form.keepExtensions = true;
form.multiples = false;
form.parse(req, function (err, fields, files) {
filesdata = files;
});
form.on('end', function (fields, files) {
var userid = appconfig.ObjectID(appconfig.decrypt(req.signedCookies['gid']));
var path = this.openedFiles[0].path;
const storage = new Storage({
keyFilename: 'gcloudcred.json'
});
const myBucket = storage.bucket('onfvideo');
myBucket.upload(path).then((resp) => {
console.log('uploaded to' + resp);
res.send({ "status": true, "err": null });
}).catch(err => {
console.error('ERROR:', err);
res.send({ "status": false, "err": null });
});
});
};
The Limitation of 9mb I was facing due to .netframework data-transfer limit which i was able to resolve using
<system.web>
<customErrors mode="Off"/>
<httpRuntime targetFramework="4.5" maxRequestLength="7483648" />
</system.web>
Method 2: Using xhr calling RestApi
1. Generated Access token using google-auto-auth module
2. XMLHttpRequest
var fileInput = $("#Videofile").prop("files")[0];
var url = "https://www.googleapis.com/upload/storage/v1/b/bucketname/o?uploadType=media&name=" + fileInput.name;
var http = new XMLHttpRequest();
http.open('POST', url, true);
http.setRequestHeader('Content-type', 'video/mp4');
http.setRequestHeader("Authorization", "Bearer " + token);
http.send(fileInput);
http.onprogress = function (ev) {
if (ev.lengthComputable) {
var percentage = Math.round((ev.loaded / ev.total) * 100);
console.log("percent " + percentage + '%');
}else {
console.log("Unable to compute progress information since the total size is unknown");
}
}
http.onloadstart = function (ev) {console.log("start")}
http.onloadend = function (ev) {}
http.onreadystatechange = function () {
if (http.readyState == 4 && http.status == 200) {
var response = JSON.parse(http.responseText);
alert("Successfully Uploaded Video");
}
}

Is it possible to send a file from a client to restler.file?

Normally restler.post() takes a restler.file(...) object to send a POST. This works if you have the file path on the server.
For example:
//#############################################################################
// Video upload from file system on node console server
//#############################################################################
function(done){
var fileStats = fs.statSync(videoFileName);
var fileSizeInBytes = fileStats["size"];
var data = {
'name': objectVideoName,
'type':'video',
'content': 'application/mp4',
'file': restler.file(videoFileName, null, fileSizeInBytes, null, "video" )
};
restler.post('https://api.astra.io/v0/bucket/'+bucketName+'/object', {
multipart: true,
data : data,
headers : { "Astra-Secret": astraSecret }
}).on('complete', function(response) {
console.log('Upload video to bucket: \n', response);
done(null);
});
},
How can I POST the file from the client directly without saving it on the server?
I'm trying with busboy.
//#############################################################################
// Video upload from client then POST to cdn
//#############################################################################
app.post('/addVideo', function (req, res) {
var video = {};
var busboy = new Busboy({headers: req.headers});
busboy.on('file', function (fieldname, file, filename, encoding, mimetype) {
var ext = mimetype.split('/').pop();
var token = generateToken(crypto);
var fileSizeInBytes = 900000;
filename = token + ext;
var videoName = filename;
var bucketName = 'sampleVideos';
var data = {
'name': videoName,
'type':'video',
'content': 'application/mp4',
'file': ***** WHAT GOES HERE ******
};
restler.post('https://api.astra.io/v0/bucket/'+bucketName+'/object', {
multipart: true,
data : data,
headers : { "Astra-Secret": astraSecret }
}).on('complete', function(response) {
console.log('Upload video to bucket: \n', response);
res.send("200");
});
});
busboy.on('field', function (fieldname, val, fieldnameTruncated, valTruncated) {
video[fieldname] = val;
});
busboy.on('finish', function () {
console.log('busboy finished');
});
req.pipe(busboy);
});
There are two helper functions exported by restler to help create your data options. You don't use either and specify a JS Object literal. restler.data() is what you want to use
data: function(filename, contentType, data) {
return new Data(filename, contentType, data);
}
So instead of
var data = {
'name': videoName,
'type':'video',
'content': 'application/mp4',
'file': ***** WHAT GOES HERE ******
};
in your example you would do this as it allows you to use arbitrary content instead of a file name
var data = restler.data(videoName, 'application/mp4', file);

Upload a file to Amazon S3 with NodeJS

I ran into a problem while trying to upload a file to my S3 bucket. Everything works except that my file paramters do not seem appropriate. I am using Amazon S3 sdk to upload from nodejs to s3.
These are my routes settings:
var multiparty = require('connect-multiparty'),
multipartyMiddleware = multiparty();
app.route('/api/items/upload').post(multipartyMiddleware, items.upload);
This is items.upload() function:
exports.upload = function(req, res) {
var file = req.files.file;
var s3bucket = new AWS.S3({params: {Bucket: 'mybucketname'}});
s3bucket.createBucket(function() {
var params = {
Key: file.name,
Body: file
};
s3bucket.upload(params, function(err, data) {
console.log("PRINT FILE:", file);
if (err) {
console.log('ERROR MSG: ', err);
} else {
console.log('Successfully uploaded data');
}
});
});
};
Setting Body param to a string like "hello" works fine. According to doc, Body param must take (Buffer, Typed Array, Blob, String, ReadableStream) Object data. However, uploading a file object fails with the following error message:
[Error: Unsupported body payload object]
This is the file object:
{ fieldName: 'file',
originalFilename: 'second_fnp.png',
path: '/var/folders/ps/l8lvygws0w93trqz7yj1t5sr0000gn/T/26374-7ttwvc.png',
headers:
{ 'content-disposition': 'form-data; name="file"; filename="second_fnp.png"',
'content-type': 'image/png' },
ws:
{ _writableState:
{ highWaterMark: 16384,
objectMode: false,
needDrain: true,
ending: true,
ended: true,
finished: true,
decodeStrings: true,
defaultEncoding: 'utf8',
length: 0,
writing: false,
sync: false,
bufferProcessing: false,
onwrite: [Function],
writecb: null,
writelen: 0,
buffer: [],
errorEmitted: false },
writable: true,
domain: null,
_events: { error: [Object], close: [Object] },
_maxListeners: 10,
path: '/var/folders/ps/l8lvygws0w93trqz7yj1t5sr0000gn/T/26374-7ttwvc.png',
fd: null,
flags: 'w',
mode: 438,
start: undefined,
pos: undefined,
bytesWritten: 261937,
closed: true },
size: 261937,
name: 'second_fnp.png',
type: 'image/png' }
Any help will be greatly appreciated!
So it looks like there are a few things going wrong here. Based on your post it looks like you are attempting to support file uploads using the connect-multiparty middleware. What this middleware does is take the uploaded file, write it to the local filesystem and then sets req.files to the the uploaded file(s).
The configuration of your route looks fine, the problem looks to be with your items.upload() function. In particular with this part:
var params = {
Key: file.name,
Body: file
};
As I mentioned at the beginning of my answer connect-multiparty writes the file to the local filesystem, so you'll need to open the file and read it, then upload it, and then delete it on the local filesystem.
That said you could update your method to something like the following:
var fs = require('fs');
exports.upload = function (req, res) {
var file = req.files.file;
fs.readFile(file.path, function (err, data) {
if (err) throw err; // Something went wrong!
var s3bucket = new AWS.S3({params: {Bucket: 'mybucketname'}});
s3bucket.createBucket(function () {
var params = {
Key: file.originalFilename, //file.name doesn't exist as a property
Body: data
};
s3bucket.upload(params, function (err, data) {
// Whether there is an error or not, delete the temp file
fs.unlink(file.path, function (err) {
if (err) {
console.error(err);
}
console.log('Temp File Delete');
});
console.log("PRINT FILE:", file);
if (err) {
console.log('ERROR MSG: ', err);
res.status(500).send(err);
} else {
console.log('Successfully uploaded data');
res.status(200).end();
}
});
});
});
};
What this does is read the uploaded file from the local filesystem, then uploads it to S3, then it deletes the temporary file and sends a response.
There's a few problems with this approach. First off, it's not as efficient as it could be, as for large files you will be loading the entire file before you write it. Secondly, this process doesn't support multi-part uploads for large files (I think the cut-off is 5 Mb before you have to do a multi-part upload).
What I would suggest instead is that you use a module I've been working on called S3FS which provides a similar interface to the native FS in Node.JS but abstracts away some of the details such as the multi-part upload and the S3 api (as well as adds some additional functionality like recursive methods).
If you were to pull in the S3FS library your code would look something like this:
var fs = require('fs'),
S3FS = require('s3fs'),
s3fsImpl = new S3FS('mybucketname', {
accessKeyId: XXXXXXXXXXX,
secretAccessKey: XXXXXXXXXXXXXXXXX
});
// Create our bucket if it doesn't exist
s3fsImpl.create();
exports.upload = function (req, res) {
var file = req.files.file;
var stream = fs.createReadStream(file.path);
return s3fsImpl.writeFile(file.originalFilename, stream).then(function () {
fs.unlink(file.path, function (err) {
if (err) {
console.error(err);
}
});
res.status(200).end();
});
};
What this will do is instantiate the module for the provided bucket and AWS credentials and then create the bucket if it doesn't exist. Then when a request comes through to upload a file we'll open up a stream to the file and use it to write the file to S3 to the specified path. This will handle the multi-part upload piece behind the scenes (if needed) and has the benefit of being done through a stream, so you don't have to wait to read the whole file before you start uploading it.
If you prefer, you could change the code to callbacks from Promises. Or use the pipe() method with the event listener to determine the end/errors.
If you're looking for some additional methods, check out the documentation for s3fs and feel free to open up an issue if you are looking for some additional methods or having issues.
I found the following to be a working solution::
npm install aws-sdk
Once you've installed the aws-sdk , use the following code replacing values with your where needed.
var AWS = require('aws-sdk');
var fs = require('fs');
var s3 = new AWS.S3();
// Bucket names must be unique across all S3 users
var myBucket = 'njera';
var myKey = 'jpeg';
//for text file
//fs.readFile('demo.txt', function (err, data) {
//for Video file
//fs.readFile('demo.avi', function (err, data) {
//for image file
fs.readFile('demo.jpg', function (err, data) {
if (err) { throw err; }
params = {Bucket: myBucket, Key: myKey, Body: data };
s3.putObject(params, function(err, data) {
if (err) {
console.log(err)
} else {
console.log("Successfully uploaded data to myBucket/myKey");
}
});
});
I found the complete tutorial on the subject here in case you're looking for references ::
How to upload files (text/image/video) in amazon s3 using node.js
Or Using promises:
const AWS = require('aws-sdk');
AWS.config.update({
accessKeyId: 'accessKeyId',
secretAccessKey: 'secretAccessKey',
region: 'region'
});
let params = {
Bucket: "yourBucketName",
Key: 'someUniqueKey',
Body: 'someFile'
};
try {
let uploadPromise = await new AWS.S3().putObject(params).promise();
console.log("Successfully uploaded data to bucket");
} catch (e) {
console.log("Error uploading data: ", e);
}
Using aws SDK v3
npm install #aws-sdk/client-s3
Upload code
import { S3Client, PutObjectCommand } from "#aws-sdk/client-s3";
/**
* advisable to save your AWS credentials and configurations in an environmet file. Not inside the code
* AWS lib will automatically load the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY if available in your environment
*/
const s3Client = new S3Client({ region: process.env.AWS_S3_REGION });
/**
* upload a file
* #param file the file object to be uploaded
* #param fileKey the fileKey. could be separated with '/' to nest the file into a folder structure. eg. members/user1/profile.png
*/
export function uploadFile(file, fileKey){
s3Client.send(new PutObjectCommand({
Bucket: process.env.MY_AWS_S3_BUCKET,
Key: fileKey,
Body: file
}));
}
And if you want to download
import { GetObjectCommand } from "#aws-sdk/client-s3";
/**
* download a file from AWS and send to your rest client
*/
app.get('/download', function(req, res, next){
var fileKey = req.query['fileKey'];
var bucketParams = {
Bucket: 'my-bucket-name',
Key: fileKey,
};
res.attachment(fileKey);
var fileStream = await s3Client.send(new GetObjectCommand(bucketParams));
// for TS you can add: if (fileStream.Body instanceof Readable)
fileStream.Body.pipe(res)
});
Uploading a file to AWS s3 and sending the url in response for accessing the file.
Multer is a node.js middleware for handling multipart/form-data, which is primarily used for uploading files. It is written on top of busboy for maximum efficiency. check this npm module here.
When you are sending the request, make sure the headers, have Content-Type is multipart/form-data.
We are sending the file location in the response, which will give the url, but if you want to access that url, make the bucket public or else you will not be able to access it.
upload.router.js
const express = require('express');
const router = express.Router();
const AWS = require('aws-sdk');
const multer = require('multer');
const storage = multer.memoryStorage()
const upload = multer({storage: storage});
const s3Client = new AWS.S3({
accessKeyId: 'your_access_key_id',
secretAccessKey: 'your_secret_access_id',
region :'ur region'
});
const uploadParams = {
Bucket: 'ur_bucket_name',
Key: '', // pass key
Body: null, // pass file body
};
router.post('/api/file/upload', upload.single("file"),(req,res) => {
const params = uploadParams;
uploadParams.Key = req.file.originalname;
uploadParams.Body = req.file.buffer;
s3Client.upload(params, (err, data) => {
if (err) {
res.status(500).json({error:"Error -> " + err});
}
res.json({message: 'File uploaded successfully','filename':
req.file.originalname, 'location': data.Location});
});
});
module.exports = router;
app.js
const express = require('express');
const app = express();
const router = require('./app/routers/upload.router.js');
app.use('/', router);
// Create a Server
const server = app.listen(8080, () => {
console.log("App listening at 8080");
})
Upload CSV/Excel
const fs = require('fs');
const AWS = require('aws-sdk');
const s3 = new AWS.S3({
accessKeyId: XXXXXXXXX,
secretAccessKey: XXXXXXXXX
});
const absoluteFilePath = "C:\\Project\\test.xlsx";
const uploadFile = () => {
fs.readFile(absoluteFilePath, (err, data) => {
if (err) throw err;
const params = {
Bucket: 'testBucket', // pass your bucket name
Key: 'folderName/key.xlsx', // file will be saved in <folderName> folder
Body: data
};
s3.upload(params, function (s3Err, data) {
if (s3Err) throw s3Err
console.log(`File uploaded successfully at ${data.Location}`);
debugger;
});
});
};
uploadFile();
Works for me :)
const fileContent = fs.createReadStream(`${fileName}`);
return new Promise(function (resolve, reject) {
fileContent.once('error', reject);
s3.upload(
{
Bucket: 'test-bucket',
Key: `${fileName + '_' + Date.now().toString()}`,
ContentType: 'application/pdf',
ACL: 'public-read',
Body: fileContent
},
function (err, result) {
if (err) {
reject(err);
return;
}
resolve(result.Location);
}
);
});```
var express = require('express')
app = module.exports = express();
var secureServer = require('http').createServer(app);
secureServer.listen(3001);
var aws = require('aws-sdk')
var multer = require('multer')
var multerS3 = require('multer-s3')
aws.config.update({
secretAccessKey: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
accessKeyId: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
region: 'us-east-1'
});
s3 = new aws.S3();
var upload = multer({
storage: multerS3({
s3: s3,
dirname: "uploads",
bucket: "Your bucket name",
key: function (req, file, cb) {
console.log(file);
cb(null, "uploads/profile_images/u_" + Date.now() + ".jpg"); //use
Date.now() for unique file keys
}
})
});
app.post('/upload', upload.single('photos'), function(req, res, next) {
console.log('Successfully uploaded ', req.file)
res.send('Successfully uploaded ' + req.file.length + ' files!')
})
Thanks to David as his solution helped me come up with my solution for uploading multi-part files from my Heroku hosted site to S3 bucket. I did it using formidable to handle incoming form and fs to get the file content. Hopefully, it may help you.
api.service.ts
public upload(files): Observable<any> {
const formData: FormData = new FormData();
files.forEach(file => {
// create a new multipart-form for every file
formData.append('file', file, file.name);
});
return this.http.post(uploadUrl, formData).pipe(
map(this.extractData),
catchError(this.handleError));
}
}
server.js
app.post('/api/upload', upload);
app.use('/api/upload', router);
upload.js
const IncomingForm = require('formidable').IncomingForm;
const fs = require('fs');
const AWS = require('aws-sdk');
module.exports = function upload(req, res) {
var form = new IncomingForm();
const bucket = new AWS.S3(
{
signatureVersion: 'v4',
accessKeyId: process.env.AWS_ACCESS_KEY_ID,
secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
region: 'us-east-1'
}
);
form.on('file', (field, file) => {
const fileContent = fs.readFileSync(file.path);
const s3Params = {
Bucket: process.env.AWS_S3_BUCKET,
Key: 'folder/' + file.name,
Expires: 60,
Body: fileContent,
ACL: 'public-read'
};
bucket.upload(s3Params, function(err, data) {
if (err) {
throw err;
}
console.log('File uploaded to: ' + data.Location);
fs.unlink(file.path, function (err) {
if (err) {
console.error(err);
}
console.log('Temp File Delete');
});
});
});
// The second callback is called when the form is completely parsed.
// In this case, we want to send back a success status code.
form.on('end', () => {
res.status(200).json('upload ok');
});
form.parse(req);
}
upload-image.component.ts
import { Component, OnInit, ViewChild, Output, EventEmitter, Input } from '#angular/core';
import { ApiService } from '../api.service';
import { MatSnackBar } from '#angular/material/snack-bar';
#Component({
selector: 'app-upload-image',
templateUrl: './upload-image.component.html',
styleUrls: ['./upload-image.component.css']
})
export class UploadImageComponent implements OnInit {
public files: Set<File> = new Set();
#ViewChild('file', { static: false }) file;
public uploadedFiles: Array<string> = new Array<string>();
public uploadedFileNames: Array<string> = new Array<string>();
#Output() filesOutput = new EventEmitter<Array<string>>();
#Input() CurrentImage: string;
#Input() IsPublic: boolean;
#Output() valueUpdate = new EventEmitter();
strUploadedFiles:string = '';
filesUploaded: boolean = false;
constructor(private api: ApiService, public snackBar: MatSnackBar,) { }
ngOnInit() {
}
updateValue(val) {
this.valueUpdate.emit(val);
}
reset()
{
this.files = new Set();
this.uploadedFiles = new Array<string>();
this.uploadedFileNames = new Array<string>();
this.filesUploaded = false;
}
upload() {
this.api.upload(this.files).subscribe(res => {
this.filesOutput.emit(this.uploadedFiles);
if (res == 'upload ok')
{
this.reset();
}
}, err => {
console.log(err);
});
}
onFilesAdded() {
var txt = '';
const files: { [key: string]: File } = this.file.nativeElement.files;
for (let key in files) {
if (!isNaN(parseInt(key))) {
var currentFile = files[key];
var sFileExtension = currentFile.name.split('.')[currentFile.name.split('.').length - 1].toLowerCase();
var iFileSize = currentFile.size;
if (!(sFileExtension === "jpg"
|| sFileExtension === "png")
|| iFileSize > 671329) {
txt = "File type : " + sFileExtension + "\n\n";
txt += "Size: " + iFileSize + "\n\n";
txt += "Please make sure your file is in jpg or png format and less than 655 KB.\n\n";
alert(txt);
return false;
}
this.files.add(files[key]);
this.uploadedFiles.push('https://gourmet-philatelist-assets.s3.amazonaws.com/folder/' + files[key].name);
this.uploadedFileNames.push(files[key].name);
if (this.IsPublic && this.uploadedFileNames.length == 1)
{
this.filesUploaded = true;
this.updateValue(files[key].name);
break;
}
else if (!this.IsPublic && this.uploadedFileNames.length == 3)
{
this.strUploadedFiles += files[key].name;
this.updateValue(this.strUploadedFiles);
this.filesUploaded = true;
break;
}
else
{
this.strUploadedFiles += files[key].name + ",";
this.updateValue(this.strUploadedFiles);
}
}
}
}
addFiles() {
this.file.nativeElement.click();
}
openSnackBar(message: string, action: string) {
this.snackBar.open(message, action, {
duration: 2000,
verticalPosition: 'top'
});
}
}
upload-image.component.html
<input type="file" #file style="display: none" (change)="onFilesAdded()" multiple />
<button mat-raised-button color="primary"
[disabled]="filesUploaded" (click)="$event.preventDefault(); addFiles()">
Add Files
</button>
<button class="btn btn-success" [disabled]="uploadedFileNames.length == 0" (click)="$event.preventDefault(); upload()">
Upload
</button>

Resources