MEAN Stack File uploads - node.js

I've recently started programming with the MEAN Stack, and I'm currently implementing some sort of social network. Been using the MEAN.io framework to do so.
My main problem right now is getting the file upload to work, because what I want to do is receive the file from the form into the AngularJS Controller and pass it along with more info's to ExpressJS so I can finally send everything to MongoDB. (I'm building a register new user form).
I dont want to store the file itself on the database but I want to store a link to it.
I've searched dozens of pages on google with different search queries but I couldn't find anything that I could understand or worked. Been searching for hours to no result. That's why I've came here.
Can anyone help me with this?
Thanks :)
EDIT: Maybe a bit of the code would help understand.
The default MEAN.io Users Angular controller which I'm using as foundation has this:
$scope.register = function(){
$scope.usernameError = null;
$scope.registerError = null;
$http.post('/register', {
email: $scope.user.email,
password: $scope.user.password,
confirmPassword: $scope.user.confirmPassword,
username: $scope.user.username,
name: $scope.user.fullname
})//... has a bit more code but I cut it because the post is the main thing here.
};
What I want to do is:
Receive a file from a form, onto this controller and pass it along with email, password, name, etc, etc and be able to use the json on expressjs, which sits on the server side.
The '/register' is a nodejs route so a server controller which creates the user (with the user schema) and sends it to the MongoDB.

I recently did something just like this. I used angular-file-upload. You'll also want node-multiparty for your endpoint to parse the form data. Then you could use s3 for uploading the file to s3.
Here's some of my [edited] code.
Angular Template
<button>
Upload <input type="file" ng-file-select="onFileSelect($files)">
</button>
Angular Controller
$scope.onFileSelect = function(image) {
$scope.uploadInProgress = true;
$scope.uploadProgress = 0;
if (angular.isArray(image)) {
image = image[0];
}
$scope.upload = $upload.upload({
url: '/api/v1/upload/image',
method: 'POST',
data: {
type: 'profile'
},
file: image
}).progress(function(event) {
$scope.uploadProgress = Math.floor(event.loaded / event.total);
$scope.$apply();
}).success(function(data, status, headers, config) {
AlertService.success('Photo uploaded!');
}).error(function(err) {
$scope.uploadInProgress = false;
AlertService.error('Error uploading file: ' + err.message || err);
});
};
Route
var uuid = require('uuid'); // https://github.com/defunctzombie/node-uuid
var multiparty = require('multiparty'); // https://github.com/andrewrk/node-multiparty
var s3 = require('s3'); // https://github.com/andrewrk/node-s3-client
var s3Client = s3.createClient({
key: '<your_key>',
secret: '<your_secret>',
bucket: '<your_bucket>'
});
module.exports = function(app) {
app.post('/api/v1/upload/image', function(req, res) {
var form = new multiparty.Form();
form.parse(req, function(err, fields, files) {
var file = files.file[0];
var contentType = file.headers['content-type'];
var extension = file.path.substring(file.path.lastIndexOf('.'));
var destPath = '/' + user.id + '/profile' + '/' + uuid.v4() + extension;
var headers = {
'x-amz-acl': 'public-read',
'Content-Length': file.size,
'Content-Type': contentType
};
var uploader = s3Client.upload(file.path, destPath, headers);
uploader.on('error', function(err) {
//TODO handle this
});
uploader.on('end', function(url) {
//TODO do something with the url
console.log('file opened:', url);
});
});
});
}
I changed this from my code, so it may not work out of the box, but hopefully it's helpful!

Recently a new package was added to the list of packages on mean.io. It's a beauty!
Simply run:
$ mean install mean-upload
This installs the package into the node folder but you have access to the directives in your packages.
http://mean.io/#!/packages/53ccd40e56eac633a3eee335
On your form view, add something like this:
<div class="form-group">
<label class="control-label">Images</label>
<mean-upload file-dest="'/packages/photos/'" upload-file-callback="uploadFileArticleCallback(file)"></mean-upload>
<br>
<ul class="list-group">
<li ng-repeat="image in article.images" class="list-group-item">
{{image.name}}
<span class="glyphicon glyphicon-remove-circle pull-right" ng-click="deletePhoto(image)"></span>
</li>
</ul>
</div>
And in your controller:
$scope.uploadFileArticleCallback = function(file) {
if (file.type.indexOf('image') !== -1){
$scope.article.images.push({
'size': file.size,
'type': file.type,
'name': file.name,
'src': file.src
});
}
else{
$scope.article.files.push({
'size': file.size,
'type': file.type,
'name': file.name,
'src': file.src
});
}
};
$scope.deletePhoto = function(photo) {
var index = $scope.article.images.indexOf(photo);
$scope.article.images.splice(index, 1);
}
Enjoy!

Mean-upload has been obsoleted and is now called "upload". It is managed in - https://git.mean.io/orit/upload

I know this post is old. I came across it and #kentcdodds had an answer that i really liked, but the libraries he used are now out of date and I could not get them to work. So after some research i have a newer similar solution I want to share.
HTML using ng-upload
<form >
<div style="margin-bottom: 15px;">
<button type="file" name="file" id="file" ngf-select="uploadFiles($file, $invalidFiles)" accept="image/*" ngf-max-height="1000" ngf-max-size="1MB">Select File</button>
</div>
</form>
INCLUDE ng-upload module
download it, references the files and include the module
var app = angular.module('app', ['ngFileUpload']);
this will give you access to the Upload service.
Controller code
$scope.uploadFiles = function(file, errFiles) {
$scope.f = file;
$scope.errFile = errFiles && errFiles[0];
if (file) {
file.upload = Upload.upload({
url: 'you-api-endpoint',
data: {file: file}
});
//put promise and event watchers here if wanted
}
};
NODE api code
All the code below is in a separate route file which is required in my main server.js file.
require('./app/app-routes.js')(app, _);
var fs = require('fs');
var uuid = require('uuid');
var s3 = require('s3fs'); // https://github.com/RiptideElements/s3fs
var s3Impl = new s3('bucketname', {
accessKeyId: '<your access key id>',
secretAccessKey: '< your secret access key >'
});
var multiparty = require('connect-multiparty');
var multipartyMiddleware = multiparty();
module.exports = function(app, _) {
app.use(multipartyMiddleware);
app.post('/your-api-endpoint',function(req, res){
var file = req.files.file; // multiparty is what allows the file to to be accessed in the req
var stream = fs.createReadStream(file.path);
var extension = file.path.substring(file.path.lastIndexOf('.'));
var destPath = '/' + req.user._id + '/avatar/' + uuid.v4() + extension;
var base = 'https://you-bucket-url';
return s3Impl.writeFile(destPath, stream, {ContentType: file.type}).then(function(one){
fs.unlink(file.path);
res.send(base + destPath);
});
});
All i was trying to do was upload a unique avatar for a user. Hope this helps!!

Related

What is the best way to multiple file upload through a single input in nodejs express 4?

I'm using nodejs with express 4. I'm trying to upload multiple file through a single input field. Also I submit my form through ajax.
I'm using express-fileupload middleware for uploading files. When i upload multiple files, it works fine. But when upload a single file, it's not working.
html code-
<input type="file" name="attach_file" id="msg_file" multiple />
ajax code-
var data = new FormData();
$.each($('#msg_file')[0].files, function(i, file) {
data.append('attach_file', file);
});
$.ajax({
url: '/send_message',
data: data,
cache: false,
contentType: false,
processData: false,
method: 'POST',
type: 'POST',
success: function(data){
console.log(data);
}
});
server side js code-
router.post('/send_message', function(req, res, next){
if (!req.files)
res.json('No files were uploaded.');
let sampleFile = req.files.attach_file;
console.log(sampleFile.length);
var file_info = [];
var count = 0;
sampleFile.forEach(function(ele, key) {
ele.mv(path.resolve(`./public/upload/${ele.name}`), function(err) {
if (err){
console.log(err);
}else{
file_info.push(ele.name);
}
count++;
if(sampleFile.length == count){
res.json({file_name: file_info });
}
});
});
});
if i upload a single file console.log(sampleFile.length); show undefined.
After different kind of testing, I found the issue. Everything is ok except when I upload a single file.
When ajax send a single file, it was not an array. That's why, length was undefined and forEach did not run. Need to check first, then use mv() function, like as-
if(sampleFile instanceof Array){
// here is the forEach block
}else{
// run a single mv() function
}
Another complete example and also late answer; in case that help someone.
this is not ajax-based; I used a form in client-side; but you can send data using JS and XHR.
It supports both single and also multiple uploads.
upload.js
'use strict';
const fss = require('fs')
const pth = require('path');
const exp = require('express');
const swg = require('swig');
const efm = require("formidable");
const app = exp();
const thm = swg.compileFile(pth.join(__dirname, '', 'upload.html'));
app.listen(9009);
app.get(`/`, async (q, r) => r.send(thm({ msg: "Select a File to Upload" })));
app.get(`/:msg`, async (q, r) => r.send(thm({ msg: q.params.msg })));
app.post('/upload', (r, q) => {
const form = efm({ multiples: true });
form.parse(r, (e, p, files) => {
let dir = pth.join(__dirname, '', '/media/');
if (!fss.existsSync(dir)) fss.mkdirSync(dir);
let uploaded = 0, exists = 0, error = 0;
//if single file uploaded
if(!Array.isArray(files.file)){
files.file=[files.file]
}
files.file.forEach(f => {
let nPth = dir + f.name;
try {
fss.accessSync(nPth, fss.F_OK);
exists++;
} catch (file_e) {
let err = fss.renameSync(f.path, nPth);
if (err) error++; else uploaded++;
}
});
q.redirect(`Upoader -> All: ${files.file.length}, Uploaded: ${uploaded}, Existed: ${exists}, Error: ${error}`);
});
});
**Its better use "A-Sync" functions.
**I think its also better if you run uploader script on another port.
upload.html
<h3>{{msg}}</h3>
<br/>
<form action="upload" method="post" enctype="multipart/form-data">
<input type="file" name="file" multiple>
<input type="submit">
</form>

Put data and attachment to Cloudant DB IBM bluemix

I am trying to PUT my data within its attachment. I am doing it with NodeJS
Here is my code:
var date = new Date();
var data = {
name : obj.name,
serving : obj.serving,
cuisine : obj.cuisine,
uploadDate : date,
ingredients : obj.ing,
directions: obj.direction
} //Assume that I read this from html form and it is OK
db.insert(data, function(err, body){
if(!err){
console.log(body);
var id = body.id
var rev = body.rev
var headers = {
'Content-Type': req.files.image.type
};
var dataString = '#' + req.files.image.path;
var options = {
url: 'https://username:password#username.bluemix.cloudant.com/db_name/' + id + '/' + req.files.image.name +'?' + 'rev=' + rev,
method: 'PUT',
headers : headers,
body : dataString
}
console.log(options)
function callback(error, response, body) {
if (!error && response.statusCode == 200) {
console.log(body);
}
}
request(options, callback);
}
});
I am getting a 201 response after an image attachment has been sent. But in the cloudant dashboard I see "length": 38 of uploaded image which is impossible.
If I try to access uploaded image it gives:
{"error":"not_found","reason":"Document is missing attachment"}.
How I can fix this problem?
It looks like you are uploading the path to the image and not the contents of the image itself:
var dataString = '#' + req.files.image.path;
This will just upload the string '#xyz' where xyz is the path of the file. You need to upload the contents of the image. See the following for more information:
https://wiki.apache.org/couchdb/HTTP_Document_API#Attachments
I am not sure how to get the contents of the uploaded file from req.files. I believe req.files no longer works in Express 4, so I use multer:
https://github.com/expressjs/multer
This is how I upload a file to Cloudant that was uploaded to my app:
Client
<form action="/processform" method="post" enctype="multipart/form-data">
<input type="file" name="myfile">
<input type="submit" value="Submit">
</form>
Node.js Routing
var multer = require('multer');
...
var app = express();
...
var uploadStorage = multer.memoryStorage();
var upload = multer({storage: uploadStorage})
app.post('/processform', upload.single('myfile'), processForm);
Note: 'myfile' is the name of the file input type in the form.
Node.js Upload to Cloudant
function processForm() {
// insert the document first...
var url = cloudantService.config.url;
url += "/mydatabase/" + doc.id;
url += "/" + encodeURIComponent(req.file.originalname);
url += "?rev=" + doc.rev;
var headers = {
'Content-Type': req.file.mimetype,
'Content-Length': req.file.size
};
var requestOptions = {
url: url,
headers: headers,
body: req.file.buffer
};
request.put(requestOptions, function(err, response, body) {
if (err) {
// handle error
}
else {
// success
}
});
...
}

Nodejs Amazon S3 File Upload

I am trying to create a form that uploads my local files to my S3 bucket, but I'm a little confused as to where parts of the uploading logic should exist. I wanted to keep my POST route clean and reference the AWS logic that is housed in a separate file, but I'm a little confused as to what value should be used for the Body property for the params level and how I should reference this module when setting the completed upload URL to my database property at the fileAttachment line. Can anyone point me in the right direction?
Here is my aws-s3.js file:
module.exports = function() {
var path = require('path');
var async = require('async');
var fs = require('fs');
var AWS = require('aws-sdk');
var config = require(path.resolve(__dirname, '..', '..','./config/config.js'));
AWS.config.region = config.region;
var s3 = new AWS.S3({params: {Bucket: config.awsBucket}});
var params = {Key: config.awsAccessKeyId, Body: req.body.fileAttachment};
s3.upload(params, function(err, data){
if (err) {
console.log("Error uploading data: ", err);
} else {
console.log("Successfully uploaded data to " + config.awsBucket + " /myKey");
}
});
return s3;
};
Here is my route:
appRoutes.route('/create/file')
.get(function(req, res){
models.DiscoverySource.findAll({
where: {
organizationId: req.user.organizationId
}, attributes: ['discoverySource']
}).then(function(discoverySource){
res.render('pages/app/file-create.hbs',{
discoverySource: discoverySource
});
});
})
.post(function(req, res){
models.File.create({
discovery: req.body.discovery,
discoverySource: req.body.discoverySource,
fileAttachment:
userId: req.user.user_id
}).then(function(){
res.redirect('/app');
});
});
Form:
<form action="/app/create/file" method="post">
<div class="form-header">
<label for="data-discovery-source">Discovery Source:</label>
<select name="discoverySource">
{{#each discoverySource}}
<option value="{{this.discoverySource}}">{{this.discoverySource}}</option>
{{/each}}
</select>
<label for="data-discovery">Discovery:</label>
<textarea id="discovery-text-field" name="discovery"></textarea>
<label for="report-link">File Attachment:</label>
<input type="file" name="fileAttachment">
</div>
<button type="submit" id="create-button">Create File</button>
</form>
You can try using the multer-s3 module.
It allows you to upload your file with storaage configured to AWS.
This code uses the aws-sdk module and more about it's configuration can be found here.
Here is my code example:
It uses the recommended amazon AWS SDK for JavaScript in Node.js with
And it also uses multer express middleware for uploading files.
var aws = require('aws-sdk')
var express = require('express')
var multer = require('multer')
var multerS3 = require('multer-s3')
var app = express()
var s3 = new aws.S3({ {accessKeyId: 'akid', secretAccessKey: 'secret'})
//this can also be configured in config file and through code
var upload = multer({
storage: multerS3({
s3: s3,
bucket: 'some-bucket',
key: function (req, file, cb) {
cb(null, Date.now().toString())
}
})
})
//the upload.array -means that you can load multiple files(max 3) named photos maximum 3, the resulting AWS full path urls will be returned in req.files
app.post('/upload', upload.array('photos', 3), function(req, res, next) {
//reference results that can be stored in database for example in req.files
res.send('Successfully uploaded ' + req.files.length + ' files!')
})
this codealso uses the multer npm module. More about its configuration possibilities like: single, any upload.array, fields can be found here.
You can alternatively use Minio-js here is an example of presigned-postpolicy.js. I hope it helps.
var Minio = require('minio')
var s3Client = new Minio({
endPoint: 's3.amazonaws.com',
accessKey: 'YOUR-ACCESSKEYID',
secretKey: 'YOUR-SECRETACCESSKEY',
insecure: false // Default is false.
})
// Construct a new postPolicy.
var policy = s3Client.newPostPolicy()
// Set the object name my-objectname.
policy.setKey("my-objectname")
// Set the bucket to my-bucketname.
policy.setBucket("my-bucketname")
var expires = new Date
expires.setSeconds(24 * 60 * 60 * 10) //10 days
policy.setExpires(expires)
policy.setContentLengthRange(1024, 1024*1024) // Min upload length is 1KB Max upload size is 1MB
s3Client.presignedPostPolicy(policy, function(e, formData) {
if (e) return console.log(e)
var curl = []
curl.push('curl https://s3.amazonaws.com/my-bucketname')
for (var key in formData) {
if (formData.hasOwnProperty(key)) {
var value = formData[key]
curl.push(`-F ${key}=${value}`)
}
}
// Print curl command to upload files.
curl.push('-F file=#<FILE>')
console.log(curl.join(' '))
})
Disclaimer: I work for Minio, Open Source object storage compatible with AWS S3.

Express / torrent-stream: writing a file from stream, sending socket to client with url but client cannot find file

I'm working on a personal project which basically takes a magnet link, starts downloading the file and then renders the video in the torrent in the browser. I'm using an npm module called torrent-stream to do most of this. Once I create the readable stream and begin writing the file I want to send a socket message to the client with the video url so that the client can render a html5 video element and begin streaming the video.
The problem I am having is that once the client renders the video element and tries to find the source mp4 I get a 404 error not found on the video file. Any advice on this would be highly appreciated mates. :)
Controller function:
uploadVideo: function (req, res) {
var torrentStream = require('torrent-stream');
var mkdirp = require('mkdirp');
var rootPath = process.cwd();
var magnetLink = req.param('magnet_link');
var fs = require('fs');
var engine = torrentStream(magnetLink);
engine.on('ready', function() {
engine.files.forEach(function(file) {
var fileName = file.name;
var filePath = file.path;
console.log(fileName + ' - ' + filePath);
var stream = file.createReadStream();
mkdirp(rootPath + '/assets/videos/' + fileName, function (err) {
if (err) {
console.log(err);
} else {
var videoPath = rootPath + '/assets/videos/' + fileName + '/video.mp4';
var writer = fs.createWriteStream(videoPath);
var videoSent = false;
stream.on('data', function (data) {
writer.write(data);
if (!videoSent) {
fs.exists(videoPath, function(exists) {
if (exists) {
sails.sockets.broadcast(req.param('room'), 'video_ready', {videoPath: '/videos/' + fileName + '/video.mp4'});
videoSent = true;
}
});
}
});
// stream is readable stream to containing the file content
}
});
});
});
res.json({status: 'downloading'});
}
Client javascript:
$(document).ready(function () {
io.socket.on('connect', function () {
console.log('mrah');
io.socket.get('/join', {roomName: 'cowboybebop'});
io.socket.on('message', function (data) {
console.log(data);
});
io.socket.on('video_ready', function (data) {
var video = $('<video width="320" height="240" controls>\
<source src="' + data.videoPath + '" type="video/mp4">\
Your browser does not support the video tag.\
</video>');
$('body').append(video);
});
});
$('form').submit(function (e) {
e.preventDefault();
var formData = $(this).serialize();
$.ajax({
url: '/upload-torrent',
method: 'POST',
data: formData
}).success(function (data) {
console.log(data);
}).error(function (err) {
console.log(err);
});
});
});
Form:
<form action="/upload-torrent" method="POST">
<input name="magnet_link" type="text" value="magnet:?xt=urn:btih:565DB305A27FFB321FCC7B064AFD7BD73AEDDA2B&dn=bbb_sunflower_1080p_60fps_normal.mp4&tr=udp%3a%2f%2ftracker.openbittorrent.com%3a80%2fannounce&tr=udp%3a%2f%2ftracker.publicbt.com%3a80%2fannounce&ws=http%3a%2f%2fdistribution.bbb3d.renderfarming.net%2fvideo%2fmp4%2fbbb_sunflower_1080p_60fps_normal.mp4"/>
<input type="hidden" name="room" value="cowboybebop">
<input type="submit" value="Link Torrent">
You may be interested in Torrent Stream Server. It a server that downloads and streams video at the same time, so you can watch the video without fully downloading it. It's based on the same torrent-stream library which you are exploring and has functionally you are trying to implement, so it may be a good reference for you.
Also, I suggest taking a look at webtorrent. It's a nice torrent library that works in both: NodeJs & browser and has streaming support. It sounds great, but from my experience, it doesn't have very good support in the browser.

Send PDF file from AngularJS to NodeJS

i need to send a PDF file from angularjs client to NodeJS service.
I did the angularjs service, and when i receive the file its a string like this:
%PDF-1.3
3 0 obj
<</Type /Page
/Parent 1 0 R
/Reso
How can i reconvert this string to PDF in NodeJS?
This is the client code:
var sendByEmail = function () {
$scope.generatingPdf = true;
$('#budget').show();
var pdf = new JsPDF('p', 'pt', 'letter');
var source = $('#budget')[0];
pdf.addHTML(source, 0, 0, function () {
var resultPdf = pdf.output();
BillService.sendByEmail("rbrlnx#gmail.com", resultPdf).then(function () {
});
$('#budget').hide();
});
};
var sendByEmail = function (email, file) {
var deferred = $q.defer();
var data = {
email: email,
file: file
};
BillService.sendByEmail(data, function (result) {
deferred.resolve(result);
}, function () {
deferred.reject();
});
return deferred.promise;
};
The server code controller its empty:
var sendByEmail = function (req, res, next) {
var file = req.body.file;
};
I experimented with this a while ago, and I came up with this. It's not production ready by a long shot maybe you find it useful. It's free of front end libraries (except Angular ofcourse), but assumes you're using Express 4x and body-parser.
The result:
In the browser:
On the server:
What you're seeing:
You're seeing a tiny node server, serving static index.html and angular files, and a POST route receiving a PDF in base64 as delivered by the HTML FileReader API, and saves it to disk.
Instead of saving to disk, you can send it as an email attachment. See for instance here or here for some info on that.
The example below assumes uploading a PDF by a user through a file input, but the idea is the same for all other ways of sending a document to your back end system. The most important thing is to send the pdf data as BASE64, because this is the format that most file writers and email packages use (as opposed to straight up binary for instance..). This also goes for images, documents etc.
How did I do that:
In your HTML:
<div pdfs>Your browser doesn't support File API.</div>
A directive called pdfs:
myApp.directive('pdfs', ['upload', function(upload) {
return {
replace: true,
scope: function() {
files = null;
},
template: '<input id="files" type="file">',
link: function(scope,element) {
element.bind('change', function(evt) {
scope.$apply(function() {
scope.files = evt.target.files;
});
});
},
controller: function($scope, $attrs) {
$scope.$watch('files', function(files) {
//upload.put(files)
if(typeof files !== 'undefined' && files.length > 0) {
for(var i = 0; i<files.length;i++) {
readFile(files[i])
}
}
}, true);
function readFile(file) {
var reader = new FileReader();
reader.addEventListener("loadend", function(evt) {
upload.post({name: file.name, data: reader.result})
})
if(reader.type = 'application/pdf') {
reader.readAsDataURL(file);
}
}
}
}
}]);
A tiny service:
myApp.service('upload', function($http) {
this.post = function(file) {
$http.post('/pdf', file);
}
});
And a node server:
var express = require('express');
var bodyParser = require('body-parser')
var fs = require("fs");
var app = express();
app.use(express.static('.'));
app.use( bodyParser.json({limit: '1mb'}) );
app.post('/pdf', function(req, res){
var name = req.body.name;
var pdf = req.body.data;
var pdf = pdf.replace('data:application/pdf;base64,', '');
res.send('received');
fs.writeFile(name, pdf, 'base64', function(err) {
console.log(err);
});
});
var server = app.listen(3000, function() {
console.log('Listening on port %d', server.address().port);
});

Resources