node uploading file $http.post from angularjs has undefined req.body - node.js

I'm building a file upload functionality with my angularjs app that would upload a file to my node api that will ftp to a cdn server. Right now I'm stuck with just getting hte file. I tried using multer but I'm not sure how to prevent the save to redirect to an ftp.
Anyway, this is my code withoout multer
<input type="file" multiple file-model="fileRepo"/>
myApp.directive('fileModel', ['$parse', function ($parse) {
return {
restrict: 'A',
link: function(scope, element, attrs) {
element.bind('change', function(){
$parse(attrs.fileModel).assign(scope,element[0].files)
scope.$apply();
});
}
};
}]);
///controller///
$scope.saveFile = function(){
var fd=new FormData();
angular.forEach($scope.fileRepo,function(file){
fd.append('file',file);
});
$scope.newFile.files = fd;
FileService.uploadFile($scope.newFile)
.....
/// fileservice ///
uploadFile: function(file){
var deferred = $q.defer();
var uploadUrl = '/api/file/ftp/new';
var requestFileUpload = {
method: 'POST',
url: uploadUrl,
data: file.files
}
var requestFileUploadConfig = {
transformRequest: angular.identity,
headers: { 'Content-Type': undefined }
}
$http.post(uploadUrl, file.files, requestFileUploadConfig)
.then(function(){
})
/// node part ///
router.post('/ftp/new', function(req, res) {
console.log('file is ' + JSON.stringify(req.body));
});

You'll have to use an HTML parser you are not going to be able to catch the file just by reading the request.
I'd recommend use busboy and connect-busboy then you are going to be able to read your file, this a small example:
req.pipe(req.busboy);
req.busboy.on('file',function(fieldname, file, filename, encoding, contentType){
// get data
file.on('data',function(data){
}).on('end', function(){
});
});
req.busboy.on('field',function(fieldname, val){
req.body[fieldname] = val;
});
req.busboy.on('finish', function() {
// save file here
});

Related

Nodejs is not receiving any code from Flask app.

I am really new in node js and a little bit more experienced in flaks. I am trying to connect a nodejs backend with a flask api. Basically I am sending a file that was uploaded in the nodejs app for processing (converting to another format) to my flask app.
For sending the data I am using request. In this way:
app.post('/converttest', uploader.single('file'), function(req,res){
var file = req.file,
result = {
error: 0,
uploaded: []
};
flow.exec(
function() { // Read temp File
fs.readFile(file.path, this);
},
function(err, data) { // Upload file to S3
var formData = {
file: data,
};
requestPack.post({url:'http://127.0.0.1:5000/api/resource/converter', formData: formData});
},
function(err, httpResponse, body) { //Upload Callback
if (err) {
return console.error('upload failed:', err);
}
res.redirect('/console');
});
});
Then I am receiving the file for processing in the flask app, like:
#app.route('/api/resource/converter', methods = ['POST','GET'])
def converter_csv():
if request.method == 'POST':
f = request.form['file']
if not f:
abort(400)
print('-----Converting-------')
file = open("temp/converting.txt","w")
file.write(f)
#....conversion process...
# Finish the process
return Response(converted_file,status=200)
In my console for the localhost of the flask app, I am getting:
127.0.0.1 - - [09/Aug/2017 15:47:59] "POST /api/resource/converter HTTP/1.1" 200 -
However my nodejs app did not receive any response. It just got frozen.
I appreciate any orientation anyone can give me. Thanks.
I think flow.exec is not in proper order
router.post('/converttest', uploader.single('file'), function(req, res) {
var filePath = req.file.path;
fs.readFile(filePath, 'utf8', function(err, data) { //change format reading as required
try {
formData = {file:data}
requestPack.post({url:'http://127.0.0.1:5000/api/resource/converter', formData: formData});
} catch(err) {
return console.error('upload failed:', err);
res.redirect('/console')
}
fs.unlink(filePath);}); });
I ended up using requestify. Seems like they make it a little bit easier for beginners like me:
var requestify = require('requestify');
app.get('/convertupload', function(req,res){
res.render('pages/convertupload');
});
app.post('/converttest', uploader.single('file'), function(req,res){
var file = req.file,
result = {
error: 0,
uploaded: []
};
flow.exec(
function() { // Read temp File
fs.readFile(file.path,this);
},
function(err, data) { // Upload file to S3
var formData = {
file: data
};
requestify.post('http://127.0.0.1:5000/api/resource/converter', {
form: formData
})
.then(function(response) {
// Get the response body (JSON parsed or jQuery object for XMLs)
console.log(response)
response.getBody();
});
res.redirect('/login');
});
});

node.js - streaming upload to cloud storage (busboy, request)

I'm new to node.js. What I'm trying to do is to stream the upload of a file from web browser to a cloud storage through my node.js server.
I'm using 'express', 'request' and 'busboy' modules.
var express = require("express");
var request = require("request");
var BusBoy = require("busboy");
var router = express.Router();
router.post("/upload", function(req, res, next) {
var busboy = new BusBoy({ headers: req.headers });
var json = {};
busboy.on("file", function (fieldname, file, filename, encoding, mimetype) {
file.on("data", function(data) {
console.log(`streamed ${data.length}`);
});
file.on("end", function() {
console.log(`finished streaming ${filename}`);
});
var r = request({
url: "http://<my_cloud_storage_api_url>",
method: "POST",
headers: {
"CUSTOM-HEADER": "Hello",
},
formData: {
"upload": file
}
}, function(err, httpResponse, body) {
console.log("uploaded");
json.response = body;
});
});
busboy.on("field", function(name, val) {
console.log(`name: ${name}, value: ${value}`);
});
busboy.on("finish", function() {
res.send(json);
});
req.pipe(busboy);
});
module.exports = router;
But I keep getting the following error on the server. What am I doing wrong here? Any help is appreciated.
Error: Part terminated early due to unexpected end of multipart data
at node_modules\busboy\node_modules\dicer\lib\Dicer.js:65:36
at nextTickCallbackWith0Args (node.js:420:9)
at process._tickCallback (node.js:349:13)
I realize this question is some 7 months old, but I shall answer it here in an attempt help anyone else currently banging their head against this.
You have two options, really: Add the file size, or use something other than Request.
Note: I edited this shortly after first posting it to hopefully provide a bit more context.
Using Something Else
There are some alternatives you can use instead of Request if you don't need all the baked in features it has.
form-data can be used by itself in simple cases, or it can be used with, say, got. request uses this internally.
bhttp advertises Streams2+ support, although in my experience Streams2+ support has not been an issue for me. No built in https support, you have to specify a custom agent
got another slimmed down one. Doesn't have any special handling of form data like request does, but is trivially used with form-data or form-data2. I had trouble getting it working over a corporate proxy, though, but that's likely because I'm a networking newb.
needle seems pretty light weight, but I haven't actually tried it.
Using Request: Add the File Size
Request does not (as of writing) have any support for using transfer-encoding: chunked so to upload files with it, you need to add the file's size along with the file, which if you're uploading from a web client means that client needs to send that file size to your server in addition to the file itself.
The way I came up with to do this is to send the file metadata in its own field before the file field.
I modified your example with comments describing what I did. Note that I did not include any validation of the data received, but I recommend you do add that.
var express = require("express");
var request = require("request");
var BusBoy = require("busboy");
var router = express.Router();
router.post("/upload", function(req, res, next) {
var busboy = new BusBoy({ headers: req.headers });
var json = {};
// Use this to cache any fields which are file metadata.
var fileMetas = {};
busboy.on("file", function (fieldname, file, filename, encoding, mimetype) {
// Be sure to match this prop name here with the pattern you use to detect meta fields.
var meta = fileMetas[fieldname + '.meta'];
if (!meta) {
// Make sure to dump the file.
file.resume();
// Then, do some sort of error handling here, because you cannot upload a file
// without knowing it's length.
return;
}
file.on("data", function(data) {
console.log(`streamed ${data.length}`);
});
file.on("end", function() {
console.log(`finished streaming ${filename}`);
});
var r = request({
url: "http://<my_cloud_storage_api_url>",
method: "POST",
headers: {
"CUSTOM-HEADER": "Hello",
},
formData: {
// value + options form of a formData field.
"upload": {
value: file,
options: {
filename: meta.name,
knownLength: meta.size
}
}
}
}, function(err, httpResponse, body) {
console.log("uploaded");
json.response = body;
});
});
busboy.on("field", function(name, val) {
// Use whatever pattern you want. I used (fileFieldName + ".meta").
// Another good one might be ("meta:" + fileFieldName).
if (/\.meta$/.test(name)) {
// I send an object with { name, size, type, lastModified },
// which are just the public props pulled off a File object.
// Note: Should probably add error handling if val is somehow not parsable.
fileMetas[name] = JSON.parse(val);
console.log(`file metadata: name: ${name}, value: ${value}`);
return;
}
// Otherwise, process field as normal.
console.log(`name: ${name}, value: ${value}`);
});
busboy.on("finish", function() {
res.send(json);
});
req.pipe(busboy);
});
module.exports = router;
On the client, you need to then send the metadata on the so-named field before the file itself. This can be done by ordering an <input type="hidden"> control before the file and updating its value onchange. The order of values sent is guaranteed to follow the order of inputs in appearance. If you're building the request body yourself using FormData, you can do this by appending the appropriate metadata before appending the File.
Example with <form>
<script>
function extractFileMeta(file) {
return JSON.stringify({
size: file.size,
name: file.name,
type: file.type,
lastUpdated: file.lastUpdated
});
}
function onFileUploadChange(event) {
// change this to use arrays if using the multiple attribute on the file input.
var file = event.target.files[0];
var fileMetaInput = document.querySelector('input[name=fileUpload.meta]');
if (fileMetaInput) {
fileMetaInput.value = extractFileMeta(file);
}
}
</script>
<form action="/upload-to-cloud">
<input type="hidden" name="fileUpload.meta">
<input type="file" name="fileUpload" onchange="onFileUploadChange(event)">
</form>
Example with FormData:
function onSubmit(event) {
event.preventDefault();
var form = document.getElementById('my-upload-form');
var formData = new FormData();
var fileUpload = form.elements['fileUpload'];
var fileUploadMeta = JSON.stringify({
size: fileUpload.size,
name: fileUpload.name,
type: fileUpload.type,
lastUpdated: fileUpload.lastUpdated
});
// Append fileUploadMeta BEFORE fileUpload.
formData.append('fileUpload.meta', fileUploadMeta);
formData.append('fileUpload', fileUpload);
// Do whatever you do to POST here.
}

upload binary file to redmine with node

I try to upload a file to redmine with node, I can upload and attach text files, but when I try to upload a binary file I get the token but the file doesn't work. I tried with json, xml and binary, ascii, base64 encoding.
I want upload binary files because I'm doing end to end test testing I want open Issues with screenshots, and upload a report.
I'm using node-rest-client for service calling
Could someone give me any suggestion to fix this problem?
Thanks,
I define the class RMClient
var Client = require('node-rest-client').Client;
var Q = require('q');
var RMClient = function(baseUri, apiToken){
this._apiToken = apiToken;
var client = new Client();
client.registerMethod('openIssue', baseUri+'/issues.json', 'POST');
client.registerMethod('uploadFile', baseUri+'/uploads.json', 'POST');
client.registerMethod('getIssues', baseUri+'/issues.json', 'GET');
this._client = client;
};
option 1:
var deferred = Q.defer();
var file fs.readFileSync(filePath);
//code for sending file to redmine uploads.json
return deferred.promise;
Option 2
var deferred = Q.defer();
var rs = fs.createReadStream(filePath, {'flags': 'r', 'encoding': null, 'autoClose': true});
var size = fs.statSync(filePath).size;
var file = '';
rs.on('error', function(err){
deferred.reject(err);
});
rs.on('data', function(chunk){ file += chunk; });
rs.on('end', function(){
//code for sending file to redmine uploads.json
});
return deferred.promise;
Code that I use to upload the file:
try{
if(!file){
throw new Error('File must\'nt be void');
}
var rmc = new RMClient(myRMURI, myAPItoken);
var headers = {
'X-Redmine-API-Key': rmc._apiToken,
'Content-Type': 'application/octet-stream',
'Accept':'application/json',
'Content-Length': size
};
var args = {
'data':file,
'headers': headers
};
if(parameters){
args.parameters = parameters;
}
rmc._client.methods.uploadFile(args, function(data, response){
if(response.statusCode != 201){
var err = new Error(response.statusMessage);
deferred.reject(err);
return;
}
var attach = JSON.parse(data);
console.log(attach);
if(data.errors){
var msg = ''.concat.apply('', attach.errors.map(function(item, i){
return ''.concat(i+1,'- ',item,(i+1<attach.errors.length)?'\n':'');
}));
console.error(msg);
deferred.reject(Error(msg));
}else{
deferred.resolve(attach.upload.token);
}
});
}catch(err){
console.error(err);
deferred.reject(err);
}
I faced the same issue and solved it this way:
Use "multer";
When you have an uploaded file, make a request using node "request" module, with req.file.buffer as body.
Then uploading files using the Rest API, you have to send the raw file contents in the request body, typically with Content-Type: application/octet-stream. The uploaded file doesn't need any further encoding or wrapping, esp. not as multipart/form-data, JSON or XML.
The response of the POST request to /uploads.xml contains the token to attach the attachment to other objects in Redmine.

busboy - is there a way to send the response when all files have been uploaded?

I'm trying to upload files to a server using node.js as backend and angular.js as frontend. I'm using express 4 + busboy for this. I have a table in the frontend where I should display all the files I'm uploading. So if I have 3 files and click on upload, angular should post these files to node.js and after getting the response back, refresh the table with those three files.
This is the function I'm using in angular:
function uploadFiles(files){
var fd = new FormData();
for(var i = 0; i<files.length; i++){
fd.append("file", files[i]);
}
$http.post('http://localhost:3000/upload', fd, {
withCredentials: false,
headers: {'Content-Type': undefined },
transformRequest: angular.identity
}).success(refreshTable()).error(function(){
console.log("error uploading");
});
}
and this is from node.js:
app.post('/upload', function(req, res) {
var busboy = new Busboy({ headers: req.headers });
busboy.on('file', function (fieldname, file, filename) {
console.log("Uploading: " + filename);
var fstream = fs.createWriteStream('./files/' + filename);
file.pipe(fstream);
});
busboy.on('finish', function(){
res.writeHead(200, { 'Connection': 'close' });
res.end("");
});
return req.pipe(busboy);
});
the problem is that if I upload three files, as soon as the first file has been uploaded node.js sends the response and hence the table is updated only with the first file uploaded, if I refresh the page, the rest of the files appear.
I think the problem is with this line in node: return req.pipe(busboy); if I remove that line, the post response keeps on pending for a long time and nothing happens, I think this is an async problem, anybody knows if there's a way to send the response back only when all files have been uploaded?
thanks
A simple and common solution to this particular problem is to use a counter variable and listening for the finish event on the fs Writable stream. For example:
app.post('/upload', function(req, res) {
var busboy = new Busboy({ headers: req.headers });
var files = 0, finished = false;
busboy.on('file', function (fieldname, file, filename) {
console.log("Uploading: " + filename);
++files;
var fstream = fs.createWriteStream('./files/' + filename);
fstream.on('finish', function() {
if (--files === 0 && finished) {
res.writeHead(200, { 'Connection': 'close' });
res.end("");
}
});
file.pipe(fstream);
});
busboy.on('finish', function() {
finished = true;
});
return req.pipe(busboy);
});
The reason for this is that busboy's finish event is emitted once the entire request has been fully processed, that includes files. However, there is some delay between when there is no more data to write to a particular file and when the OS/node has flushed its internal buffers to disk (and the closing of the file descriptor). Listening for the finish event for a fs Writable stream lets you know that the file descriptor has been closed and no more writes are going to occur.

jQuery's Ajax callbacks not firing after file upload

I have a relatively simple ajax call that uploads a file from an input form to a server:
$(function(){
$("form").submit(function(){
var infile = $('#pickedfile');
var actFile = infile[0].files[0];
$.ajax(
{
type: "POST",
url: "http://localhost:3000/file_upload",
data: [
{
file: actFile
}],
dataType: 'text',
success: function ()
{
alert("Data Uploaded");
},
beforeSend: function ()
{
alert("Before Send");
return false;
},
error: function ()
{
alert("Error detected");
},
complete: function ()
{
alert("Completed");
}
});
});
});
And I have a node.js server that successfully receives the file, and reports back:
var express = require('express'),
wines = require('./routes/testscripts');
var app = express();
app.configure(function(){
app.use(express.bodyParser());
app.use(app.router);
});
app.post('/file_upload', function(req, res) {
//file should be in req.files.pickedfile
// get the temporary location of the file
// undefined if file was not uploaded
var tmp_path = req.files.pickedfile.path;
res.send("Hello, this is server");
});
});
app.listen(3000);
The file is uploaded successfully, and everything seems to work fine on the server side.
However, on the front end, none of the alerts are firing, so I'm not able to respond to any new data from the server. Is there some additional step that I'm missing?
Turns out this was a combination of two issues:
1) The jQuery, as identified in the comments, was being aborted.
2) Because the jQuery was canceled, it did not prevent the associated HTML form from submitting normally.
This combination meant that the form would submit and receive a response successfully, but because the jQuery code was silently failing, no callbacks were received.

Resources