I am really new in node js and a little bit more experienced in flaks. I am trying to connect a nodejs backend with a flask api. Basically I am sending a file that was uploaded in the nodejs app for processing (converting to another format) to my flask app.
For sending the data I am using request. In this way:
app.post('/converttest', uploader.single('file'), function(req,res){
var file = req.file,
result = {
error: 0,
uploaded: []
};
flow.exec(
function() { // Read temp File
fs.readFile(file.path, this);
},
function(err, data) { // Upload file to S3
var formData = {
file: data,
};
requestPack.post({url:'http://127.0.0.1:5000/api/resource/converter', formData: formData});
},
function(err, httpResponse, body) { //Upload Callback
if (err) {
return console.error('upload failed:', err);
}
res.redirect('/console');
});
});
Then I am receiving the file for processing in the flask app, like:
#app.route('/api/resource/converter', methods = ['POST','GET'])
def converter_csv():
if request.method == 'POST':
f = request.form['file']
if not f:
abort(400)
print('-----Converting-------')
file = open("temp/converting.txt","w")
file.write(f)
#....conversion process...
# Finish the process
return Response(converted_file,status=200)
In my console for the localhost of the flask app, I am getting:
127.0.0.1 - - [09/Aug/2017 15:47:59] "POST /api/resource/converter HTTP/1.1" 200 -
However my nodejs app did not receive any response. It just got frozen.
I appreciate any orientation anyone can give me. Thanks.
I think flow.exec is not in proper order
router.post('/converttest', uploader.single('file'), function(req, res) {
var filePath = req.file.path;
fs.readFile(filePath, 'utf8', function(err, data) { //change format reading as required
try {
formData = {file:data}
requestPack.post({url:'http://127.0.0.1:5000/api/resource/converter', formData: formData});
} catch(err) {
return console.error('upload failed:', err);
res.redirect('/console')
}
fs.unlink(filePath);}); });
I ended up using requestify. Seems like they make it a little bit easier for beginners like me:
var requestify = require('requestify');
app.get('/convertupload', function(req,res){
res.render('pages/convertupload');
});
app.post('/converttest', uploader.single('file'), function(req,res){
var file = req.file,
result = {
error: 0,
uploaded: []
};
flow.exec(
function() { // Read temp File
fs.readFile(file.path,this);
},
function(err, data) { // Upload file to S3
var formData = {
file: data
};
requestify.post('http://127.0.0.1:5000/api/resource/converter', {
form: formData
})
.then(function(response) {
// Get the response body (JSON parsed or jQuery object for XMLs)
console.log(response)
response.getBody();
});
res.redirect('/login');
});
});
Related
I'm trying to download an image from my server using request.
I've managed to download something but i get more data than the image.
function _download(uri, save_as, destination) {
let options = {
uri: uri,
timeout: 100000,
followAllRedirects: true
};
return new Promise(( _resolve,_reject) => {
let ext, filename, bar, total, downloaded, req;
req = request(options).on('response', (resp) => {
if (resp.statusCode === 200){
ext = _getFileType(resp.headers['content-type']);
filename = destination+'/'+save_as+ext;
var stream = fs.createWriteStream(filename)
resp.pipe(stream).on('error',function(err){
_reject(err);
}).on('finish',function(){
_resolve(filename);
});
} else {
_reject("unable to download image %s",uri);
}
}).on('error', function(err) {
console.log(err)
_reject(err);
})
});
}
My original url is in form of https://www.test.com/image/original/12345, my server than redirects with a 301 status to my s3 bucket where image is stored.
Unfortunately due to the url of the image i have to wait that for the response header content type to determinate what kind of image it's and use it to pipe the image.
Everything works quite as expected... but i get more data than what is stored in s3.
Does anyone have any suggestion ?
please refer to the link below
var fs = require('fs'),
request = require('request');
var download = function(uri, filename, callback){
request.head(uri, function(err, res, body){
console.log('content-type:', res.headers['content-type']);
console.log('content-length:', res.headers['content-length']);
request(uri).pipe(fs.createWriteStream(filename)).on('close', callback);
});
};
download('https://www.google.com/images/srpr/logo3w.png', 'google.png', function(){
console.log('done');
});
please visit this link
And this link also
I am creating a application using mean stack, in which i am using angular 2 for the client side. I had created a form that contain some input fields and a image. Now, for submitting the form i am using formdata to send data to the node server. now I am unable to show, access and save the data at the node server. Please somebody help me as I am new to mean stack.
data array:
const newProduct = {
category: this.pcategory,
name: this.name,
description: this.description,
price: this.price,
quantity: this.quantity,
image: this.fileName
}
here is the code for sending data:
imagedata contain the data of the file
addProduct(newProduct, imagedata:File) {
let formData: FormData = new FormData();
formData.append('body', JSON.stringify(newProduct));
formData.append('file', image, newProduct.imagedata);
let headers = new Headers();
headers.append("enctype", "multipart/form-data");
headers.append("Accept", "application/json");
let options = new RequestOptions({ headers: headers });
return this.http.post('http://localhost:3000/product/add' ,formData, options).map((response: Response) => response.json());
}
here is the code for receiving and saving data:
function (req, res) {
var storage = multer.diskStorage({//multers disk storage settings
destination: function (req, file, callback) {
callback(null, './uploads');
}
});
var upload = multer({//multer settings
storage: storage
}).any();
var model = new Model(req.body);
model.save(function (err) {
if (err) {
return res.status(400).send({
message: errorHandler.getErrorMessage(err)
});
} else {
res.status(201).json(model);
}
});
upload(req, res, function (err) {
if (err) {
// An error occurred when uploading
console.log(err);
return res.status(422).send("an Error occured")
}
});
}
In angular 2 you cannot upload image with this approach consider using this Angular 2 module ng2-file-uploader. You can see the demo app here Angular File Uploads with an Express Backend
.
One solution could be to convert your image to base64 string and pass that string in your model. And then have that base64 string convert back to image in the server.
I'm making a NodeJS app which should download a file given the public share URL that doesn't request login auth.
I have created the app inside the developer section and generated the proper tokens.
On the git documentation I've found this method that should be used to download a file. The first parameter is the fileId but I don't find written anywhere what this id is and where to find it.
https://rawgit.com/box/box-node-sdk/master/docs/jsdoc/Files.html#getReadStream
I've tried this code
var fs = require('fs'),
BoxSDK = require('box-node-sdk'),
sdk = new BoxSDK({
clientID: '...',
clientSecret: '...'
}),
client = sdk.getAnonymousClient(),
file_id = process.argv[2].replace(/\S*\/(\S+)$/,"$1");
client.files.getReadStream(file_id, null, function(error, stream)
{
if (error) {
throw error;
}
// write the file to disk
var output = fs.createWriteStream('./output/'+file_id+".zip"); //I know for sure there will only be zip files
stream.pipe(output);
});
But running it with this command
nodejs index.js https://adrime.box.com/s/s5w7lzm4xfifmxrod9r7eeniv9nhtenk
I get this error:
Error: Unexpected API Response [404 Not Found] (not_found: "Not Found")
at Object.module.exports.buildResponseError (/home/andrea/dev/node/box_api/node_modules/box-node-sdk/lib/util/errors.js:57:23)
at Object.module.exports.buildUnexpectedResponseError (/home/andrea/dev/node/box_api/node_modules/box-node-sdk/lib/util/errors.js:94:15)
at /home/andrea/dev/node/box_api/node_modules/box-node-sdk/lib/managers/files.js:148:20
at BoxClient._handleResponse (/home/andrea/dev/node/box_api/node_modules/box-node-sdk/lib/box-client.js:291:2)
at handleResponseCallback (/home/andrea/dev/node/box_api/node_modules/box-node-sdk/lib/box-client.js:233:9)
at /home/andrea/dev/node/box_api/node_modules/box-node-sdk/lib/api-request.js:285:3
at nextTickCallbackWith0Args (node.js:436:9)
at process._tickCallback (node.js:365:13)
Can anyone help me in programmatically downloading a public shared file from box.com?
Thank you in advance!
At the moment I've found this solution.
To me it works pretty well. I hope it will be useful to others as well.
var fs = require("fs"),
request = require("request");
function downloadFile(source, target, callback)
{
var wr = fs.createWriteStream(target);
wr.on("error", function (err)
{
callback(err);
});
wr.on("close", function (ex)
{
callback();
});
request
.get(source)
.on("error",function(err)
{
callback(err);
})
.on('response', function(response)
{
if((""+response.statusCode).charAt(0) != "2")
callback({
msg: "File not found: "+response.request.href,
path: response.request.href,
toString: function(){
return this.msg;
}
});
})
.on("end", function (ex)
{
console.log("request ended");
})
.pipe(wr);
}
function onDownloadComplete(err)
{
if(err)
throw err;
console.log("DOWNLOAD COMPLETE");
}
function init()
{
var static_url = process.argv[2].replace(/\/s\//,"/shared/static/") + ".zip";
console.log(static_url);
downloadFile(static_url, "./output/template.zip", onDownloadComplete);
}
init();
I'm new to node.js. What I'm trying to do is to stream the upload of a file from web browser to a cloud storage through my node.js server.
I'm using 'express', 'request' and 'busboy' modules.
var express = require("express");
var request = require("request");
var BusBoy = require("busboy");
var router = express.Router();
router.post("/upload", function(req, res, next) {
var busboy = new BusBoy({ headers: req.headers });
var json = {};
busboy.on("file", function (fieldname, file, filename, encoding, mimetype) {
file.on("data", function(data) {
console.log(`streamed ${data.length}`);
});
file.on("end", function() {
console.log(`finished streaming ${filename}`);
});
var r = request({
url: "http://<my_cloud_storage_api_url>",
method: "POST",
headers: {
"CUSTOM-HEADER": "Hello",
},
formData: {
"upload": file
}
}, function(err, httpResponse, body) {
console.log("uploaded");
json.response = body;
});
});
busboy.on("field", function(name, val) {
console.log(`name: ${name}, value: ${value}`);
});
busboy.on("finish", function() {
res.send(json);
});
req.pipe(busboy);
});
module.exports = router;
But I keep getting the following error on the server. What am I doing wrong here? Any help is appreciated.
Error: Part terminated early due to unexpected end of multipart data
at node_modules\busboy\node_modules\dicer\lib\Dicer.js:65:36
at nextTickCallbackWith0Args (node.js:420:9)
at process._tickCallback (node.js:349:13)
I realize this question is some 7 months old, but I shall answer it here in an attempt help anyone else currently banging their head against this.
You have two options, really: Add the file size, or use something other than Request.
Note: I edited this shortly after first posting it to hopefully provide a bit more context.
Using Something Else
There are some alternatives you can use instead of Request if you don't need all the baked in features it has.
form-data can be used by itself in simple cases, or it can be used with, say, got. request uses this internally.
bhttp advertises Streams2+ support, although in my experience Streams2+ support has not been an issue for me. No built in https support, you have to specify a custom agent
got another slimmed down one. Doesn't have any special handling of form data like request does, but is trivially used with form-data or form-data2. I had trouble getting it working over a corporate proxy, though, but that's likely because I'm a networking newb.
needle seems pretty light weight, but I haven't actually tried it.
Using Request: Add the File Size
Request does not (as of writing) have any support for using transfer-encoding: chunked so to upload files with it, you need to add the file's size along with the file, which if you're uploading from a web client means that client needs to send that file size to your server in addition to the file itself.
The way I came up with to do this is to send the file metadata in its own field before the file field.
I modified your example with comments describing what I did. Note that I did not include any validation of the data received, but I recommend you do add that.
var express = require("express");
var request = require("request");
var BusBoy = require("busboy");
var router = express.Router();
router.post("/upload", function(req, res, next) {
var busboy = new BusBoy({ headers: req.headers });
var json = {};
// Use this to cache any fields which are file metadata.
var fileMetas = {};
busboy.on("file", function (fieldname, file, filename, encoding, mimetype) {
// Be sure to match this prop name here with the pattern you use to detect meta fields.
var meta = fileMetas[fieldname + '.meta'];
if (!meta) {
// Make sure to dump the file.
file.resume();
// Then, do some sort of error handling here, because you cannot upload a file
// without knowing it's length.
return;
}
file.on("data", function(data) {
console.log(`streamed ${data.length}`);
});
file.on("end", function() {
console.log(`finished streaming ${filename}`);
});
var r = request({
url: "http://<my_cloud_storage_api_url>",
method: "POST",
headers: {
"CUSTOM-HEADER": "Hello",
},
formData: {
// value + options form of a formData field.
"upload": {
value: file,
options: {
filename: meta.name,
knownLength: meta.size
}
}
}
}, function(err, httpResponse, body) {
console.log("uploaded");
json.response = body;
});
});
busboy.on("field", function(name, val) {
// Use whatever pattern you want. I used (fileFieldName + ".meta").
// Another good one might be ("meta:" + fileFieldName).
if (/\.meta$/.test(name)) {
// I send an object with { name, size, type, lastModified },
// which are just the public props pulled off a File object.
// Note: Should probably add error handling if val is somehow not parsable.
fileMetas[name] = JSON.parse(val);
console.log(`file metadata: name: ${name}, value: ${value}`);
return;
}
// Otherwise, process field as normal.
console.log(`name: ${name}, value: ${value}`);
});
busboy.on("finish", function() {
res.send(json);
});
req.pipe(busboy);
});
module.exports = router;
On the client, you need to then send the metadata on the so-named field before the file itself. This can be done by ordering an <input type="hidden"> control before the file and updating its value onchange. The order of values sent is guaranteed to follow the order of inputs in appearance. If you're building the request body yourself using FormData, you can do this by appending the appropriate metadata before appending the File.
Example with <form>
<script>
function extractFileMeta(file) {
return JSON.stringify({
size: file.size,
name: file.name,
type: file.type,
lastUpdated: file.lastUpdated
});
}
function onFileUploadChange(event) {
// change this to use arrays if using the multiple attribute on the file input.
var file = event.target.files[0];
var fileMetaInput = document.querySelector('input[name=fileUpload.meta]');
if (fileMetaInput) {
fileMetaInput.value = extractFileMeta(file);
}
}
</script>
<form action="/upload-to-cloud">
<input type="hidden" name="fileUpload.meta">
<input type="file" name="fileUpload" onchange="onFileUploadChange(event)">
</form>
Example with FormData:
function onSubmit(event) {
event.preventDefault();
var form = document.getElementById('my-upload-form');
var formData = new FormData();
var fileUpload = form.elements['fileUpload'];
var fileUploadMeta = JSON.stringify({
size: fileUpload.size,
name: fileUpload.name,
type: fileUpload.type,
lastUpdated: fileUpload.lastUpdated
});
// Append fileUploadMeta BEFORE fileUpload.
formData.append('fileUpload.meta', fileUploadMeta);
formData.append('fileUpload', fileUpload);
// Do whatever you do to POST here.
}
I'm building a file upload functionality with my angularjs app that would upload a file to my node api that will ftp to a cdn server. Right now I'm stuck with just getting hte file. I tried using multer but I'm not sure how to prevent the save to redirect to an ftp.
Anyway, this is my code withoout multer
<input type="file" multiple file-model="fileRepo"/>
myApp.directive('fileModel', ['$parse', function ($parse) {
return {
restrict: 'A',
link: function(scope, element, attrs) {
element.bind('change', function(){
$parse(attrs.fileModel).assign(scope,element[0].files)
scope.$apply();
});
}
};
}]);
///controller///
$scope.saveFile = function(){
var fd=new FormData();
angular.forEach($scope.fileRepo,function(file){
fd.append('file',file);
});
$scope.newFile.files = fd;
FileService.uploadFile($scope.newFile)
.....
/// fileservice ///
uploadFile: function(file){
var deferred = $q.defer();
var uploadUrl = '/api/file/ftp/new';
var requestFileUpload = {
method: 'POST',
url: uploadUrl,
data: file.files
}
var requestFileUploadConfig = {
transformRequest: angular.identity,
headers: { 'Content-Type': undefined }
}
$http.post(uploadUrl, file.files, requestFileUploadConfig)
.then(function(){
})
/// node part ///
router.post('/ftp/new', function(req, res) {
console.log('file is ' + JSON.stringify(req.body));
});
You'll have to use an HTML parser you are not going to be able to catch the file just by reading the request.
I'd recommend use busboy and connect-busboy then you are going to be able to read your file, this a small example:
req.pipe(req.busboy);
req.busboy.on('file',function(fieldname, file, filename, encoding, contentType){
// get data
file.on('data',function(data){
}).on('end', function(){
});
});
req.busboy.on('field',function(fieldname, val){
req.body[fieldname] = val;
});
req.busboy.on('finish', function() {
// save file here
});