I'm making a NodeJS app which should download a file given the public share URL that doesn't request login auth.
I have created the app inside the developer section and generated the proper tokens.
On the git documentation I've found this method that should be used to download a file. The first parameter is the fileId but I don't find written anywhere what this id is and where to find it.
https://rawgit.com/box/box-node-sdk/master/docs/jsdoc/Files.html#getReadStream
I've tried this code
var fs = require('fs'),
BoxSDK = require('box-node-sdk'),
sdk = new BoxSDK({
clientID: '...',
clientSecret: '...'
}),
client = sdk.getAnonymousClient(),
file_id = process.argv[2].replace(/\S*\/(\S+)$/,"$1");
client.files.getReadStream(file_id, null, function(error, stream)
{
if (error) {
throw error;
}
// write the file to disk
var output = fs.createWriteStream('./output/'+file_id+".zip"); //I know for sure there will only be zip files
stream.pipe(output);
});
But running it with this command
nodejs index.js https://adrime.box.com/s/s5w7lzm4xfifmxrod9r7eeniv9nhtenk
I get this error:
Error: Unexpected API Response [404 Not Found] (not_found: "Not Found")
at Object.module.exports.buildResponseError (/home/andrea/dev/node/box_api/node_modules/box-node-sdk/lib/util/errors.js:57:23)
at Object.module.exports.buildUnexpectedResponseError (/home/andrea/dev/node/box_api/node_modules/box-node-sdk/lib/util/errors.js:94:15)
at /home/andrea/dev/node/box_api/node_modules/box-node-sdk/lib/managers/files.js:148:20
at BoxClient._handleResponse (/home/andrea/dev/node/box_api/node_modules/box-node-sdk/lib/box-client.js:291:2)
at handleResponseCallback (/home/andrea/dev/node/box_api/node_modules/box-node-sdk/lib/box-client.js:233:9)
at /home/andrea/dev/node/box_api/node_modules/box-node-sdk/lib/api-request.js:285:3
at nextTickCallbackWith0Args (node.js:436:9)
at process._tickCallback (node.js:365:13)
Can anyone help me in programmatically downloading a public shared file from box.com?
Thank you in advance!
At the moment I've found this solution.
To me it works pretty well. I hope it will be useful to others as well.
var fs = require("fs"),
request = require("request");
function downloadFile(source, target, callback)
{
var wr = fs.createWriteStream(target);
wr.on("error", function (err)
{
callback(err);
});
wr.on("close", function (ex)
{
callback();
});
request
.get(source)
.on("error",function(err)
{
callback(err);
})
.on('response', function(response)
{
if((""+response.statusCode).charAt(0) != "2")
callback({
msg: "File not found: "+response.request.href,
path: response.request.href,
toString: function(){
return this.msg;
}
});
})
.on("end", function (ex)
{
console.log("request ended");
})
.pipe(wr);
}
function onDownloadComplete(err)
{
if(err)
throw err;
console.log("DOWNLOAD COMPLETE");
}
function init()
{
var static_url = process.argv[2].replace(/\/s\//,"/shared/static/") + ".zip";
console.log(static_url);
downloadFile(static_url, "./output/template.zip", onDownloadComplete);
}
init();
Related
I am using MEAN Stack for our project. To generate pdf using pdfmake,
scenario goes like, At first have images from local server so every thing works fine. For scalibility have to move images to AWS and data from other server.
Now processs is like, first download images from aws then make buffer, convert it to base64 and give it to pdfmake. Now issue arises for client-server connection.
client makes http request for pdf, server process it but it takes too much time to download image from aws, so client connection breaks mean while
server processing the request and send response back but no one is there to listen.
/** client **/
function logicTriplogs() {
$rootScope.isLoading = true;
AssignsService.logicTriplogs({
driverId: vm.driver
}, {
_ids: vm.selectedTrips,
scheduleTime: vm.scheduleTime,
companyImage: vm.companyImage
},
function(response) {
console.log(response);
$rootScope.isLoading = false;
var Name = response.pdfName;
var data = response.content;
SaveFile.downloadURI(Name, data);
console.log('PDF Name:', Name);
},
function(err) {
console.log(err);
vm.error = err.data.message;
$rootScope.isLoading = false;
});
}
/** Server **/
getAssignedTripQuery(query, type)
.exec(function(err, assigns) {
if (err) {
console.log('Manifest');
return res.status(400).send({
message: errorHandler.getErrorMessage(err)
});
}
if (assigns.length) {
logicMan(req.body, user, driver, assigns, function(docDefinition) {
var pdfName = `${moment_tz.tz(startDay, req.user.timeZone).format('MM-DD-YYYY')}.pdf`;
config.pdfBuffer(docDefinition, function(err, pdfDoc) {
console.log('EROROR PDF: ', err);
if (err) {
console.log('pdfmake function call error');
return res.status(400).send({
message: 'Error while generate pdf'
});
}
console.log('PDF Name:', pdfName);
return res.json({
message: 'Generated Susscessfuly',
pdfName: pdfName,
content: pdfDoc
});
});
});
} else {
return res.status(400).send({
message: 'Some thing went wrong. Please try later.'
});
}
});
I am really new in node js and a little bit more experienced in flaks. I am trying to connect a nodejs backend with a flask api. Basically I am sending a file that was uploaded in the nodejs app for processing (converting to another format) to my flask app.
For sending the data I am using request. In this way:
app.post('/converttest', uploader.single('file'), function(req,res){
var file = req.file,
result = {
error: 0,
uploaded: []
};
flow.exec(
function() { // Read temp File
fs.readFile(file.path, this);
},
function(err, data) { // Upload file to S3
var formData = {
file: data,
};
requestPack.post({url:'http://127.0.0.1:5000/api/resource/converter', formData: formData});
},
function(err, httpResponse, body) { //Upload Callback
if (err) {
return console.error('upload failed:', err);
}
res.redirect('/console');
});
});
Then I am receiving the file for processing in the flask app, like:
#app.route('/api/resource/converter', methods = ['POST','GET'])
def converter_csv():
if request.method == 'POST':
f = request.form['file']
if not f:
abort(400)
print('-----Converting-------')
file = open("temp/converting.txt","w")
file.write(f)
#....conversion process...
# Finish the process
return Response(converted_file,status=200)
In my console for the localhost of the flask app, I am getting:
127.0.0.1 - - [09/Aug/2017 15:47:59] "POST /api/resource/converter HTTP/1.1" 200 -
However my nodejs app did not receive any response. It just got frozen.
I appreciate any orientation anyone can give me. Thanks.
I think flow.exec is not in proper order
router.post('/converttest', uploader.single('file'), function(req, res) {
var filePath = req.file.path;
fs.readFile(filePath, 'utf8', function(err, data) { //change format reading as required
try {
formData = {file:data}
requestPack.post({url:'http://127.0.0.1:5000/api/resource/converter', formData: formData});
} catch(err) {
return console.error('upload failed:', err);
res.redirect('/console')
}
fs.unlink(filePath);}); });
I ended up using requestify. Seems like they make it a little bit easier for beginners like me:
var requestify = require('requestify');
app.get('/convertupload', function(req,res){
res.render('pages/convertupload');
});
app.post('/converttest', uploader.single('file'), function(req,res){
var file = req.file,
result = {
error: 0,
uploaded: []
};
flow.exec(
function() { // Read temp File
fs.readFile(file.path,this);
},
function(err, data) { // Upload file to S3
var formData = {
file: data
};
requestify.post('http://127.0.0.1:5000/api/resource/converter', {
form: formData
})
.then(function(response) {
// Get the response body (JSON parsed or jQuery object for XMLs)
console.log(response)
response.getBody();
});
res.redirect('/login');
});
});
We are trying to upload images to a NodeJS server that uses expressjs/multer to parse multipart/form-data input. Unfortunately, we encounter a TypeError like so:
TypeError: Cannot read property 'path' of undefined
at /var/www/app/services/media/multer.js:41:31
at Immediate.<anonymous> (/var/www/app/node_modules/multer/lib/make-middleware.js:52:37)
at Immediate.immediate._onImmediate (timers.js:590:18)
at tryOnImmediate (timers.js:543:15)
at processImmediate [as _immediateCallback] (timers.js:523:5)
[nodemon] app crashed - waiting for file changes before starting.
After looking into it, we figured out that this backend error meant that multer.js didn't add a file field unto the req, thus resulting in a req.file.path ===> TypeError in a later manipulation of the image.
Using Wireshark to sniff the packet, we found out that the method we employed to upload was sending a path rather than a file's content.
Here's the ReactJS code (using superagent):
selectPhotoTapped() {
ImagePicker.showImagePicker({
title: 'Select Image'
}, (response) => {
console.log('Response = ', response);
if (response.didCancel) {
console.log('User cancelled photo picker');
} else if (response.error) {
console.log('ImagePicker Error: ', response.error);
} else if (response.customButton) {
console.log('User tapped custom button: ', response.customButton);
} else {
var promise = new Promise(function(resolve, reject) {
request
.put(Environment.BASE_URL + '/media/img/')
.attach('upload', response.path, response.fileName)
.field('usertype', 'Barber')
.field('name', 'profilePic')
.field('description', 'Holla')
.field('type', 'test:barber:haircut')
.end(function(err, res) {
if (err) {
reject(err);
} else {
resolve(res);
}
});
}).then((response) => {
console.log(response);
alert("HERE");
response.json();
}).then((responseData) => {
alert("Succes " + responseData)
}).catch((error) => {
alert("ERROR " + error)
console.log(error);
});
}
});
}
and here is our backend multer code:
var storage = Multer.diskStorage({
destination: path.join(__dirname, '../../../', config.media.img),
filename: generate_filename
});
var multer = Multer({
storage: storage,
fileSize: 25 * 1024 * 1024,
fileFilter: filter
});
router.put('/media/img/', auth.authenticate, multer.single('upload'), media.img.put);
So the question is: Why does superagent send only the path when we use the attach method? We have tested the exact same superagent code in our mocha test of the backend and everything turned out fine. So why the difference between React and Node?
If you want more information, you can check this print of the request before and after the multer middleware parsed it.
I am running node server on ubuntu.
i am using dropbox module in nodejs.(https://www.npmjs.com/package/dropbox)
node -v = v0.10.38
npm -v = 1.4.28
I am using angularjs in front-end to upload file using (https://github.com/danialfarid/ng-file-upload) library.
Uploading small files < 25mb works fine. It is uploading to the server and uploading to dropbox. But when file if larger like (50mb). It is giving error.
(node) warning: Recursive process.nextTick detected. This will break in the next version of node. Please use setImmediate for recursive deferral.
RangeError: Maximum call stack size exceeded.
Here are relevant codes which i am using :
var client = new Dropbox.Client({
"key": "XXXXXXXXXXX",
"secret": "XXXXXXXXXXXX",
"token": "XXXXXXXXXXXXXXXXXXx",
"uid": "XXXX"
});
app.all('/test', function (req, res) {
console.log(req.files);
var f = req.files.file;
var dbx_file_stat;
var short_url;
var new_file_name = 'generate file name';
fs.readFile(f.path, function (error, data) {
if (error) {
console.log('read error');
return console.log(error);
}
client.writeFile(new_file_name, data, function (error, stat) {
if (error) {
console.log('write error');
return console.log(error);
}
//stopReportingProgress();
client.makeUrl(new_file_name, {downloadHack:true},function (error,url) {
if (error) {
return console.log(error);
}
res.send("it works");
});
});
});
})
The Large files are uploading to the server but unable to upload on dropbox server.
I did some research . Some suggested to use (https://nodejs.org/api/timers.html#timers_setimmediate_callback_arg). But how to implement. what is causing this problem?
I also tried running app through node --stack-size=320000 app.js
New error came after this ,
(node) warning: Recursive process.nextTick detected. This will break in the next version of node. Please use setImmediate for recursive deferral.
Segmentation fault
I can't reproduce your error using the following code and a 75MB file. Perhaps the issue relates to the other parts of your app. (It looks like maybe you're using Express?)
var fs = require('fs'),
dropbox = require('dropbox');
var client = new dropbox.Client({ token: "REDACTED" });
fs.readFile('testfile', function (error, data) {
client.writeFile('testfile', data, function (error, stat) {
if (error) {
console.log('ERROR: ' + error);
} else {
console.log(stat);
}
});
});
If you're still able to reproduce the error, please share a smaller repro.
So I'm new to OAuth and Node.JS, but I have to create a Flickr bot to upload photos for a university project. From my understanding Node doesn't have native support of multipart/formdata requests so I constructed the POST request manually.
I know the following code isn't necessarily clean or best-practice, but it's meant to be a one-off hack: the server is being requested by a single computer and only running for a few hours.
this.post = function(text, image) {
var me = this;
if (me.authenticated) {
fs.readFile(image, 'utf8', function(err,data) {
if (err) throw new Error(err);
var bound = (crypto.createHash('md5').update((Math.random()*9999999).toString()).digest('hex'));
var req = http.request({host:'api.flickr.com',port:80,path:'/services/upload/',method:'POST',
headers:{
'Content-Type':'multipart/form-data; boundary='+bound,
'Content-Length':data.length,
'Connection':'keep-alive'
}
}, function(res) {
res.setEncoding('utf8');
res.on('data', function(chunk) {
console.log(chunk);
});
console.log(me);
});
bound = "--"+bound;
req.on('error', function(msg) {console.log('FLICKR UPLOAD ERROR: '+msg.message);});
req.write(bound+"\r\n");
req.write('Content-Disposition: form-data; name="api_key"\r\n\r\n'+flickroptions.key);
req.write("\r\n"+bound+"\r\n");
console.log("sending token "+me.token);
req.write('Content-Disposition: form-data; name="auth_token"\r\n\r\n'+me.token);
req.write("\r\n"+bound+"\r\n");
req.write('Content-Disposition: form-data; name="api_sig"\r\n\r\n'+function() {
var str = 'api_key'+flickroptions.key;
str += 'api_token'+me.token;
return crypto.createHash('md5').update(str).digest('hex');
}());
req.write("\r\n"+bound+"\r\n");
req.write('Content-Disposition: form-data; name="photo"; filename="'+image+'"\r\n');
req.write('Content-Type: image/jpeg\r\n\r\n');
req.write(data);
req.write('\r\n'+bound+"--"); req.end();
});
}
}
The above function generates a POST request more or less the same as recommended in the documentation. But for some reason, the request responds saying that auth_token is invalid.
I'm setting up authentication with the following two functions:
this.init = function(text,image) {
var me = this;
fauth.getOAuthRequestToken({'oauth_callback':'http://192.168.0.19:7001/flickr'}, function(err, token, secret, results) {
if (err) console.log(err);
if (token && secret) {
me.secret = secret; me.token = token;
me.location = 'http://flicker.com/services/oauth/authorize?perms=write&oauth_token='+token;
}
});
}
//?oauth_token&oauth_verifier
this.auth = function(token, verifier) {
var me = this;
this.token = token; this.verifier = verifier;
fauth.getOAuthAccessToken(this.token, this.secret, this.verifier, function(err,token,secret,results) {
if (err) {console.log(err); return;}
me.results = results; me.authenticated = true;
me.token = token;
me.secret = secret;
me.post('hello world', '../eyes/memory/eyes000001.jpg');
});
}
Where this.init gets a Flickr authentication page and this.auth processes the redirected callback and then calls this.post. fauth is an instance of node-oauth. Am I missing a step in the process or have I messed up along the way? Thanks.
When using OAuth you no longer us the api_key, api_sig and auth_token parameters anymore, even for uploading. You should be using oauth_consumer_key, oauth_signature and oauth_token instead.
An OAuth Token is not a valid old style auth_token, and vice versa.
Check out my blog series on Flickr and OAuth for more details: http://www.wackylabs.net/2011/12/oauth-and-flickr-part-1/