I try to upload a file to redmine with node, I can upload and attach text files, but when I try to upload a binary file I get the token but the file doesn't work. I tried with json, xml and binary, ascii, base64 encoding.
I want upload binary files because I'm doing end to end test testing I want open Issues with screenshots, and upload a report.
I'm using node-rest-client for service calling
Could someone give me any suggestion to fix this problem?
Thanks,
I define the class RMClient
var Client = require('node-rest-client').Client;
var Q = require('q');
var RMClient = function(baseUri, apiToken){
this._apiToken = apiToken;
var client = new Client();
client.registerMethod('openIssue', baseUri+'/issues.json', 'POST');
client.registerMethod('uploadFile', baseUri+'/uploads.json', 'POST');
client.registerMethod('getIssues', baseUri+'/issues.json', 'GET');
this._client = client;
};
option 1:
var deferred = Q.defer();
var file fs.readFileSync(filePath);
//code for sending file to redmine uploads.json
return deferred.promise;
Option 2
var deferred = Q.defer();
var rs = fs.createReadStream(filePath, {'flags': 'r', 'encoding': null, 'autoClose': true});
var size = fs.statSync(filePath).size;
var file = '';
rs.on('error', function(err){
deferred.reject(err);
});
rs.on('data', function(chunk){ file += chunk; });
rs.on('end', function(){
//code for sending file to redmine uploads.json
});
return deferred.promise;
Code that I use to upload the file:
try{
if(!file){
throw new Error('File must\'nt be void');
}
var rmc = new RMClient(myRMURI, myAPItoken);
var headers = {
'X-Redmine-API-Key': rmc._apiToken,
'Content-Type': 'application/octet-stream',
'Accept':'application/json',
'Content-Length': size
};
var args = {
'data':file,
'headers': headers
};
if(parameters){
args.parameters = parameters;
}
rmc._client.methods.uploadFile(args, function(data, response){
if(response.statusCode != 201){
var err = new Error(response.statusMessage);
deferred.reject(err);
return;
}
var attach = JSON.parse(data);
console.log(attach);
if(data.errors){
var msg = ''.concat.apply('', attach.errors.map(function(item, i){
return ''.concat(i+1,'- ',item,(i+1<attach.errors.length)?'\n':'');
}));
console.error(msg);
deferred.reject(Error(msg));
}else{
deferred.resolve(attach.upload.token);
}
});
}catch(err){
console.error(err);
deferred.reject(err);
}
I faced the same issue and solved it this way:
Use "multer";
When you have an uploaded file, make a request using node "request" module, with req.file.buffer as body.
Then uploading files using the Rest API, you have to send the raw file contents in the request body, typically with Content-Type: application/octet-stream. The uploaded file doesn't need any further encoding or wrapping, esp. not as multipart/form-data, JSON or XML.
The response of the POST request to /uploads.xml contains the token to attach the attachment to other objects in Redmine.
Related
i create simple rest api to return media file
var fs = require('fs')
var express = require('express')
var app = express()
app.get('/file.ogg', function (req, res) {
res.set({
'Content-Type': 'audio/ogg',
'Transfer-Encoding': 'chunked'
});
var inputStream = fs.createReadStream(__dirname + '/1.ogg');
inputStream.pipe(res);
});
var server = app.listen(3002);
if i call http://127.0.0.1:3002/file.ogg
server read file 1.ogg and return it in response.
now i use websocket to get file data from external device
socket.on('message', function incoming(message) {
var data = JSON.parse(message);
console.log('fileName: ' + data.fileName);
console.log('fileData : ' + data.fileData.length);
var path =__dirname + '/_' + data.fileName;
var buf = Buffer.from(data.fileData, 'base64');
console.log('buf : ' + buf.length);
// save data to file
fs.appendFile(path ,buf ,function(err){
if(err) throw err;
});
});
i want to return data (buf) in response like first example but without saving any think, how i can do this.
If your files are large, you may want to pull in an additional websocket streaming library which will allow you to stream the data back to the client. socket.io-stream was created specifically to allow that. From their examples:
// send data
ss(socket).on('file', function(stream) {
fs.createReadStream('/path/to/file').pipe(stream);
});
// receive data
ss(socket).emit('file', stream);
stream.pipe(fs.createWriteStream('file.txt'));
SDK Platform: Node.js
How can I upload file (.pdf,.jpg..jpeg) from bot Emulator and convert to binary (base64) without saving file into local drive.
Code Example:
var url = session.message.attachments[0].contentUrl;
var fileName=session.message.attachments[0].name;
var encodedData = new Buffer(fs.createWriteStream(url+"/"+fileName), 'binary').toString('base64');
something like above, but it is not working for me.
Expected Behavior:
We need to upload file from BOT emulator and convert file content into binary data.
Actual Result:
ERROR: ENOENT: no such file or directory, open 'D:\User\projects\messages\http:\localhost:61058\v3\attachments\ne1gmlim5kfg\views\original\filename.pdf'
Here appending current directory which is not required.
For reference: https://github.com/Microsoft/BotBuilder/issues/3628
Here's an example of downloading a file and base64 encoding it in memory:
var bot = new builder.UniversalBot(connector, function (session) {
var msg = session.message;
if (msg.attachments.length) {
// Message with attachment, proceed to download it.
// Skype & MS Teams attachment URLs are secured by a JwtToken, so we need to pass the token from our bot.
var attachment = msg.attachments[0];
var fileDownload = checkRequiresToken(msg) ? requestWithToken(attachment.contentUrl) : request(attachment.contentUrl);
fileDownload.then(
function (response) {
var base64String = new Buffer(response, 'binary').toString('base64');
var echoImage = new builder.Message(session).text('You sent:').addAttachment({
contentType: attachment.contentType,
contentUrl: 'data:' + attachment.contentType + ';base64,' + base64String,
name: 'Uploaded Image'
});
session.send(echoImage);
}).catch(function (err) {
console.log('Error downloading attachment:', { statusCode: err.statusCode, message: err.response.statusMessage });
});
} else {
// No attachments were sent
var reply = new builder.Message(session)
.text('Hi there! This sample is intented to show how can I receive attachments but no attachment was sent to me. Please try again sending a new message with an attachment.');
session.send(reply);
}
});
// Request file with Authentication Header
var requestWithToken = function (url) {
return obtainToken().then(function (token) {
return request({
url: url,
headers: {
'Authorization': 'Bearer ' + token,
'Content-Type': 'application/octet-stream'
}
});
});
};
// Promise for obtaining JWT Token (requested once)
var obtainToken = Promise.promisify(connector.getAccessToken.bind(connector));
var checkRequiresToken = function (message) {
return message.source === 'skype' || message.source === 'msteams';
};
An entire project demonstrating this can be found here: https://github.com/nwhitmont/botframework-node-v3-receive-attachment
I'm new to node.js. What I'm trying to do is to stream the upload of a file from web browser to a cloud storage through my node.js server.
I'm using 'express', 'request' and 'busboy' modules.
var express = require("express");
var request = require("request");
var BusBoy = require("busboy");
var router = express.Router();
router.post("/upload", function(req, res, next) {
var busboy = new BusBoy({ headers: req.headers });
var json = {};
busboy.on("file", function (fieldname, file, filename, encoding, mimetype) {
file.on("data", function(data) {
console.log(`streamed ${data.length}`);
});
file.on("end", function() {
console.log(`finished streaming ${filename}`);
});
var r = request({
url: "http://<my_cloud_storage_api_url>",
method: "POST",
headers: {
"CUSTOM-HEADER": "Hello",
},
formData: {
"upload": file
}
}, function(err, httpResponse, body) {
console.log("uploaded");
json.response = body;
});
});
busboy.on("field", function(name, val) {
console.log(`name: ${name}, value: ${value}`);
});
busboy.on("finish", function() {
res.send(json);
});
req.pipe(busboy);
});
module.exports = router;
But I keep getting the following error on the server. What am I doing wrong here? Any help is appreciated.
Error: Part terminated early due to unexpected end of multipart data
at node_modules\busboy\node_modules\dicer\lib\Dicer.js:65:36
at nextTickCallbackWith0Args (node.js:420:9)
at process._tickCallback (node.js:349:13)
I realize this question is some 7 months old, but I shall answer it here in an attempt help anyone else currently banging their head against this.
You have two options, really: Add the file size, or use something other than Request.
Note: I edited this shortly after first posting it to hopefully provide a bit more context.
Using Something Else
There are some alternatives you can use instead of Request if you don't need all the baked in features it has.
form-data can be used by itself in simple cases, or it can be used with, say, got. request uses this internally.
bhttp advertises Streams2+ support, although in my experience Streams2+ support has not been an issue for me. No built in https support, you have to specify a custom agent
got another slimmed down one. Doesn't have any special handling of form data like request does, but is trivially used with form-data or form-data2. I had trouble getting it working over a corporate proxy, though, but that's likely because I'm a networking newb.
needle seems pretty light weight, but I haven't actually tried it.
Using Request: Add the File Size
Request does not (as of writing) have any support for using transfer-encoding: chunked so to upload files with it, you need to add the file's size along with the file, which if you're uploading from a web client means that client needs to send that file size to your server in addition to the file itself.
The way I came up with to do this is to send the file metadata in its own field before the file field.
I modified your example with comments describing what I did. Note that I did not include any validation of the data received, but I recommend you do add that.
var express = require("express");
var request = require("request");
var BusBoy = require("busboy");
var router = express.Router();
router.post("/upload", function(req, res, next) {
var busboy = new BusBoy({ headers: req.headers });
var json = {};
// Use this to cache any fields which are file metadata.
var fileMetas = {};
busboy.on("file", function (fieldname, file, filename, encoding, mimetype) {
// Be sure to match this prop name here with the pattern you use to detect meta fields.
var meta = fileMetas[fieldname + '.meta'];
if (!meta) {
// Make sure to dump the file.
file.resume();
// Then, do some sort of error handling here, because you cannot upload a file
// without knowing it's length.
return;
}
file.on("data", function(data) {
console.log(`streamed ${data.length}`);
});
file.on("end", function() {
console.log(`finished streaming ${filename}`);
});
var r = request({
url: "http://<my_cloud_storage_api_url>",
method: "POST",
headers: {
"CUSTOM-HEADER": "Hello",
},
formData: {
// value + options form of a formData field.
"upload": {
value: file,
options: {
filename: meta.name,
knownLength: meta.size
}
}
}
}, function(err, httpResponse, body) {
console.log("uploaded");
json.response = body;
});
});
busboy.on("field", function(name, val) {
// Use whatever pattern you want. I used (fileFieldName + ".meta").
// Another good one might be ("meta:" + fileFieldName).
if (/\.meta$/.test(name)) {
// I send an object with { name, size, type, lastModified },
// which are just the public props pulled off a File object.
// Note: Should probably add error handling if val is somehow not parsable.
fileMetas[name] = JSON.parse(val);
console.log(`file metadata: name: ${name}, value: ${value}`);
return;
}
// Otherwise, process field as normal.
console.log(`name: ${name}, value: ${value}`);
});
busboy.on("finish", function() {
res.send(json);
});
req.pipe(busboy);
});
module.exports = router;
On the client, you need to then send the metadata on the so-named field before the file itself. This can be done by ordering an <input type="hidden"> control before the file and updating its value onchange. The order of values sent is guaranteed to follow the order of inputs in appearance. If you're building the request body yourself using FormData, you can do this by appending the appropriate metadata before appending the File.
Example with <form>
<script>
function extractFileMeta(file) {
return JSON.stringify({
size: file.size,
name: file.name,
type: file.type,
lastUpdated: file.lastUpdated
});
}
function onFileUploadChange(event) {
// change this to use arrays if using the multiple attribute on the file input.
var file = event.target.files[0];
var fileMetaInput = document.querySelector('input[name=fileUpload.meta]');
if (fileMetaInput) {
fileMetaInput.value = extractFileMeta(file);
}
}
</script>
<form action="/upload-to-cloud">
<input type="hidden" name="fileUpload.meta">
<input type="file" name="fileUpload" onchange="onFileUploadChange(event)">
</form>
Example with FormData:
function onSubmit(event) {
event.preventDefault();
var form = document.getElementById('my-upload-form');
var formData = new FormData();
var fileUpload = form.elements['fileUpload'];
var fileUploadMeta = JSON.stringify({
size: fileUpload.size,
name: fileUpload.name,
type: fileUpload.type,
lastUpdated: fileUpload.lastUpdated
});
// Append fileUploadMeta BEFORE fileUpload.
formData.append('fileUpload.meta', fileUploadMeta);
formData.append('fileUpload', fileUpload);
// Do whatever you do to POST here.
}
I'm trying to POST a raw body with restify. I have the receive side correct, when using POSTman I can send a raw zip file, and the file is correctly created on the server's file system. However, I'm struggling to write my test in mocha. Here is the code I have, any help would be greatly appreciated.
I've tried this approach.
const should = require('should');
const restify = require('restify');
const fs = require('fs');
const port = 8080;
const url = 'http://localhost:' + port;
const client = restify.createJsonClient({
url: url,
version: '~1.0'
});
const testPath = 'test/assets/test.zip';
fs.existsSync(testPath).should.equal(true);
const readStream = fs.createReadStream(testPath);
client.post('/v1/deploy', readStream, function(err, req, res, data) {
if (err) {
throw new Error(err);
}
should(res).not.null();
should(res.statusCode).not.null();
should(res.statusCode).not.undefined();
res.statusCode.should.equal(200);
should(data).not.null();
should(data.endpoint).not.undefined();
data.endpoint.should.equal('http://endpointyouhit:8080');
done();
});
Yet the file size on the file system is always 0. I'm not using my readStream correctly, but I'm not sure how to correct it. Any help would be greatly appreciated.
Note that I want to stream the file, not load it in memory on transmit and receive, the file can potentially be too large for an in memory operation.
Thanks,
Todd
One thing is that you would need to specify a content-type of multi-part/form-data. However, it looks like restify doesn't support that content type, so you're probably out of luck using the restify client to post a file.
To answer my own question, it doesn't appear to be possible to do this with the restify client. I also tried the request module, which claims to have this capability. However, when using their streaming examples, I always had a file size of 0 on the server. Below is a functional mocha integration test.
const testPath = 'test/assets/test.zip';
fs.existsSync(testPath).should.equal(true);
const readStream = fs.createReadStream(testPath);
var options = {
host: 'localhost'
, port: port
, path: '/v1/deploy/testvalue'
, method: 'PUT'
};
var req = http.request(options, function (res) {
//this feels a bit backwards, but these are evaluated AFTER the read stream has closed
var buffer = '';
//pipe body to a buffer
res.on('data', function(data){
buffer+= data;
});
res.on('end', function () {
should(res).not.null();
should(res.statusCode).not.null();
should(res.statusCode).not.undefined();
res.statusCode.should.equal(200);
const json = JSON.parse(buffer);
should(json).not.null();
should(json.endpoint).not.undefined();
json.endpoint.should.equal('http://endpointyouhit:8080');
done();
});
});
req.on('error', function (err) {
if (err) {
throw new Error(err);
}
});
//pipe the readstream into the request
readStream.pipe(req);
/**
* Close the request on the close of the read stream
*/
readStream.on('close', function () {
req.end();
console.log('I finished.');
});
//note that if we end up with larger files, we may want to support the continue, much as S3 does
//https://nodejs.org/api/http.html#http_event_continue
I am using node.js to create file service for Azure File storage. I am using azure-storage-node (http://azure.github.io/azure-storage-node/) for this.
I am trying to download a file from Azure file storage. Below is my code snippet
// Download a file from Share
exports.get = function(request, response){
var shareName = request.headers.sharename;
var dirPath = request.headers.directorypath;
var fileName = request.headers.filename;
var fileService = azure.createFileService();
var readStream = fileService.createReadStream(shareName, dirPath, fileName);
var dataLength = 0;
var body = '';
readStream.on('data', function (chunk) {
dataLength += chunk.length;
})
readStream.on('end', function(){
console.log('The length was:', dataLength);
});
response.setHeader('Content-Type', 'application/json');
response.send(statusCodes.OK, JSON.stringify("Success!"));
}
I able to get the stream of data. But how can I sent the stream in the response so that we can get it in the rest call.
I tried readStream.pipe(response); and
response.write(typeof chunk);
response.end() but it doesnt work;
I am new to node.js. Please help me on this.
Updated:
I tried the following.
response.writeHead(200, {'Content-Type': 'application/json'});
var readStream = fileService.createReadStream(shareName, dirPath, fileName);
readStream.pipe(response);
But its throwing follwing error.
ERROR
An unhandled exception occurred. Error: Can't set headers after they are sent.
at ServerResponse.OutgoingMessage.setHeader (http.js:679:11)
at ServerResponse.res.setHeader (D:\home\site\wwwroot\node_modules\express\node_modules\connect\lib\patch.js:59:22)
at ServerResponse.res.set.res.header (D:\home\site\wwwroot\node_modules\express\lib\response.js:518:10)
at addDefaultHeaders (D:\home\site\wwwroot\runtime\request\requesthandler.js:582:9)
at ServerResponse.<anonymous> (D:\home\site\wwwroot\runtime\request\requesthandler.js:291:13)
at ServerResponse._.wrap [as end] (D:\home\site\wwwroot\node_modules\underscore\underscore.js:692:22)
at ChunkStream.onend (stream.js:66:10)
at ChunkStream.EventEmitter.emit (events.js:126:20)
at ChunkStream.end (D:\home\site\wwwroot\App_Data\config\scripts\node_modules\azure-storage\lib\common\streams\chunkstream.js:90:8)
at Request.onend (stream.js:66:10)
The return datatype of fileService.createReadStream(shareName, dirPath, fileName); is ChunkStream
Updated:
This is my updated code which works.
var option = new Object();
option.disableContentMD5Validation = true;
option.maximumExecutionTimeInMs = 20 * 60000;
option.timeoutIntervalInMs = 20 * 6000;
fileService.getFileToStream(shareName, dirPath, fileName, response, option, function(error, result, res) {
if(!error) {
if(res.isSuccessful) {
console.log(result);
console.log(res);
console.log("Success!");
}
}
});
But more frequently I am getting below error.
ERROR
An unhandled exception occurred. Error: Can't set headers after they are sent.
at ServerResponse.OutgoingMessage.setHeader (http.js:679:11)
at ServerResponse.res.setHeader (D:\home\site\wwwroot\node_modules\express\node_modules\connect\lib\patch.js:59:22)
at ServerResponse.res.set.res.header (D:\home\site\wwwroot\node_modules\express\lib\response.js:518:10)
at addDefaultHeaders (D:\home\site\wwwroot\node_modules\azure-mobile-services\runtime\request\requesthandler.js:590:9)
at ServerResponse. (D:\home\site\wwwroot\node_modules\azure-mobile-services\runtime\request\requesthandler.js:299:13)
at ServerResponse._.wrap as end
at Request.onend (stream.js:66:10)
at Request.EventEmitter.emit (events.js:126:20)
at IncomingMessage.Request.onRequestResponse.strings (D:\home\site\wwwroot\App_Data\config\scripts\node_modules\azure-storage\node_modules\request\request.js:1153:12)
at IncomingMessage.EventEmitter.emit (events.js:126:20)
The NodeJS Class http.ServerResponse implements the Writable Stream interface, please refer to the NodeJS API https://nodejs.org/api/http.html#http_class_http_serverresponse and https://nodejs.org/api/stream.html#stream_class_stream_writable_1.
So you just need to use the object response instead of the stream writer fs.createStreamWriter(...) in the sample code "getFileToStream" http://azure.github.io/azure-storage-node/#toc8 when you send data stream into response for NodeJS.
This is my sample code as below:
var http = require('http');
var azure = require('azure-storage');
var fileService = azure.createFileService('<storage_key_name>','<storage_access_key>');
http.createServer(function (request, response) {
var shareName = request.headers.sharename;
var dirPath = request.headers.directorypath;
var fileName = request.headers.filename;
response.setHeader('Content-Type', 'application/json');
fileService.getFileToStream(shareName, dirPath, fileName, response, {disableContentMD5Validation: true}, function(error, result, response) {
if(!error) {
//console.log(result);
//console.log(response);
if(response.isSuccessful) {
console.log("Success!");
}
}
});
}).listen(1337, "127.0.0.1");
console.log('Server running at http://127.0.0.1:1337/');
Best Regards.
For getting File greater than 4MB from Azure File Storage, there is a request header x-ms-range-get-content-md5 that it will cause the status code 400(Bad Request) error, please refer to the Get File REST API doc of Azure File Storage https://msdn.microsoft.com/en-us/library/azure/dn194274.aspx, see below:
So I reviewed the source of Azure File Storage SDK for Node (https://github.com/Azure/azure-storage-node/blob/master/lib/services/file/fileservice.js). For the function getFileToText, getFileToLocalFile, createReadStream and getFileToStream, you need to set the options.disableContentMD5Validation attribute to avoid the error, see below.
#param {boolean} [options.disableContentMD5Validation] When set to true, MD5 validation will be disabled when downloading files.
And refer to the source of getFileToStream as example:
In my sample code, need to add the code {disableContentMD5Validation: true} as options at the front of invoking the function getFileToStream.
You might want to try in this way:
exports.get = function(request, response) {
var shareName = request.headers.sharename;
var dirPath = request.headers.directorypath;
var fileName = request.headers.filename;
var fileService = azure.createFileService();
var readStream = fileService.createReadStream(shareName, dirPath, fileName);
var dataLength = 0;
readStream.on('data', function (chunk) {
dataLength += chunk.length;
})
readStream.on('end', function(){
response.setHeader('Content-Type', 'application/json');
response.setHeader('Content-Length', dataLength);
});
readStream.pipe(response);
response.on('finish', function (chunk) {
response.send(statusCodes.OK, JSON.stringify("Success!"));
})
}