I am sending a multipart data from a nodeJS route as a response.
I used form-data library to accomplish, my requirement is to send additional header information for all binary data.
I tried the following
router.get('/getdata', async (req, res, next) => {
var form = new FormData();
var encodedImage2 = fs.readFileSync('./public/image2.png');
var CRLF = '\r\n';
var options = {
header: CRLF + '--' + form.getBoundary() + CRLF + 'X-Custom-Header: 123' + CRLF + CRLF
};
form.append('image2.png', encodedImage2, options);
res.set('Content-Type', 'multipart/form-data; boundary=' + form.getBoundary());
form.pipe(res);
});
in the output I get only the header
X-Custom-Header: 123
without the option object I can get
Content-Disposition: form-data; name="image2.png"
Content-Type: application/octet-stream
I need an output headers something like
Content-Disposition: form-data; name="image2.png"
Content-Type: application/octet-stream
X-custom-Header: 123
I found the solution myself, and its very simple.
Its posible to set custom headers in the options object header property
router.get('/getdata', async (req, res, next) => {
var form = new FormData();
var encodedImage2 = fs.readFileSync('./public/image2.png');
var options = {
header: {
'X-Custom-Header': 123
}
};
form.append('image2.png', encodedImage2, options);
res.set('Content-Type', 'multipart/form-data; boundary=' + form.getBoundary());
form.pipe(res);
});
Just set your headers in express's res object like you do with the Content-Type:
router.get('/getdata', async (req, res, next) => {
var form = new FormData();
form.append('image2.png', fs.readFileSync('./public/image2.png'));
res.set('Content-Type', 'multipart/form-data; boundary=' + form.getBoundary());
res.set('X-Custom-Header', '123'); // Set header here
form.pipe(res);
});
Related
tldr I'm having encoding issues when sending zip files as part of a multipart/form-data request body. Help please :/
I'm working on a bot that should be able to upload zip files to Slack (via their file api), but I'm running into some issues that I believe are related to encoding.
So, I'm creating my request body as follows:
var form_string = "\n--abcdefghijklmnop\nContent-Disposition: form-data; filename=\"" + filename + "\"; name=\"file\";\nContent-Type:application/octet-stream;\nContent-Transfer-Encoding:base64;\n\n" + data;
form_string += "\n--abcdefghijklmnop\nContent-Disposition: form-data; name=\"token\";\n\n" + token;
form_string += "\n--abcdefghijklmnop\nContent-Disposition: form-data; name=\"filetype\";\n\n" + filetype;
form_string += "\n--abcdefghijklmnop\nContent-Disposition: form-data; name=\"filename\";\n\n" + filename;
form_string += "\n--abcdefghijklmnop\nContent-Disposition: form-data; name=\"channels\";\n\n" + channel;
form_string += "\n--abcdefghijklmnop\nContent-Disposition: form-data; name=\"title\";\n\n" + title;
form_string += "\n--abcdefghijklmnop--";
var form = Buffer.from(form_string, "utf8");
var headers = {
"Content-Type": "multipart/form-data; boundary=abcdefghijklmnop",
"Content-Length": form.length,
"Authorization": "Bearer ....."
};
var options = {
"headers": headers,
"body": form
};
// using the sync-request node module.
var res = request("POST", url, options);
var res = request("POST", url, options);
(I've tried application/zip and application/x-zip-compressed as well. I've also tried both binary and base64 content transfer encodings.)
(And in case you're wondering, I need to make synchronous http requests...)
I created a really small zip file as a test. The base64 encoding of it is below:
UEsDBAoAAAAAAAqR+UoAAAAAAAAAAAAAAAAIABwAdGlueXppcC9VVAkAA1PBd1mDwXdZdXgLAAEE9QEAAAQUAAAAUEsDBAoAAAAAAAuR+Up6em/tAwAAAAMAAAAQABwAdGlueXppcC90aW55LnR4dFVUCQADVsF3WVzBd1l1eAsAAQT1AQAABBQAAABoaQpQSwECHgMKAAAAAAAKkflKAAAAAAAAAAAAAAAACAAYAAAAAAAAABAA7UEAAAAAdGlueXppcC9VVAUAA1PBd1l1eAsAAQT1AQAABBQAAABQSwECHgMKAAAAAAALkflKenpv7QMAAAADAAAAEAAYAAAAAAABAAAApIFCAAAAdGlueXppcC90aW55LnR4dFVUBQADVsF3WXV4CwABBPUBAAAEFAAAAFBLBQYAAAAAAgACAKQAAACPAAAAAAA=
What I'm getting from Slack seems to be similar to the original... maybe...
UEsDBAoAAAAAAArCkcO5SgAAAAAAAAAAAAAAAAgAHAB0aW55emlwL1VUCQADU8OBd1nCg8OBd1l1eAsAAQTDtQEAAAQUAAAAUEsDBAoAAAAAAAvCkcO5Snp6b8OtAwAAAAMAAAAQABwAdGlueXppcC90aW55LnR4dFVUCQADVsOBd1lcw4F3WXV4CwABBMO1AQAABBQAAABoaQpQSwECHgMKAAAAAAAKwpHDuUoAAAAAAAAAAAAAAAAIABgAAAAAAAAAEADDrUEAAAAAdGlueXppcC9VVAUAA1PDgXdZdXgLAAEEw7UBAAAEFAAAAFBLAQIeAwoAAAAAAAvCkcO5Snp6b8OtAwAAAAMAAAAQABgAAAAAAAEAAADCpMKBQgAAAHRpbnl6aXAvdGlueS50eHRVVAUAA1bDgXdZdXgLAAEEw7UBAAAEFAAAAFBLBQYAAAAAAgACAMKkAAAAwo8AAAAAAA==
Could someone explain what encoding is going on here and how I can correctly upload a file to Slack? Thanks!
How about following sample scripts? There are 2 patterns for this situation.
Sample script 1 :
For this, I modified the method you are trying. You can upload the zip file by converting to the byte array as follows. At first, it builds form-data. It adds the zip file converted to byte array and boundary using Buffer.concat(). This is used as body in request.
var fs = require('fs');
var request = require('request');
var upfile = 'sample.zip';
fs.readFile(upfile, function(err, content){
if(err){
console.error(err);
}
var token = '### access token ###';
var filetype = 'zip';
var filename = 'samplefilename';
var channel = 'sample';
var title = 'sampletitle';
var formString = "\n--abcdefghijklmnop\nContent-Disposition: form-data; name=\"token\";\n\n" + token;
formString += "\n--abcdefghijklmnop\nContent-Disposition: form-data; name=\"filetype\";\n\n" + filetype;
formString += "\n--abcdefghijklmnop\nContent-Disposition: form-data; name=\"filename\";\n\n" + filename;
formString += "\n--abcdefghijklmnop\nContent-Disposition: form-data; name=\"channels\";\n\n" + channel;
formString += "\n--abcdefghijklmnop\nContent-Disposition: form-data; name=\"title\";\n\n" + title;
formString += "\n--abcdefghijklmnop\nContent-Disposition: form-data; filename=\"" + upfile + "\"; name=\"file\";\nContent-Type:application/octet-stream;\n\n";
var options = {
method: 'post',
url: 'https://slack.com/api/files.upload',
headers: {"Content-Type": "multipart/form-data; boundary=abcdefghijklmnop"},
body: Buffer.concat([
Buffer.from(formString, "utf8"),
new Buffer(content, 'binary'),
Buffer.from("\n--abcdefghijklmnop\n", "utf8"),
]),
};
request(options, function(error, response, body) {
console.log(body);
});
});
Sample script 2 :
This is a simpler way than sample 1. You can use fs.createReadStream() as a file for uploading to Slack.
var fs = require('fs');
var request = require('request');
request.post({
url: 'https://slack.com/api/files.upload',
formData: {
file: fs.createReadStream('sample.zip'),
token: '### access token ###',
filetype: 'zip',
filename: 'samplefilename',
channels: 'sample',
title: 'sampletitle',
},
}, function(error, response, body) {
console.log(body);
});
Result :
Both sample 1 and sample 2 can be uploaded zip file to Slack as follows. For both, even if filetype is not defined, the uploaded file is used automatically as a zip file.
If I misunderstand your question, I'm sorry.
How can I POST multiple binary files to a server with Content-Type: "form-data", using only the http module?
For example, my keys would look like this:
{
Image1: <binary-data>,
Image2: <binary-data>
}
TL:DR; See the full example code at the bottom of this answer.
I was trying to figure out how to POST multiple binary image files to a server in NodeJS using only core NodeJs modules (without using anything like npm install request, etc). After about 3 hours typing the wrong search terms into DuckDuckGo and finding no examples online, I was about to switch careers. But after banging my head against the desk for almost half the day, my sense of urgency was dulled and I managed to hobble together a working solution. This would not have been possible without the good people who wrote the answer to this Stackoverflow post, and this Github Gist. I used PostMan and Charles Proxy to analyze successful HTTP POSTS, as I dug around in the NodeJS docs.
There are a few things to understand to POST two binary images and a text field as multipart/form-data, relying only on core NodeJS modules:
1) Boundary Identifiers
What is the boundary in multipart/form-data?
The first part of the solution is to create a "boundary identifier", which is a string of dashes -, appended with a random number. You could probably use whatever you wish, foorbar and such.
------------0123456789
Then place that boundary between each blob of data; whether that is binary data or just text data. When you have listed all your data, you add the boundary identifier at the end, and append two dashes:
------------0123456789--
You also need to add the boundary to the headers so that server receiving the post understands which lines of your post data form the boundary between fields.
const headers = {
// Inform the server what the boundary identifier looks like
'Content-Type': `multipart/form-data; boundary=${partBoundary}`,
'Content-Length': binaryPostData.length
}
2) Form Field Meta Descriptors
(This probably is not what they are called)
You will also need a way to write the meta data for each form-field you send, whether that form field contains a binary or a text object. Here are the descriptors for an image file, which contain the mime type:
Content-Disposition: form-data; name="Image1"; filename="image-1.jpg"
Content-Type: image/jpeg
And here is the descriptor for a text field:
Content-Disposition: form-data; name="comment"
The Post Data Output
So the entire post data that is sent to the server will look like this:
----------------------------9110957588266537
Content-Disposition: form-data; name="Image1"; filename="image-1.jpg"
Content-Type: image/jpeg
ÿØÿàJFIFHHÿáLExifMMi
ÿí8Photoshop 3.08BIM8BIM%ÔÙ²é ìøB~ÿÀ... <<<truncated for sanity>>>
----------------------------9110957588266537
Content-Disposition: form-data; name="Image2"; filename="image-2.jpg"
Content-Type: image/jpeg
ÿØÿàJFIFHHÿáLExifMMi
ÿí8Photoshop 3.08BIM8BIM%ÔÙ²é ìøB~ÿÀ... <<<truncated for sanity>>>
----------------------------9110957588266537
Content-Disposition: form-data; name="comment"
This is a comment.
----------------------------9110957588266537--
Once this post data is generated, it can be converted to binary and written to the HTTP POST request: request.write(binaryPostData).
Example Code
Here is the full example code that will allow you to POST binary file and text data without having to include other NodeJS libraries and packages in your code.
// This form data lists 2 binary image fields and text field
const form = [
{
name: 'Image1',
type: 'file',
value: 'image-1.jpg'
},
{
name: 'Image2',
type: 'file',
value: 'image-2.jpg'
},
{
name: 'comment',
type: 'text',
value: 'This is a comment.'
}
]
// Types of binary files I may want to upload
const types = {
'.json': 'application/json',
'.jpg': 'image/jpeg'
}
const config = {
host: 'ec2-192.168.0.1.compute-1.amazonaws.com',
port: '80',
path: '/api/route'
}
// Create an identifier to show where the boundary is between each
// part of the form-data in the POST
const makePartBoundary = () => {
const randomNumbers = (Math.random() + '').split('.')[1]
return '--------------------------' + randomNumbers
}
// Create meta for file part
const encodeFilePart = (boundary, type, name, filename) => {
let returnPart = `--${boundary}\r\n`
returnPart += `Content-Disposition: form-data; name="${name}"; filename="${filename}"\r\n`
returnPart += `Content-Type: ${type}\r\n\r\n`
return returnPart
}
// Create meta for field part
const encodeFieldPart = (boundary, name, value) => {
let returnPart = `--${boundary}\r\n`
returnPart += `Content-Disposition: form-data; name="${name}"\r\n\r\n`
returnPart += value + '\r\n'
return returnPart
}
const makePostData = {
// Generate the post data for a file
file: (item, partBoundary) => {
let filePostData = ''
// Generate the meta
const filepath = path.join(__dirname, item.value)
const extention = path.parse(filepath).ext
const mimetype = types[extention]
filePostData += encodeFilePart(partBoundary, mimetype, item.name, item.value)
// Add the binary file data
const fileData = fs.readFileSync(filepath, 'binary')
filePostData += fileData
filePostData += '\r\n'
return filePostData
},
// Generate post data for the text field part of the form
text: (item, partBoundary) => {
let textPostData = ''
textPostData += encodeFieldPart(partBoundary, item.name, item.value)
return textPostData
}
}
const post = () => new Promise((resolve, reject) => {
let allPostData = ''
// Create a boundary identifier (a random string w/ `--...` prefixed)
const partBoundary = makePartBoundary()
// Loop through form object generating post data according to type
form.forEach(item => {
if (Reflect.has(makePostData, item.type)) {
const nextPostData = makePostData[item.type](item, partBoundary)
allPostData += nextPostData
}
})
// Create the `final-boundary` (the normal boundary + the `--`)
allPostData += `--${partBoundary}--`
// Convert the post data to binary (latin1)
const binaryPostData = Buffer.from(allPostData, 'binary')
// Generate the http request options
const options = {
host: config.host,
port: config.port,
path: config.path,
method: 'POST',
headers: {
// Inform the server what the boundary identifier looks like
'Content-Type': `multipart/form-data; boundary=${partBoundary}`,
'Content-Length': binaryPostData.length
}
}
// Initiate the HTTP request
const req = http.request(options, res => {
res.setEncoding('utf8')
let body = ''
// Accumulate the response data
res.on('data', chunk => {
body += chunk
})
// Resolve when done
res.on('end', () => {
resolve(body)
})
res.on('close', () => {
resolve(body)
})
res.on('error', err => {
reject(err)
})
})
// Send the binary post data to the server
req.write(binaryPostData)
// Close the HTTP request object
req.end()
})
// Post and log response
post().then(data => {
console.log(data)
})
.catch(err => {
console.error(err)
})
Is there any way to force download when i pipe stream to response?
If I look into Chrome tools, I see that response is OK, with fine headers:
HTTP/1.1 200 OK
Server: Cowboy
Connection: keep-alive
X-Powered-By: Express
Content-Type: application/pdf
Date: Mon, 10 Oct 2016 20:22:51 GMT
Transfer-Encoding: chunked
Via: 1.1 vegur
Request Headers
view source
I even see the pdf file code in detailed response, but no file download is initiated. Maybe I miss something?
My code on route looks like:
router.post('/reports/create', access.Regular, function (req, res, next) {
...
pdf.create(html).toBuffer(function(err, buffer){
res.writeHead(200, {
'Content-Type': 'application/pdf',
'Content-Disposition': 'attachment; filename=some_file.pdf',
'Content-Length': buffer.length
});
res.end(buffer)
});
});
You need to add a Content-Disposition Header to signal to the browser that there is content attached that needs to be downloaded.
app.get('/:filename', (req, res) => {
// Do whatever work to retrieve file and contents
// Set Content-Disposition Header on the response
// filename is the name of the file the browser needs to download when
// receiving the response to this particular request
res.setHeader('Content-Disposition', `attachment; filename=${req.params.filename}`);
// stream the fileContent back to the requester
return res.end(fileContent)
});
it works for me like this:
var html = '<h1>Hello World</h1>';
let options = {
format: 'Letter',
border: '1cm'
};
pdf.create(html, options).toBuffer(function (err, Buffer) {
if (err) {
res.json({
data: 'error PDF'
})
} else {
res.set({
'Content-Disposition': 'attachment; filename=test.pdf',
'Content-Type': 'application/pdf; charset=utf-8'
});
res.write(Buffer);
res.end();
}
});
I'm trying to post images to Twitter using the Oauth module. Here is what I have:
It throws a 403 error, I know im doing something wrong with how I add the media to the post but Im just not sure where to go from here.
var https = require('https');
var OAuth= require('oauth').OAuth;
var keys = require('./twitterkeys');
var twitterer = new OAuth(
"https://api.twitter.com/oauth/request_token",
"https://api.twitter.com/oauth/access_token",
keys.consumerKey,
keys.consumerSecret,
"1.0",
null,
"HMAC-SHA1"
);
var params = {
status : "Tiger!",
media : [("data:" + mimeType + ";base64,") + fs.readFileSync(path,'base64')]
};
//function(url, oauth_token, oauth_token_secret, post_body, post_content_type, callback)
twitterer.post("https://upload.twitter.com/1/statuses/update_with_media.json",
keys.token, keys.secret, params, "multipart/form-data",
function (error, data, response2) {
if(error){
console.log('Error: Something is wrong.\n'+JSON.stringify(error)+'\n');
}else{
console.log('Twitter status updated.\n');
console.log(response2+'\n');
}
});
Here is what I belive im supose to be doing but I don't know how to do that in the Node.js Oauth module.
Posting image to twitter using Twitter+OAuth
Reviewing the code, it looks like there's no multipart/form-data handling at all in the node-oauth package right now. You can still use the node-oauth function to create the authorization header, but you'll have to do the multipart stuff on your own.
There are probably third-party libraries that can help out with that, but here's how I got it to work constructed by hand.
var data = fs.readFileSync(fileName);
var oauth = new OAuth(
'https://api.twitter.com/oauth/request_token',
'https://api.twitter.com/oauth/access_token',
twitterKey, twitterSecret,
'1.0', null, 'HMAC-SHA1');
var crlf = "\r\n";
var boundary = '---------------------------10102754414578508781458777923';
var separator = '--' + boundary;
var footer = crlf + separator + '--' + crlf;
var fileHeader = 'Content-Disposition: file; name="media"; filename="' + photoName + '"';
var contents = separator + crlf
+ 'Content-Disposition: form-data; name="status"' + crlf
+ crlf
+ tweet + crlf
+ separator + crlf
+ fileHeader + crlf
+ 'Content-Type: image/jpeg' + crlf
+ crlf;
var multipartBody = Buffer.concat([
new Buffer(contents),
data,
new Buffer(footer)]);
var hostname = 'upload.twitter.com';
var authorization = oauth.authHeader(
'https://upload.twitter.com/1/statuses/update_with_media.json',
accessToken, tokenSecret, 'POST');
var headers = {
'Authorization': authorization,
'Content-Type': 'multipart/form-data; boundary=' + boundary,
'Host': hostname,
'Content-Length': multipartBody.length,
'Connection': 'Keep-Alive'
};
var options = {
host: hostname,
port: 443,
path: '/1/statuses/update_with_media.json',
method: 'POST',
headers: headers
};
var request = https.request(options);
request.write(multipartBody);
request.end();
request.on('error', function (err) {
console.log('Error: Something is wrong.\n'+JSON.stringify(err)+'\n');
});
request.on('response', function (response) {
response.setEncoding('utf8');
response.on('data', function (chunk) {
console.log(chunk.toString());
});
response.on('end', function () {
console.log(response.statusCode +'\n');
});
});
I want to send plain html instead of a json response for one of my routes in restify. I tried setting the contentType and header property of the response but it doesn't seem to set the contentType in the header (the browser tries to download the file rather than render it).
res.contentType = 'text/html';
res.header('Content-Type','text/html');
return res.send('<html><body>hello</body></html>');
Quick way to manipulate headers without changing formatters for the whole server:
A restify response object has all the "raw" methods of a node ServerResponse on it as well.
var body = '<html><body>hello</body></html>';
res.writeHead(200, {
'Content-Length': Buffer.byteLength(body),
'Content-Type': 'text/html'
});
res.write(body);
res.end();
If you've overwritten the formatters in the restify configuration, you'll have to make sure you have a formatter for text/html. So, this is an example of a configuration that will send json and jsonp-style or html depending on the contentType specified on the response object (res):
var server = restify.createServer({
formatters: {
'application/json': function(req, res, body){
if(req.params.callback){
var callbackFunctionName = req.params.callback.replace(/[^A-Za-z0-9_\.]/g, '');
return callbackFunctionName + "(" + JSON.stringify(body) + ");";
} else {
return JSON.stringify(body);
}
},
'text/html': function(req, res, body){
return body;
}
}
});
Another option is to call
res.end('<html><body>hello</body></html>');
Instead of
res.send('<html><body>hello</body></html>');
It seems like the behaviour #dlawrence describes in his answer has changed since when the answer was posted. The way it works now (at least in Restify 4.x) is:
const app = restify.createServer(
{
formatters: {
'text/html': function (req, res, body, cb) {
cb(null, body)
}
}
})