i m writing a localhost chrome app by using chrome.sockets.tcpServer API
the app basically fetches some files from some remote url and then serves them
all the text files are being rendered well
here is a snippet of what i m doing with the text file
const createResponse = (content, type) => {
var sep = '\r\n';
var header = [
'HTTP/1.1 200 OK',
'Server: chrome24',
'Content-Length: ' + content.length,
'Connection: Close',
'Content-Type: ' + type
].join(sep);
return header + sep + sep + content;
}
string2ArrayBuffer(createResponse("html string from a fetch request", "text/html"), (buffer) => {
chrome.sockets.tcp.send(info.clientSocketId, buffer, (resultCode) => {
console.log("Data sent to new TCP client connection.", resultCode)
});
});
now how to send images or other media files in chrome.sockets.tcp.send
please help
Related
How can I convert an incoming NodeJS/Express Request (req) object into something that can be sent over in a proxy request?
My NodeJS/express service uses node-fetch to proxy requests to a separate server. The incoming curl request looks like this (Firefox POSIX curl)
curl 'https://localhost:8080/bluecost/spreadsheet/upload/SSCSpreadsheetUpload-09192020.xlsx' -H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:85.0) Gecko/20100101 Firefox/85.0' -H 'Accept: application/json, text/plain, */*' -H 'Accept-Language: en-US,en;q=0.5' --compressed -H 'Content-Type: multipart/form-data; boundary=---------------------------37552304939101372432093632492' -H 'Origin: https://localhost:8080' -H 'DNT: 1' -H 'Connection: keep-alive' -H 'Referer: https://localhost:8080/spreadsheet-upload' -H 'Cookie: connect.sid=s%3ARxnRck1VzcCXk05LKV1CJ5PeslCK1sWC.WqqK%2B0CHsAP0MpcFzRFbHTh19YOoeBi0mIAKCPQ%2BSGU; access_token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzcHJpbnQtbWlkZGxld2FyZS1pc3N1ZXIiLCJzdWIiOiJCZW4uUHJhY2h0MUBpYm0uY29tIiwibm90ZXNJZCI6IkJlbiBQcmFjaHQiLCJzZXJpYWxOdW0iOiI4Njc1NTU4OTciLCJleHAiOjE2MTQ4MjgwNzksImJsdWVHcm91cHMiOlsiQkxVRUNPU1RfU1BSRUFEU0hFRVRfVVBMT0FEX1RFU1QiXSwiaWF0IjoxNjE0ODIwODk3fQ.jWhvsHZiJLRvAHG7SDTmOkdcpMRHxiaLUWDhfJmRXv0' -H 'Pragma: no-cache' -H 'Cache-Control: no-cache' --data-binary $'-----------------------------37552304939101372432093632492\r\nContent-Disposition: form-data; name="file"; filename="SSCSpreadsheetUpload-09192020.xlsx"\r\nContent-Type: application/vnd.openxmlformats-officedocument.spreadsheetml.sheet\r\n\r\n-----------------------------37552304939101372432093632492--\r\n'
A simplified version of the routing looks like this:
app.use('/multipart',(req,res,next) => {
const URL = require('url').URL;
var spreadsheet_client_url = new URL(config.spreadsheet_client_host);
spreadsheet_client_url.pathname = req.originalUrl;
// spreadsheet_client_url.pathname = spreadsheet_client_url.pathname.replace('/spreadsheet-upload', '/');
logger.debug('/spreadsheet-upload: Generating token based on proxyReqOpts.user=' + req.user);
logger.info(`##Request ${req.method} ${spreadsheet_client_url.href}`);
var accessToken = generateToken(req.user);
/**
* Set bearer token based on code from Sprint-API's lua code
*/
// req.setHeader('Authorization', 'Bearer ' + accessToken);
logger.debug('Route: /spreadsheet-upload: adding Authorization: Bearer ' + accessToken);
var newHeaders = req.headers;
newHeaders['Authorization'] = 'Bearer ' + accessToken;
if (isMultipartRequest(req)) {
newHeaders['Content-Length'] = req.body.length;
// build a string in multipart/form-data format with the data you need
const formdataUser =
`--${request.headers['content-type'].replace(/^.*boundary=(.*)$/, '$1')}\r\n` +
`Content-Disposition: form-data; name="reqUser"\r\n` +
`\r\n` +
`${JSON.stringify(req.user)}\r\n`
} else {
const options = {
port: spreadsheet_client_url.port,
path: spreadsheet_client_url.pathname,
body: req.body,
method: req.method,
headers: newHeaders,
redirect: true
};
console.log('###');
console.log('### url: ' + spreadsheet_client_url.href + 'post-options: ' + JSON.stringify(options));
console.log('###');
const fetch = require('node-fetch');
fetch(spreadsheet_client_url, options).then(r => {
console.log('## reached fetch result retrieved');
return r.text();
}).then(fetchText => {
console.log('## reached fetch text result retrieved');
/*
// This code complains that headders can't be set after sending to client
res.writeHead(res.statusCode,res.headers);
res.send(fetchText);
*/
// res.body=fetchText;
res.status(200).send(fetchText);
next();
}
}
)
.catch(err => { console.log('Error fetching ' + err) });
})
function isMultipartRequest(req) {
let contentTypeHeader = req.headers['content-type'];
return contentTypeHeader && contentTypeHeader.indexOf('multipart') > -1;
}
So I have an incoming request, and I believe I can reliably distinguish between normal and multi-part requests. Many examples suggest creating this "form-data" object, then sending it over like this:
const FormData = require('form-data');
const form = new FormData();
const buffer = // e.g. `fs.readFileSync('./fileLocation');
const fileName = 'test.txt';
form.append('file', buffer, {
contentType: 'text/plain',
name: 'file',
filename: fileName,
});
fetch('https://httpbin.org/post', { method: 'POST', body: form })
.then(res => res.json())
.then(json => console.log(json));
I'd like to keep using node-fetch because it's the devil I know best. Should I use a form-data object and pass it over like above? Is there a better way? I'd like to know how to build a form-data object from a request object given I only know that it's multi-part, nothing else.
The ultimate destination is a Java Spring-Boot server that takes a MultipartFile file as follows:
#PostMapping("/spreadsheet/upload/{filename}")
public ResponseEntity<HashMap<String,Object>> uploadSpreadsheet(#RequestBody MultipartFile file, #PathVariable("filename") String filename) {
With normal non-multi-part form requests, it matches enough to reach this method, but the "file" object is null.
i got a problem here.
Lets start from we have API, this API returns data with next headers:
Content-Type image/png
Transfer-Encoding chunked
Connection keep-alive
And response body where only file is.
When im try to write this body data into file - it's always broken. I mean i've binary data, his mime, body.length is match original filesize, but this image could not be opened in any viewer after i save it.
What i'm do:
public userFile(req, res: Response) {
const data = {
fileId: parseInt(req.body.fileId),
};
let params = {
headers: {
'Authorization': keys.token,
},
};
axios.post('/api/getfile/', data, params,)
.then((response: AxiosResponse) => {
const fs = require('fs');
const dir = require('path').resolve(__dirname + '../../../files/storage');
const ext = {
'image/png': '.png'
};
fs.writeFile(dir + '/' + img + ext[response.headers['content-type']], response.data, (er) => {
res.send(response.data);
});
})
.catch((err) => {
logger.error("AXIOS ERROR: ", err)
})
}
BUT! When i get this file with postman... here it is!
So, i need your help - what i do wrong?
You must specifically declare which response type it is in params as
responseType: 'stream'
then save stream to a file using pipe
response.data.pipe(fs.createWriteStream("/dir/xyz.png"))
I am receiving a PDF file from a node server (it is running jsreport in this server) and i need to download this PDF in the client (i am using react in the client) but the problem is that when i download the file, it comes all blank and the title some strange symbols. After a lot of tests and researchs, i found that the problem may be that the file is coming enconded as chunked (i can see that in the headers of the response) and i need to decode do become a file again.
So, how to decode this chunked string to a file again?
In the client i am just downloading the file that comes in the responde:
handleGerarRelatorioButtonClick(){
axios.post(`${REQUEST_URL}/relatorios`, this.state.selectedExam).then((response) => {
fileDownload(response.data, this.state.selectedExam.cliente.nome.replace(' ', '_') + ".pdf");
});
}
In my server, i am making a request to my jsreport that is other node server and it returns the report as a PDF:
app.post('/relatorios', (request, response) => {
var exame = new Exame(request.body);
var pdf = '';
var body = {
"template": {
"shortid": "S1C9birB-",
"data": exame
}
};
var options = {
hostname: 'localhost',
port: 5488,
path: '/api/report',
method: 'POST',
headers: {
'Content-Type': 'application/json'
}
};
var bodyparts = [];
var bodylength = 0;
var post = http.request(options, (res) => {
res.on('data', (chunk) => {
bodyparts.push(chunk);
bodylength += chunk.length;
});
res.on('end', () => {
var pdf = new Buffer(bodylength);
var pdfPos = 0;
for(var i=0;i<bodyparts.length;i++){
bodyparts[i].copy(pdf, pdfPos, 0, bodyparts[i].length);
pdfPos += bodyparts[i].length;
}
response.setHeader('Content-Type', 'application/pdf');
response.setHeader('Content-disposition', exame._id + '.pdf');
response.setHeader('Content-Length', bodylength);
response.end(Buffer.from(pdf));
});
});
post.write(JSON.stringify(body));
post.end();
});
I am sure that my report is being rendered as expected because if i make a request from postman, it returns the PDF just fine.
Your solution is simply relaying data chunks but you are not telling your front end what to expect of these chunks or how to assemble them. At a minimum you should be setting the the Content-Type response header to application/pdf and to be complete should also be sending the Content-disposition as well as Content-Length. You may need to collect the PDF from your 3rd party source into a buffer and then send that buffer to your client if you are not able to set headers and pipe to response successfully.
[edit] - I'm not familiar with jsreport but it is possible (and likely) that the response they send is a buffer. If that is the case you could use something like this in place of your response to the client:
myGetPDFFunction(params, (err, res) => {
if (err) {
//handle it
} else {
response.writeHead(200, {
'Content-Type': 'application/pdf',
'Content-Length': [your buffer's content length]
});
response.end(Buffer.from([the res PDF buffer]));
}
}
What you haven't shown is the request made to obtain that PDF, so I couldn't be more specific at this time. You should look into the documentation of jsreport to see what it sends in its response, and you can also read up on buffers here
This is rough pseudo code but the point is to respond with the PDF buffer after setting the headers to their proper values.
How can I POST multiple binary files to a server with Content-Type: "form-data", using only the http module?
For example, my keys would look like this:
{
Image1: <binary-data>,
Image2: <binary-data>
}
TL:DR; See the full example code at the bottom of this answer.
I was trying to figure out how to POST multiple binary image files to a server in NodeJS using only core NodeJs modules (without using anything like npm install request, etc). After about 3 hours typing the wrong search terms into DuckDuckGo and finding no examples online, I was about to switch careers. But after banging my head against the desk for almost half the day, my sense of urgency was dulled and I managed to hobble together a working solution. This would not have been possible without the good people who wrote the answer to this Stackoverflow post, and this Github Gist. I used PostMan and Charles Proxy to analyze successful HTTP POSTS, as I dug around in the NodeJS docs.
There are a few things to understand to POST two binary images and a text field as multipart/form-data, relying only on core NodeJS modules:
1) Boundary Identifiers
What is the boundary in multipart/form-data?
The first part of the solution is to create a "boundary identifier", which is a string of dashes -, appended with a random number. You could probably use whatever you wish, foorbar and such.
------------0123456789
Then place that boundary between each blob of data; whether that is binary data or just text data. When you have listed all your data, you add the boundary identifier at the end, and append two dashes:
------------0123456789--
You also need to add the boundary to the headers so that server receiving the post understands which lines of your post data form the boundary between fields.
const headers = {
// Inform the server what the boundary identifier looks like
'Content-Type': `multipart/form-data; boundary=${partBoundary}`,
'Content-Length': binaryPostData.length
}
2) Form Field Meta Descriptors
(This probably is not what they are called)
You will also need a way to write the meta data for each form-field you send, whether that form field contains a binary or a text object. Here are the descriptors for an image file, which contain the mime type:
Content-Disposition: form-data; name="Image1"; filename="image-1.jpg"
Content-Type: image/jpeg
And here is the descriptor for a text field:
Content-Disposition: form-data; name="comment"
The Post Data Output
So the entire post data that is sent to the server will look like this:
----------------------------9110957588266537
Content-Disposition: form-data; name="Image1"; filename="image-1.jpg"
Content-Type: image/jpeg
ÿØÿàJFIFHHÿáLExifMMi
ÿí8Photoshop 3.08BIM8BIM%ÔÙ²é ìøB~ÿÀ... <<<truncated for sanity>>>
----------------------------9110957588266537
Content-Disposition: form-data; name="Image2"; filename="image-2.jpg"
Content-Type: image/jpeg
ÿØÿàJFIFHHÿáLExifMMi
ÿí8Photoshop 3.08BIM8BIM%ÔÙ²é ìøB~ÿÀ... <<<truncated for sanity>>>
----------------------------9110957588266537
Content-Disposition: form-data; name="comment"
This is a comment.
----------------------------9110957588266537--
Once this post data is generated, it can be converted to binary and written to the HTTP POST request: request.write(binaryPostData).
Example Code
Here is the full example code that will allow you to POST binary file and text data without having to include other NodeJS libraries and packages in your code.
// This form data lists 2 binary image fields and text field
const form = [
{
name: 'Image1',
type: 'file',
value: 'image-1.jpg'
},
{
name: 'Image2',
type: 'file',
value: 'image-2.jpg'
},
{
name: 'comment',
type: 'text',
value: 'This is a comment.'
}
]
// Types of binary files I may want to upload
const types = {
'.json': 'application/json',
'.jpg': 'image/jpeg'
}
const config = {
host: 'ec2-192.168.0.1.compute-1.amazonaws.com',
port: '80',
path: '/api/route'
}
// Create an identifier to show where the boundary is between each
// part of the form-data in the POST
const makePartBoundary = () => {
const randomNumbers = (Math.random() + '').split('.')[1]
return '--------------------------' + randomNumbers
}
// Create meta for file part
const encodeFilePart = (boundary, type, name, filename) => {
let returnPart = `--${boundary}\r\n`
returnPart += `Content-Disposition: form-data; name="${name}"; filename="${filename}"\r\n`
returnPart += `Content-Type: ${type}\r\n\r\n`
return returnPart
}
// Create meta for field part
const encodeFieldPart = (boundary, name, value) => {
let returnPart = `--${boundary}\r\n`
returnPart += `Content-Disposition: form-data; name="${name}"\r\n\r\n`
returnPart += value + '\r\n'
return returnPart
}
const makePostData = {
// Generate the post data for a file
file: (item, partBoundary) => {
let filePostData = ''
// Generate the meta
const filepath = path.join(__dirname, item.value)
const extention = path.parse(filepath).ext
const mimetype = types[extention]
filePostData += encodeFilePart(partBoundary, mimetype, item.name, item.value)
// Add the binary file data
const fileData = fs.readFileSync(filepath, 'binary')
filePostData += fileData
filePostData += '\r\n'
return filePostData
},
// Generate post data for the text field part of the form
text: (item, partBoundary) => {
let textPostData = ''
textPostData += encodeFieldPart(partBoundary, item.name, item.value)
return textPostData
}
}
const post = () => new Promise((resolve, reject) => {
let allPostData = ''
// Create a boundary identifier (a random string w/ `--...` prefixed)
const partBoundary = makePartBoundary()
// Loop through form object generating post data according to type
form.forEach(item => {
if (Reflect.has(makePostData, item.type)) {
const nextPostData = makePostData[item.type](item, partBoundary)
allPostData += nextPostData
}
})
// Create the `final-boundary` (the normal boundary + the `--`)
allPostData += `--${partBoundary}--`
// Convert the post data to binary (latin1)
const binaryPostData = Buffer.from(allPostData, 'binary')
// Generate the http request options
const options = {
host: config.host,
port: config.port,
path: config.path,
method: 'POST',
headers: {
// Inform the server what the boundary identifier looks like
'Content-Type': `multipart/form-data; boundary=${partBoundary}`,
'Content-Length': binaryPostData.length
}
}
// Initiate the HTTP request
const req = http.request(options, res => {
res.setEncoding('utf8')
let body = ''
// Accumulate the response data
res.on('data', chunk => {
body += chunk
})
// Resolve when done
res.on('end', () => {
resolve(body)
})
res.on('close', () => {
resolve(body)
})
res.on('error', err => {
reject(err)
})
})
// Send the binary post data to the server
req.write(binaryPostData)
// Close the HTTP request object
req.end()
})
// Post and log response
post().then(data => {
console.log(data)
})
.catch(err => {
console.error(err)
})
Node Version: 0.10.33
I'm able to successfully use CURL to issue my POST request from the CLI and get a 201 response back, as well as use Postman in Chrome to get a successful 201 response back, but when I try to issue the POST request from NodeJS, using either the http module, or the request third party library, I get this error message from Node:
error: Error: connect ECONNREFUSED
at errnoException (net.js:904:11)
at Object.afterConnect [as oncomplete] (net.js:895:19)
And I don't get any logs at all from Rails (as if Rails isn't found by Node for some reason).
NodeJS code that I've tried: (note that createPostData returns a JSON serializable object that becomes the post body)
In all examples, I made sure that railsHost was 'localhost' and railsPort was 3000.
First attempt:
redisClient.on('pmessage', function(pattern, channel, message) {
var postData = createPostData(channel),
options = {
uri: 'http://' + config.railsHost + ':' + config.railsPort + '/resource',
method: 'POST',
json: postData
};
if (postData === null) {
return;
}
winston.info('Creating request using options ' + JSON.stringify(options) + '\n and POST data ' + JSON.stringify(postData));
request(options, function (err, res, body) {
if (err) {
winston.error(err);
return;
}
var status = res ? res.statusCode : null;
if (status === 201) {
winston.info('Notification processed successfully by Rails');
} else {
winston.error('Rails could not create the Notification');
winston.error('HTTP Status: ' + status + ' -> ' + (res ? res.statusMessage : 'response object was null'));
}
});
});
winston.log('subscribing to */*/queue1/pubsub');
redisClient.psubscribe('*/*/queue1/pubsub');
winston.log('log.txt', 'subscribing to */*/queue2/pubsub');
redisClient.psubscribe('*/*/queue2/pubsub');
Second attempt
redisClient.on('pmessage', function(pattern, channel, message) {
var postData = createPostData(channel),
options = {
uri: 'http://' + config.railsHost + ':' + config.railsPort + '/resource',
method: 'POST',
json: true,
body: postData
};
// ... the rest is the same as the First Attempt
Third Attempt
redisClient.on('pmessage', function(pattern, channel, message) {
var postData = createPostData(channel),
options = {
uri: 'http://' + config.railsHost + ':' + config.railsPort + '/resource',
method: 'POST',
// I also tried replacing 'body' below with 'formData' and 'postData'
body: JSON.stringify(postData),
headers: {
'content-type': 'application/json'
// I also messed around with upper-casing the 'c' and 't' in 'content-type'
}
};
// ... the rest is the same as the First Attempt
Fourth Attempt - Now Using the HTTP Module Provided Natively by NodeJS
redisClient.on('pmessage', function(pattern, channel, message) {
var postData = JSON.stringify(createPostData(channel)),
options = {
host: config.railsHost,
port: config.railsPort,
path: '/resource',
method: 'POST',
headers: {
'Content-Type': 'application/json'
}
};
if (postData === null) {
return;
}
winston.info('Creating request using options ' + JSON.stringify(options) + '\n and POST data ' + postData);
var req = http.request(options, function (res) {
var status = res ? res.statusCode : null;
if (status === 201) {
winston.info('Notification processed successfully by Rails');
} else {
winston.error('Rails could not create the Notification');
winston.error('HTTP Status: ' + status + ' -> ' + (res ? res.statusMessage : 'response object was null'));
}
});
req.on('error', function (err) {
winston.error(err);
}
req.write(postData)
req.end();
});
winston.log('subscribing to */*/queue1/pubsub');
redisClient.psubscribe('*/*/queue1/pubsub');
winston.log('log.txt', 'subscribing to */*/queue2/pubsub');
redisClient.psubscribe('*/*/queue2/pubsub');
CURL Command that Works
the test.json file contains the same json that the winston logs say I'm sending through the NodeJS request - copy-pasted from the terminal.
curl -d #test.json --header "Content-Type: application/json" http://localhost:3000/resource
Got back a 201 response.
Postman Request that Works
Same JSON as in the above CURL command - I just copy-pasted the Winston printout from the terminal that my code from the above examples prints.
URL: http://localhost:3000/resource
Request Type: POST
Headers: key = Content-Type value = application/json
For the body, chose 'raw' and 'JSON' and then just pasted in my JSON that I copied from the terminal log statements.
got back a 201 response.
It turns out an application I was using had mapped localhost to ::1 (IPv6), so Rails preferred that on startup and, thus, was not available on 127.0.0.1. It appears that Node was translating localhost to 127.0.0.1, thus the connection refused errors.
Running Rails on 127.0.0.1 using the -b option resolved my issue. I'll rely on Google to help me fix localhost on my machine k will probably involve some trivial changes to the hosts file)