I'm trying to send a POST request to a server which decodes with SHIFT-JIS.
This string サービス is being translated to 繧オ繝シ繝薙せ after being decoded in SHIFT-JIS.
It seems like the request will always be encoded in UTF-8 whenever the request is being sent over.
I'm using nodejs for posting the request.
Question is how do I send over the characters in shift-jis encoding? It seemed easy but I just couldn't find out how to.
Listening server
var iconv = require('iconv-lite');
const http = require('http');
http.createServer((request, response) =>
{
const
{
headers,
method,
url
} = request;
let body = [];
request.on('error', (err) =>
{
console.error(err);
}
).on('data', (chunk) =>
{
body.push(chunk);
}
).on('end', () =>
{
body = Buffer.concat(body).toString();
// BEGINNING OF NEW STUFF
body = iconv.decode(Buffer.from(body), 'shift_jis');
response.on('error', (err) =>
{
console.error(err);
}
);
response.statusCode = 200;
response.setHeader('Content-Type', 'application/json');
// Note: the 2 lines above could be replaced with this next one:
// response.writeHead(200, {'Content-Type': 'application/json'})
const responseBody =
{
headers,
method,
url,
body
};
response.write(JSON.stringify(responseBody));
console.log(body);
console.log(responseBody);
response.end();
// Note: the 2 lines above could be replaced with this next one:
// response.end(JSON.stringify(responseBody))
// END OF NEW STUFF
}
);
}
).listen(8000);
Request
var request = require('request');
request({
url: 'http://localhost:8000',
headers: { 'Content-Type': 'application/x-www-form-urlencoded; charset=shift_jis' },
method: 'POST',
body: 'サービス'
}, function(error, response, body){
if(error) {
console.log(error);
} else {
console.log(response.statusCode, body);
}
});
EDIT: It turns out that the axios module we're using for HTTPS POST will encode the payload in UTF-8 before sending out the request. We cloned the axios module and modifying it to encode in SHIFT-JIS instead.
If you want to send a string with Shift-JIS encoding, you have to convert the target string (it is represented in UTF-16 internally) into Shift-JIS before adding it into the request body.
The standard TextEncoder only supports UTF-8 encoding and cannot handle Shift-JIS encoding. So you have to use additional modules like encoding.js or text-encoding for this purpose.
Related
I am trying to upload an image file from my node.js application to a group's drive in Sharepoint.
As the official documentation states, I'm making my request as follows:
PUT /groups/{group-id}/drive/items/{parent-id}:/{filename}:/content
With the binary image in the body: "The contents of the request body should be the binary stream of the file to be uploaded."
The problem is that the image is uploaded but as a corrupted file. I tried different solutions and still don't see why is always the image corrupted.
Here is my code:
//i get my image from a URL first
https.get(url.parse(attachment.contentUrl), function (response) {
var data = [];
response.on('data', function (chunk) {
data.push(chunk);
});
response.on('end', function () {
if (response.statusCode === 200) {
var buffer = Buffer.concat(data);
//store my image in a local file to test if image is correct (which it is)
fs.writeFile(localFileName, buffer, (fsError) => {
//error handling
});
functions.uploadImageToSharepoint(session, localFileName, buffer,
function (err, body, res) {
if (err) {
console.error(err);
}else{
console.log('OK!');
}
});
} else {
//error handling
}
});
}).on('error', function (e) {
console.log("error2: " + e);
});
//and the request to graph api
function uploadImageToSharepoint(session, fileName, data, callback) {
var options = {
url: 'https://graph.microsoft.com/v1.0/groups/xxxxxxx/drive/root:/yyyyyy/fileName.jpg:/content',
method: 'PUT',
body: data,
json: true,
headers: {
'Content-Type': 'image/jpg',
Authorization: 'Bearer ' + session.userData.accessToken
}
};
request(options, function (err, res, body) {
if (err) return callback(err, body, res);
if (parseInt(res.statusCode / 100, 10) !== 2) {
if (body.error) {
return callback(new Error(res.statusCode + ': ' + (body.error.message || body.error)), body, res);
}
return callback(err, body, res);
}
callback(err, body ,res);
});
}
The file is most likely getting corrupted due to the following option for request:
var options = {
json: true, //<--setting this option sets body to JSON representation of value
//another properties are omitted for clarity
};
In that case request sets body to JSON representation of value and adds accept header to application/json for Upload endpoint and binary file get corrupted.
The solution would be to omit json option from a request and use the proper content-type only:
var options = {
url: '/me/drive/root:/filename.jpg:/content',
method: 'PUT',
body: data,
headers: {
'Content-Type': 'image/jpg',
Authorization: 'Bearer ' + accessToken
}
};
At the moment I am using npm module request to upload file with application/octet-stream content type. The problem is that I cannot get response body back. It is happening due to a known bug: https://github.com/request/request/issues/3108
Can you provide me an alternate ways to upload file to an API with an application/octet-stream content type?
Have you tried loading the file to a buffer rather than a stream? I appreciate in many contexts a stream is preferable, but often just loading into memory is acceptable. I've used this approach with no issues:
const imageBuffer = fs.readFileSync(fileName); // Set filename here..
const options = {
uri: url, /* Set url here. */
body: imageBuffer,
headers: {
'Content-Type': 'application/octet-stream'
}
};
request.post(options, (error, response, body) => {
if (error) {
console.log('Error: ', error);
return;
}
To do the same thing using a stream:
const options = {
uri: url, /* Set url here. */
body: fs.createReadStream(fileName),
headers: {
'Content-Type': 'application/octet-stream'
}
};
request.post(options, (error, response, body) => {
if (error) {
console.log('Error: ', error);
return;
}
..
I have the following code that takes base64 string, and send it to API that accepts binary. I am getting empty response from the api call
let base64String = event.base64String;
// pass the base64 string into buffer
let buffer = new Buffer(base64String, 'base64');
// TODO check file type
processImage(buffer)
.then(result => {
console.log("result are " + result);
callback(result);
}).catch(error => callback(error));
let processImage = function (buffer) {
// get the file extension
return new Promise((resolve, reject) => {
var options = {
method: 'POST',
url: 'https://<UR - not visible for privacy>',
headers:
{
'Content-Type': 'application/octet-stream'
},
body: buffer.toString('binary')
};
request(options, function (error, response, body) {
if (error) reject(error);
console.log(body);
resolve(body);
});
}
Equivalent in postman is that I simply specify binary in body and attach file, again post call but URL is removed from pic, the header is content-type: application/octet-stream, it works in postman but not in node.js
I use Nodejs version 10.10.0 on Debian 9 with this simple code but it doesn't show any data in console nor response.
code:
const http = require('http');
const { parse } = require('querystring');
http.createServer((request, response) => {
const { headers, method, url } = request;
let body = [];
request.on('error', (err) => {
console.error(err);
}).on('data', (chunk) => {
body.push(chunk);
}).on('end', () => {
body = Buffer.concat(body).toString();
console.log(parse(body));
response.statusCode = 200;
response.setHeader('Content-Type', 'text/html');
response.setHeader('Access-Control-Allow-Origin', '*');
response.setHeader('Access-Control-Allow-Credentials', true);
response.write('body: ' + body);
response.end();
});
}).listen(3030); // Activates this server, listening on port 8080.
result:
{}
Thank you for the comment. I am new to Nodejs and was thinking data should include either GET or POST requests' data; looks like it takes care of only POST requests and GET should be parsed from headers.
var fsHeaders = fs.createWriteStream('headers.html', 'utf-8');
var getHeaders = request('http://google.com', {method: 'HEAD'}, function (error, response, body)
{
if (error || response.statusCode !== 200)
{
return res.redirect('/');
}
if (!error && response.statusCode == 200)
{
var tester = response.body;
console.log(tester + 'response.body is EMPTY... so the headers are stored not in the "body"... \n');
var targetHeaders = response.headers;
body = body + JSON.stringify(targetHeaders);
console.log(body + 'OK! \n');
}
}).pipe(fsHeaders);
but my headers.html is empty... how to properly save response.headers to file? Can I later modify headers and sent them to the user without problem or it's restricted in some way? I want to get data from server, modify it via node, and send it to the user.
It doesn't matter if you are using request or http module. When you request headers only with {method: 'HEAD'} method body is empty but headers are resident in response stream. I guess you problem is with requiring headers and display them as [objest Object]. You need to change object to string in that matter - to simply display it with console.log or pipe it to the file.
var targetHeaders = objectToString(resp.headers, targetHeaders);
function objectToString(object, outputString)
{
var outputString = '';
for (var i in object)
{
if (object.hasOwnProperty(i))
{
outputString += i + ': ' + object[i] + ' \n';
}
}
return outputString;
}
now console.log(targetheaders); // OK! no [object Object] anymore :)
You can latter pass those headers without any problem.
First, to verify that you are using http.request
Your problem also can be because request by default returns buffer and your stream is UTF-8 format.
Also the better way "catch" your headers might be by events that are emitted by http request:
var postData = querystring.stringify({
'msg': 'Hello World!'
});
var options = {
hostname: 'www.google.com',
port: 80,
path: '/upload',
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': Buffer.byteLength(postData)
}
};
var req = http.request(options, (res) => {
console.log(`STATUS: ${res.statusCode}`);
console.log(`HEADERS: ${JSON.stringify(res.headers)}`);
res.setEncoding('utf8');
res.on('data', (chunk) => {
console.log(`BODY: ${chunk}`);
});
res.on('end', () => {
console.log('No more data in response.');
});
});
req.on('error', (e) => {
console.log(`problem with request: ${e.message}`);
});
// write data to request body
req.write(postData);
req.end();