Setting charset in post request on nodejs - node.js

I want to send post form data to some website using euc-kr charset by request module. And I use iconv-lite module too because nodejs supported charset isn't plenty.
Anyway, The website use euc-kr charset, So I have to handle form data's encoding(Node's default charset is utf-8). But it dosen't work well, I tried to change some options so many times but I stucked untill now, So could you tell me some hints.?
// added module request, iconv-lite(extendNodeEncoding) already.
function postDocumentForm() {
//Lets configure and request
request({
url: 'http://finance.naver.com/item/board_act.nhn', //URL to hit
headers: {
'Content-Type': 'content=text/html; charset=euc-kr'
},
method: 'POST',
encoding: 'euc-kr',
form: {
code:'000215',
mode: 'write',
temp: '',
keyCount: '0',
title: "폼 데이터 중 일부가 한글일 때",
opinion: '0',
body:'인코딩이 제대로 되지 않고 있음!'
}
}, function (error, response, body) {
if (error) {
console.log(error);
} else {
iconv.undoExtendNodeEncodings();
console.log(response.statusCode, response.body);
}
});
}
And here is result, odd characters.
I tried :
euc-kr to binary
euc-kr to null
euc-kr to utf-8
delete encoding option
delete request header

Finally I got a soultion, and I solved this problem.
If you send a data as a form using request module, the module change your form encoding to utf-8 by force. So even you setted your form encoding to another charset, the module changes your charset to utf8 again. You can see that at request.js on line 1120-1130.
So, You'd better send a data by 'body' option, not 'form' option.
(as a querystring)
body: "someKey=someValue&anotherKey=anotherValue...."

Notice how it handles receiving back the body encoding
iconv = require('iconv-lite');
function postDocumentForm() {
//Lets configure and request
request({
url: 'http://finance.naver.com/item/board_act.nhn', //URL to hit
headers: {
'Content-Type': 'content=text/html; charset=euc-kr'
},
method: 'POST',
encoding: null,
form: {
code:'000215',
mode: 'write',
temp: '',
keyCount: '0',
title: "폼 데이터 중 일부가 한글일 때",
opinion: '0',
body:'인코딩이 제대로 되지 않고 있음!'
}
}, function (error, response, body) {
if (error) {
console.log(error);
} else {
console.log(response.statusCode);
var utf8String = iconv.decode(new Buffer(body), "ISO-8859-1");
console.log(utf8String);
}
});
}

After reading the source code several hours, finally I found the simple solution.
First, write your encoding function, which input a string and output a encoded string:
const urlencode = require('urlencode');
function encoder(s){
return urlencode(s, 'gb2312');
}
Here is a Chinese encoder based on urlencode
Then just add a qsStringifyOptions option while posting:
request({
url: 'http://finance.naver.com/item/board_act.nhn', //URL to hit
headers: {
'Content-Type': 'content=text/html; charset=euc-kr'
},
method: 'POST',
encoding: null,
form: {
code:'000215',
mode: 'write',
temp: '',
keyCount: '0',
title: "폼 데이터 중 일부가 한글일 때",
opinion: '0',
body:'인코딩이 제대로 되지 않고 있음!'
},
qsStringifyOptions: {
encoder: encoder
}
});

Related

Why am I getting an empty body as response from HTTP request Noe.js

so I am trying to send a http request to a server and as response I should get the body where all the information I need is in. When I then try to .parseJSON() the body I get an exception telling me that my body is empty. There is no way that the body is really empty so there has to be a mistake in the code.
I can not give you the whole code because of passwords and stuff like that.
Code:
public addVehicle(): Promise<void>{
return new Promise<void>((resolve, reject) => {
const options = {
url: [URL],
method: "POST",
headers: {
'Content-Type': 'application/json',
'Authorization': [user/psw],
'token': [authToken]
},
body: JSON.stringify({
'vehicleID': vehicleID,
'externalID': externalID,
'brandID': vehicleBrandId,
'modelGroupID': vehicleModelId,
'typeName': typeName,
'segmentationID': vehicleSegmentationId,
'categoryID': vehicleCategoryId,
'fuelTypeID': vehicleFuelTypeId,
'transmissionTypeID': vehicleTransmissionTypeId,
'propulsionTypeID': vehiclePropulsionTypeId,
'registerationDate': registerationDate,
'powerKW': powerKW,
'description': description
})
}
let req = request.post(options, (err, res, body) => {
let rawAuth = JSON.parse(body);
console.log(body);
resolve();
})
})
}
Try removing the JSON.stringify().
I am assuming that your receiving API expects the fields in the body. Sending the data as string will hide the fields from the receiving API. It will just see one string object (not sure what the key will be tho).
const options = {
url: [URL],
method: "POST",
headers: {
'Content-Type': 'application/json',
'Authorization': [user/psw],
'token': [authToken]
},
body: {
'vehicleID': vehicleID,
'externalID': externalID,
'brandID': vehicleBrandId,
'modelGroupID': vehicleModelId,
'typeName': typeName,
'segmentationID': vehicleSegmentationId,
'categoryID': vehicleCategoryId,
'fuelTypeID': vehicleFuelTypeId,
'transmissionTypeID': vehicleTransmissionTypeId,
'propulsionTypeID': vehiclePropulsionTypeId,
'registerationDate': registerationDate,
'powerKW': powerKW,
'description': description
}
}

How to make a POST with XML body/content-type header using request promise native in Node?

I need to make an XML POST (I know don't ask it's government...) and I can't get it to work with node request promise native.
I have tried turning my string of XML into a buffer, String(), .toString etc. The POST works if I turn json:true so I don't think it's a networking issue ( when passed a xml string with json true it sends a json like { 'variablename': 'stringed XML that I want to send as the body' } )
Here is what I'm working with. I've been banging my head for a while here any help appreciated.
Would ideally be promise/async.
Maybe I should look to find a XMLHttp request npm module?
var request_promise_native_options = {
method: 'POST',
uri: 'http://requestbin.fullcontact.com/198flbs1',
headers: {
'User-Agent': 'Request-Promise',
'Content-Type': 'text/xml'
//'Content-Length': Buffer.byteLength(my_xml_string) //I've tried multiple ways to use this
},
body: {
my_xml_string //also tried many ways here Buffer, String() etc
},
json: false // automatically stringifys body to json if true
};
request_promise(request_promise_native_options)
.then(function (response) {
console.log("success");
})
.catch(function (err) {
console.log(err);
})
Kudos to #kevin-b to help me see the obvious. Simply remove {}
var request_promise_native_options = {
method: 'POST',
uri: 'http://requestbin.fullcontact.com/198flbs1',
headers: {
'User-Agent': 'Request-Promise',
'Content-Type': 'text/xml'
'Content-Length': Buffer.byteLength(my_xml_string)
},
body: my_xml_string,
json: false // automatically stringifys body to json if true
};

Unable to overwrite header attribute Content-type in http calls

Following is the code which sends the request -
var requestData = {
"trigger_id":global.config.primusConfig.trigger_id,
//"mobile":global.config.primusConfig.trigger_id,
"email": global.config.primusConfig.email,
"params":{
"Amount":"200"
}
/*,
"files":{
“sample.pdf” : fileData
}*/
};
request({
headers: {
'cid': global.config.primusConfig.client_id,
'datetime': datetime,
'hash': hash,
'Content-type': 'application/json',
'From':'sandeepan#example.com'
},
url: global.config.primusConfig.apiEndPoint,
method: 'POST',
form: requestData,
body : req.rawBody,
},
function(error,httpResponse,body) {
console.log("Response handling part "+global.config.primusConfig.apiEndPoint+" formdata "+JSON.stringify(requestData));
if(error) {
console.log("Error "+error);
}
else {
console.log("Not error case- Body: "+body+" httpResponse:"+JSON.stringify(httpResponse));
callback();
}
});
From the logged httpResponse, I can see the request which was actually sent -
httpResponse:{"statusCode":500,"body":"\n <h1> 500 - Internal server error </h
1>\n <h2> </h2>\n","headers":{"content-type":"text/html; charset=utf-8","date"
:"Tue, 31 Oct 2017 15:19:13 GMT","etag":"W/\"38-3b8a0f21\"","x-powered-by":"Expr
ess","content-length":"56","connection":"Close"},"request":{"uri":{"protocol":"h
ttps:","slashes":true,"auth":null,"host":"domain.com","port":4
43,"hostname":"domain.com","hash":null,"search":null,"query":n
ull,"pathname":"/sendnotification","path":"/sendnotification","href":"domain.com/sendnotification"},"method":"POST","headers":{"cid":"
19","datetime":"2017-10-31 00:00:00","hash":"88072482e5125ed69f28ce60af859653049
9e11d","Content-type":"application/x-www-form-urlencoded","From":"sandeepan#exam
ple.com","content-length":70}}}
I can see the From attribute set to sandeepan#example.com, as I modified it, but I cannot see the Content-type changed to application/json.
Can someone explain? Any pointers?
form option implies application/x-www-form-urlencoded content-type: https://github.com/request/request/blob/master/request.js#L1249
If you are sending json, you need to use json option instead: https://github.com/request/request/blob/master/request.js#L1278
There is no need to set content-type explicitly:
request({
headers: {
'cid': global.config.primusConfig.client_id,
'datetime': datetime,
'hash': hash,
'From':'sandeepan#example.com'
},
url: global.config.primusConfig.apiEndPoint,
method: 'POST',
json: requestData
},

Gzip decompression of http Response

Using request module, I am trying to fetch response from a web service which has following header in the API request:
accept-encoding : gzip
and correspondingly, following header in the response :
content-encoding : gzip
When I am trying to decompress the response(get the correct readable response) using zlib(referred here), I am unable to do so.
Code Snippet :
var options = {
url: url,
qs: params.qparams,
method: params.method,
json: params.body,
headers: {
'api_key': configkey,
'Content-Type': 'application/json',
'Accept-Encoding': 'gzip'
},
timeout: constants.request_timeout
};
request(options, function(err, response, body) {
var encoding = response.headers['content-encoding']
if (encoding && encoding.indexOf('gzip') >= 0) {
zlib.gunzip(body, function(err, dezipped) {
//ERROR : { [Error: incorrect header check] errno: -3, code: 'Z_DATA_ERROR' }
var json_string = dezipped.toString('utf-8');
var json = JSON.parse(json_string);
console.log('\nJSON ::\n',json);
});
} else {
console.log('\n\nRESPONSE IS NOT GZIPPED!');
}
}
I am getting an error here(as commented in the code), using zlib.
I could not figure out as where is it going wrong, tried with multiple npm modules like unzipResponse and compress-buffer and tried different approaches as well as suggested at various places for handling gzip.
If someone can help out in resolving this, I'll be really thankful.
I have got a solution as need to add one more key to the options object as :
var options = {
url: url,
qs: params.qparams,
method: params.method,
json: params.body,
headers: {
'api_key': configkey,
'Content-Type': 'application/json',
'Accept-Encoding': 'gzip'
},
timeout: constants.request_timeout,
encoding: null
};
If someone has a better approach to perform the decompression, please add-on.

How to set POST encoding with request module on node.js?

I'm using request module on node.js but there is something problem with encoding option. beneath codes are simple post request, but I don't know how to set up encoding of form field data. I already set headers to 'Content-Type': 'application/x-www-form-urlencoded; charset=euc-kr' But it doesn't works. field data is korean, like "안녕하세요", and I should post it with euc-kr encoding. (The site takes euc-kr, not utf8)
The same program on Java application, I coded like this :
PrintWriter wr = new PrintWriter(new OutputStreamWriter(conn.getOutputStream(), "euc-kr"));
But I don't know how to in nodejs. Can anyone give some solution...?
Code Sample
//Load the request module
var request = require('request');
//Lets configure and request
request({
url: 'http://google.com', //URL to hit
headers: { 'Content-Type': 'application/x-www-form-urlencoded; charset=euc-kr' },
method: 'POST',
form: {
field1: 'data',
field2: 'data'
}
}, function(error, response, body){
if(error) {
console.log(error);
} else {
console.log(response.statusCode, body);
}
});
Finally I got a soultion, and I solved this problem.
If you send a data as a form using request module, the module change your form encoding to utf-8 by force. So even you setted your form encoding to another charset, the module changes your charset to utf8 again. You can see that at request.js on line 1120-1130.
So, You'd better send a data by 'body' option, not 'form' option.
Node doesn't support EUC-KR so you can use iconv-lite to extend the native encodings available and set the encoding option in request.
List of Natively Supported Encodings
iconv.extendNodeEncodings(); only works for node pre v4+. See here to get this working for a newer version of node.
var iconv = require('iconv-lite');
var request = require('request');
// This will add to the native encodings available.
iconv.extendNodeEncodings();
request({
url: 'http://google.com', //URL to hit
method: 'POST',
form: {
field1: 'data',
field2: 'data'
},
encoding: 'EUC-KR'
}, function(error, response, body){
if(error) {
console.log(error);
} else {
console.log(response.statusCode, body);
}
});
iconv.undoExtendNodeEncodings();

Resources