I'm making a HTTP request to a password protected site using the request module in npm, putting in a password, storing a cookie, then making the request once the cookie is stored and verified. I am able to get the same headers as I would on a normal browser request, but hte body itself, instead of being the HTML document I get in a browser, just looks like this:
� �Z�r�H��c��䞙��pT���Ī$3�ƾ�~Y�#�MK8���>*��?�z)�?U���ݨ�J�혳��섯tB��x��c��?�����������0�H�����V��O'�7����}���L�"˖}/ta�xn�g#�ݱ�O�����
Any ideas what might be causing this or how I can fix it?
In addition, when I run this from the command prompt, the computer "dings"
Here is the full node.js code I am running (minus the URLs/passwords/etc.)
var parsedurl1 = url.parse( urlstring1 );
var options1 = {
hostname: parsedurl1.hostname,
port: ( parsedurl1.port || 80 ), // 80 by default
method: 'POST',
path: parsedurl1.path,
headers: {
'Host': hostname
,'User-Agent': myuseragent
,'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
,'Accept-Language':"en-US,en;q=0.5"
,'Accept-Encoding': "gzip, deflate"
,'Referer': hostname
,'Content-Type': "text/html; charset=utf-8"
,'Content-Length': Buffer.byteLength(postData)
,'Connection': "keep-alive"
,'Upgrade-Insecure-Requests': "1"},
};
var cookiefile ;
var postData=querystring.stringify({
'email': myemail
,'password':mypassword
,'action': 'login'
,'go.x': 0
,'go.y': 0
})
var cookiefile;
var callback = function ( response ) {
// display returned cookies in header
var setcookie = response.headers["set-cookie"];
if ( setcookie ) {
setcookie.forEach(
function ( cookiestr ) {
cookiefile = cookiestr;
fs.writeFile(cookiefilelocation, cookiestr);
console.log( "COOKIE:" + cookiestr );
}
);
}
var data = "";
response.on(
"data",
function ( chunk ) { data += chunk; }
);
response.on(
"end",
function () {
newcookiefile = cookiefile.substr(0, cookiefile.indexOf(";"));
var parsedurl2 = url.parse( urlstring2 );
var options2 = {
url: urlstring2,
// port: ( parsedurl2.port || 80 ), // 80 by default
method: 'GET',
// path: parsedurl2.path,
headers: {
"Host": hostname
,"User-Agent": myuseragent
,'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
,'Accept-Language':"en-US,en;q=0.5"
,'Accept-Encoding': "gzip, deflate"
,'Referer':hostname
,'Cookie': newcookiefile
,'Connection': "keep-alive"
,'Upgrade-Insecure-Requests': "1"},
};
function callback3(error, response, body){
console.log('error:', error); // Print the error if one occurred
console.log('statusCode:', response.headers ); // Print the response status code if a response was received
console.log('body:', body.substr(1,1000));
fs.writeFile('./config/pullfiles/mostrecent.txt', body);
}
requestlib(options2, callback3);
}
);
};
var request = http.request(options1, callback);
request.on(
"error",
function( err ) {
console.error( "RERROR:" + err );
}
);
request.write(postData);
request.end(); // let request know it is finished sending
I figured it out! This gibberish was caused by my lack of the 'gzip: true' option is my request(). Now the second request reads:
url: urlstring2
,gzip: true
,headers:{...
Related
I'm trying to send an sms without the twilio sdk by using the nodejs https module however the twilio post api keeps responding with this error "400, Bad Request", which means I'm probably not crafting the request the right way. I've followed the nodejs docs https example, and also twilio's. I've also tried making curl post request and it works perfectly fine. Where I'm I getting it wrong. Here's my code
// Send an SMS message via Twilio
helpers.sendTwilioSms = (phone, message, callback) => {
// validate parameters
phone =
typeof phone == "string" && phone.trim().length == 10
? phone.trim().length
: false;
message =
typeof message == "string" &&
message.trim().length > 0 &&
message.trim().length <= 1600
? message.trim()
: false;
if (phone && message) {
// Configure the request payload
const payload = {
from: config.twilio.fromPhone,
to: `+234${phone}`,
body: message
};
// stringify payload using querystring module instead of JSON.stringify because the reqeust we'll be sending is not of application/json but 'application/x-www-form-urlencoded' form content-type as specified by Twilio
const stringPayload = querystring.stringify(payload);
// Configure the request details
var requestDetails = {
hostname: "api.twilio.com",
method: "POST",
path: `/2010-04-01/Accounts/${config.twilio.accountSid}/Messages.json`,
auth: `${config.twilio.accountSid}:${config.twilio.authToken}`,
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Content-Length": Buffer.byteLength(stringPayload)
}
};
// Instantiate the request
const req = https.request(requestDetails, res => {
// grab the status of the sent request
const status = res.statusCode;
console.log([
`(sendTwilioSms) making https post request`,
`(sendTwilioSms) response completed: ${res.complete}`,
`(sendTwilioSms) response statusCode: ${res.statusCode}`,
{ "(sendTwilioSms) response headers:": res.headers },
{ "(sendTwilioSms) response body:": res.body }
]);
// callback successfully if the request went through
if (status == 200 || status == 201) {
callback(false);
} else {
callback(500, {
Error: `Status code returned was ${status}: ${res.statusMessage}`
});
}
});
// Alert the user as to a change in their check status
workers.alertUserToStatusChange = newCheckData => {
const message = `Alert: Your check for ${newCheckData.method.toUpperCase()} ${
newCheckData.protocol
}://${newCheckData.url} is currently ${newCheckData.state}`;
helpers.sendTwilioSms(newCheckData.userPhone, message, err => {
if (!err) {
console.log(
"Success: User was aterted to a status change in their check, via sms: ",
msg
);
} else {
console.log(
"Error: Could not send sms alert to user who add a state change in their check"
);
}
});
Here's the Response:
[
'(workers) making check request',
'(workers) check response completed: false',
'(workers) check response statusCode: 200'
]
logging to file succeeded
Check outcome has not changed no alert needed
[
'(sendTwilioSms) making https post request',
'(sendTwilioSms) response completed: false',
'(sendTwilioSms) response statusCode: 400',
{
'(sendTwilioSms) response headers:': {
date: 'Fri, 17 Jan 2020 09:49:39 GMT',
'content-type': 'application/json',
'content-length': '127',
connection: 'close',
'twilio-request-id': 'RQ7ee0b52d100c4ac997222f235e760fb7',
'twilio-request-duration': '0.025',
'access-control-allow-origin': '*',
'access-control-allow-headers': 'Accept, Authorization, Content-Type, If-Match, '
+
'If-Modified-Since, If-None-Match, ' +
'If-Unmodified-Since',
'access-control-allow-methods': 'GET, POST, DELETE, OPTIONS',
'access-control-expose-headers': 'ETag',
'access-control-allow-credentials': 'true',
'x-powered-by': 'AT-5000',
'x-shenanigans': 'none',
'x-home-region': 'us1',
'x-api-domain': 'api.twilio.com',
'strict-transport-security': 'max-age=31536000'
}
},
{ '(sendTwilioSms) response body:': undefined }
]
Error: Could not send sms alert to user who add a state change in their check
Try with something like this:
// authentication
var authenticationHeader = "Basic "
+ new Buffer(config.twilio.accountSid
+ ":"
+ config.twilio.authToken).toString("base64");
// Configure the request details
var requestDetails = {
host: "api.twilio.com",
port: 443,
method: "POST",
path: `/2010-04-01/Accounts/${config.twilio.accountSid}/Messages.json`,
headers: {
"Content-Type": "application/x-www-form-urlencoded",
"Content-Length": Buffer.byteLength(stringPayload),
"Authorization": authenticationHeader
}
};
and this:
// Instantiate the request
const req = https.request(requestDetails, res => {
// grab the status of the sent request
const status = res.statusCode;
res.setEncoding('utf8');
res.on('data', (chunk) => body += chunk);
res.on('end', () => {
console.log('Successfully processed HTTPS response');
// If we know it's JSON, parse it
if (res.headers['content-type'] === 'application/json') {
body = JSON.parse(body);
}
callback(null, body);
});
// callback successfully if the request went through
if (status == 200 || status == 201) {
callback(false);
} else {
callback(500, {
Error: `Status code returned was ${status}: ${res.statusMessage}`
});
}
});
I hope it works, I have not tested. If it doesn't let me know and I'll try on my side and post a complete tested code.
I am trying to do the following on a high level, using node (express, https modules):
listen for POST requests (say R1) on /example endpoint (express app)
read the post payload from the request, process it repackage it and make an https POST request (say R11) to an external REST API.
read the post payload from the response to R11, process it repackage it and send as a response to R1.
Problem is, how to send response from within callbackExternalApi ? Please see code below and question in comments.
const callbackExternalApi =
function (response) {
response.on('data',
function(data) {
// do some processing on data
var processedData = ...
**// I want response_R1 over here
// so that I can do the following
response_R1.send(processedData)
// how do I get response_R1 over here??**
})
}
const requestHandlerExample =
function (request_R1, response_R1) {
// payload
var postBodyJson = '' // some payload here
// headers
var postHeaders = {
'content-type' : 'application/json',
'accept' : 'application/json'
}
// options
var postOptions = {
'host' : 'localhost',
'port' : '9000',
'path' : '/external/rest/api',
'method' : 'POST',
'headers' : postHeaders
}
// do the post call
var postRequest = _httpsModule.request(postOptions, callbackExternalApi)
postRequest.write(postBodyJson);
postRequest.end();
postRequest.on('error', function(error) {
console.error('an error occured'+error)
})
}
_app.post('/example', requestHandlerExample)
Thanks,
Jatin
I just defined global variables for response_R1 and set those in the requestHandlerExample callback. Not sure if that's the best approach, but it works.
var _response_R1
const callbackExternalApi =
function (response) {
response.on('data',
function(data) {
// do some processing on data
var processedData = ...
_response_R1.send(processedData)
})
}
const requestHandlerExample =
function (request, response) {
_response_R1 = response // set this for use in callbackExternalApi
// payload
var postBodyJson = '' // some payload here
// headers
var postHeaders = {
'content-type' : 'application/json',
'accept' : 'application/json'
}
// options
var postOptions = {
'host' : 'localhost',
'port' : '9000',
'path' : '/external/rest/api',
'method' : 'POST',
'headers' : postHeaders
}
// do the post call
var postRequest = _httpsModule.request(postOptions, callbackExternalApi)
postRequest.write(postBodyJson);
postRequest.end();
postRequest.on('error', function(error) {
console.error('an error occured'+error)
})
}
_app.post('/example', requestHandlerExample)
I have to send a post request with a body containing russian characters to a service which seems to only accept win1251 encoding. I am doing this in node.js using the request module.
This is how I am sending the request
var rawbody = "i1=&i2=&i3=" + query + "&i4=&i5=&i8=";
var options = {
url: this.initiate_endpoint,
jar: this.session,
cert: this.fs.readFileSync('resources/r.crt.pem', 'utf-8'),
key: this.fs.readFileSync('resources/r.key.pem', 'utf-8'),
timeout: 20000,
headers: {
'Content-Type': 'application/x-www-form-urlencoded'
},
body: rawbody
};
let cookie = this.session;
request.post(options,
function (error, response, body) {
if( error ) {
callback(cookie, error);
}
else if( response.statusCode !== 200 ) {
callback(cookie, 'Responded with status code: ' + response.statusCode);
}
else {
callback(cookie, false);
}
});
Whenever the query variable is containing russian letters the request will fail.
For example:
var query = 'Путин';
So my questions are how to actually convert the encoding from the query variable to win1251 and furthermore how to set the encoding of the actual raw postbody, because I think those two things could be the cause of the issue.
I'm attempting to write a simple service will
Receive a http request
Modify some values in the body
Then, post the data(headers + newly modified body) to another endpoint.
return the exact response from the new request to the client.
I have found a sample that does the http relay nicely, but I'm struggling with modifying the post before it's sent on? I'm able to get the contents of the post, but can seem to get my head around how to modify it before sending it on it's way.
var http = require( 'http' );
var qs = require( 'querystring' );
http.createServer( function ( req, resp ) {
var h = req.headers;
h.host = "webdbg.com";
req.url = "/sandbox/FileForm.asp";
var newRequest = http.request( {
host: h.host, port: 80, path: req.url, method: req.method, headers: h
}, function ( newResp ) {
resp.writeHead( newResp.statusCode, newResp.headers );
//as we receive our response from the new request, start writing it to this response.
newResp.on( 'data', function ( respBody ) { resp.write( respBody ); });
//once we have all the data from the new request stor writing it to this response.
newResp.on( 'end', function () { resp.end(); });
});
var postData = "";
//as we receive our body write it to the new request.
req.on( 'data', function ( reqBody ) { postData += reqBody; newRequest.write( reqBody )});//here I need to replace the values of the form post
//once we have all of our data from this request, stop writing it to the new request.
req.on( 'end', function () { console.log(qs.stringify(qs.parse(postData))); newRequest.end(); });
}).listen(1337);
console.log( "Server running...");
I'm coming over from the dark side(C#) and as most, I'm struggling with the asynchronous nature of Node.js. With that, I'm committed to plugging away at the tutorials on Node School and Plural Sight until I get it! Any help you can provide with this would greatly be appreciated.
You want to modify the response and relay it to the client?
Change the newResponse handler to modify the response from the remote server.
newResp.on( 'data', function ( respBody ) { resp.write( respBody ); });
to
newResp.on( 'data', function ( respBody ) { resp.write( MODIFY(respBody) ); });
I'm unable to get the var data I sent in via a POST method. This should be easy (right?), but I'm clearly missing something (either conceptually or a setting).
At this stage, I simply want to check to see if the server side code will output the data to the console. The array is being stringify-ed correctly, eg. ['one','two','three'] becomes 0=one&1=two&2=three
but I can't pull it out on the server side.
What am I missing?
Client side
var qs = require('querystring')
, http = require('http');
var some_array = ['one','two','three'];
var data = qs.stringify(some_array);
var options = { host: 'localhost',
path: '/search',
port: '3000',
method: 'POST',
headers: { 'content-length': Buffer.byteLength(data),
'Content-Type': 'application/json' }
}
function go_post(data) {
req = http.request(options, function(res) {
// do something with response
});
req.write(data);
req.end();
};
go_post(data);
Server side
var connect = require('connect');
var qs = require('querystring');
var server = connect.createServer();
server.use(function(req,res,next) {
if ( '/search' == req.url && req.method == 'POST' ) {
// quick check to see if data came through
console.log('BODY IS ' + req.data);
} else {
next();
};
});
These objects arent available because they are still in the "raw" request. You have to use a middleware like connect().use(connect.bodyParser()) in order to get them from the request via req.data.