Node Http Post argument to Spark Job Server - node.js

The following curl command works perfectly to call, pass argument and execute my "jobified" spark program
curl 'http://someserver:8090/jobs?appName=secondtest&classPath=Works.epJob&context=hiveContext' -d "inputparms=/somepath1 /somepath2"
Here is the spark program
override def runJob(hive: HiveContext, config: Config):Any = {
var inputParms = config.getString("inputparms").split(" "); //comes from node
var path1 = inputParms.apply(0)
var path2 = inputParms.apply(1)
Instead of the curl command, I need to do a http post in node.js. Here is what I have
var postData = JSON.stringify({
"inputparms": paths
})
var options = {
hostname: 'someserver',
port: 8090,
path: '/jobs?appName=secondtest&classPath=Works.epJob context=hiveContext',
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Content-Length': Buffer.byteLength(postData , 'utf8')
}
};
http.request(options, function(response) {...
Howerever the above script does not work. Am I missing something?
Thanks!
Edit 1:
var myreq = http.request(options, function(response) { ...})
myreq.write(postData);
myreq.end();
I get a parse error
Error: Parse Error
at Error (native)
at Socket.socketOnData (_http_client.js:361:20)
at emitOne (events.js:96:13)
at Socket.emit (events.js:188:7)
at readableAddChunk (_stream_readable.js:177:18)
at Socket.Readable.push (_stream_readable.js:135:10)
at TCP.onread (net.js:542:20) bytesParsed: 2, code: 'HPE_INVALID_CONSTANT' }

The following works for me
var http = require("http");
var options = {
hostname: 'localhost',
port: 8090,
path: '/jobs?appName=test&classPath=spark.jobserver.LongPiJob',
method: 'POST',
headers: {
'Content-Type': 'application/json',
}
};
var req = http.request(options, function(res) {
console.log('Status: ' + res.statusCode);
console.log('Headers: ' + JSON.stringify(res.headers));
res.setEncoding('utf8');
res.on('data', function (body) {
console.log('Body: ' + body);
});
});
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
});
// write data to request body
req.write('stress.test.longpijob.duration=120');
req.end();

Related

Error: socket hang up - Lambda nodejs function http post request

I have a very simple function in Lambda using Nodejs.
The purpose of that function is to triggered a third party api every 1 minute.
So for that I have setup Cloud Watch Cron based event.
But that Api is throwing this error:
START RequestId: 54d90c9d-0a5b-4e5e-a26a-857d9bb6dd4e Version: $LATEST
2022-08-29T11:11:59.112Z 54d90c9d-0a5b-4e5e-a26a-857d9bb6dd4e ERROR Error: socket hang up
at connResetException (node:internal/errors:692:14)
at TLSSocket.socketOnEnd (node:_http_client:478:23)
at TLSSocket.emit (node:events:539:35)
at endReadableNT (node:internal/streams/readable:1345:12)
at processTicksAndRejections (node:internal/process/task_queues:83:21) {
code: 'ECONNRESET'
It works fine in a simple nodejs code on my local machine.
Here is my Lambda Function code:
const https = require('https');
exports.handler = async (event) => {
var postData = JSON.stringify({
"client_id": "abnbfye9-qtfnf1cj-abhrhzfyf7-m2tup-6x9kk2kc5688",
"client_secret": "fpghfh329-polk80s-ye043465p1yy-45hxnfd874z06",
"inTime": new Date(),
"outTime": new Date()
});
var options = {
hostname: 'example.com',
path: '/api/updateDataLambda',
method: 'POST',
port: 443, // 👈️ replace with 80 for HTTP requests
headers: {
'Content-Type': 'application/json',
'Authorization': 'Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJfaWQiOiJ0ekpFTlNxcDJXeXh4YlNGbSIsImFjY2Vzc1Rva2VuRXhwIjp7ImRhdGUiOiIyMDIyLTA5LTI4IiwidGltZSI6IjA2OjIwOjQ1IiwidGltZVN0YW1wIjoxNjY0MzQ2MDQ1NjQ2LCJnbXQiOiIrMDAwMCJ9LCJyZWZyZXNoVG9rZW5FeHAiOnsiZGF0ZSI6IjIwMjItMDktMjgiLCJ0aW1lIjoiMDY6MjA6NDUiLCJ0aW1lU3RhbXAiOjE2NjQzNDYwNDU2NDYsImdtdCI6IiswMDAwIn0sImlhdCI6MTY2MTc1NDA0NSwiZXhwIjoxNjYxNzU0MTA1fQ.g1e5S15Q1qxB5_s4j3LFfFf6spU8gwgBUyVNLVuWNWk'
}
};
var req = https.request(options, (res) => {
console.log('statusCode:', res.statusCode);
console.log('headers:', res.headers);
res.on('data', (d) => {
process.stdout.write(d);
});
});
req.on('error', (e) => {
console.error(e);
});
req.write(postData);
req.end();
const response = {
statusCode: 200,
body: JSON.stringify('Hello from Lambda!'),
};
return response;
};
Am I doing something wrong here?

Error: getaddrinfo ENOTFOUND error when making an HTTPS/HTTP request

Here's the code of my AWS Lambda function:
console.log('Loading function');
const AWS = require('aws-sdk');
const https = require('https');
const data = JSON.stringify({
secretKey:"someSecretKey",
userType:"someCoolUser",
})
const options = {
hostname: "backend-staging.programmingpathshala.com:8080/rest/admin",
path: '/sendEmailToUsers',
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Content-Length': data.length
}
}
exports.handler = function(event, context, callback) {
var dataString = '';
const req = https.request(options, function(res) {
res.on('data', chunk => {
dataString += chunk;
});
res.on('end', () => {
callback(null,dataString);
});
});
req.write(data)
req.end();
req.on('error', (e) => {
console.error(e);
});
}
When I test my API using postman it works fine. But when it is called from lambda function I get the following error:
Also, When I run the same API using ngrok and use its link in my lambda function it works then too.
Based on the comments, the options should be:
const options = {
hostname: "backend-staging.programmingpathshala.com",
port: 8080,
path: '/rest/admin/sendEmailToUsers',
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Content-Length': data.length
}
}

nodejs HTTP Digest Authentication not working

I've been digging deep into stack overflow but haven't been able to solve my problem. I'm trying to access an API that uses digest but have had no success, and my co-workers haven't been able to pin down the problem either. I've hit a wall and have come to Stack Overflow to ask my question.
Here is my authentication code:
var https = require("https"),
crypto = require('crypto'),
_ = require('underscore');
var options = {
host: 'api.example.com',
port: 80,
path: '/path/to/uri/',
method: 'GET',
accept: 'application/json',
acceptEncoding: 'gzip, deflate',
connection: 'keep-alive',
rejectUnauthorized: false,
requestCert: true,
agent: false
};
var username = 'username',
password = 'httppassword';
var req = https.get(options, function(res) {
res.setEncoding('utf-8');
console.log(res.url);
console.log('STATUS: ' + res.statusCode);
console.log('HEADERS: ' + JSON.stringify(res.headers));
var data = "";
res.on('data', function (chunk) {
data = data + chunk;
});
res.on('end', function(){
console.log(data);
var challengeParams = parseDigest(res.headers['www-authenticate']);
console.log(challengeParams);
var ha1 = crypto.createHash('md5').update(username + ':' + challengeParams.realm + ':' + password).digest('hex');
var ha2 = crypto.createHash('md5').update('GET:' + options.path).digest('hex');
var response = crypto.createHash('md5').update(ha1 + ':' + challengeParams.nonce + ':1::auth:' + ha2).digest('hex');
var authRequestParams = {
username : username,
realm : challengeParams.realm,
nonce : challengeParams.nonce,
uri : options.path,
qop : challengeParams.qop,
response : response,
nc : 1,
cnonce : ''
};
options.headers = { 'Authorization' : renderDigest(authRequestParams) };
console.log(options);
https.get(options, function(res) {
console.log("STATUS: " + res.statusCode);
console.log("HEADERS: " + JSON.stringify(res.headers));
res.setEncoding('utf-8');
var content = '';
res.on('data', function(chunk) {
content += chunk;
}).on('end', function() {
console.log(content);
});
})
});
});
req.on('error' ,function(err){
console.log("request");
console.log(err);
});
req.write('data\n');
req.write('data\n');
req.end();
And here is the challenge header sent back by the API
{ realm: 'API realm',
domain: 'https:/api.example.com/',
qop: 'auth',
nonce: 'UZ43b0FWC9591pMjy1i6H2okVwgMbDVO6fcgcQ' }
EDIT:
I thought it would be helpful for those looking to answer this question for me to provide what I'm actually sending back to the API, so here it is.
{ host: 'api.example.com',
port: 80,
path: '/path/to/uri/',
method: 'GET',
accept: 'application/json',
acceptEncoding: 'gzip, deflate',
connection: 'keep-alive',
rejectUnauthorized: false,
requestCert: true,
agent: false,
headers: { Authorization: 'Digest username="uname", realm="API realm", nonce="UZ43b0FWC9591pMjy1i6H2okVwgMbDVO6fcgcQ", uri="/path/to/uri", qop="auth", response="09c536e22bca031cdbcb289e4065064a", nc="1", cnonce=""' } }
You can use http-auth module that supports digest authentication
// HTTP module
var http = require('http');
// Authentication module.
var auth = require('http-auth');
var digest = auth.digest({
realm: "Simon Area.",
file: __dirname + "/../data/users.htdigest" // vivi:anna, sona:testpass
});
// Creating new HTTP server.
http.createServer(digest, function(req, res) {
res.end("Welcome to private area - " + req.user + "!");
}).listen(1337);

Updating post http request length in node.js

I'm using node.js to post a http request. the code works with if i define my post data ahead of the 'options' field, but if I initially set my post_data string to empty and update it later it doesn't pick up the new length. How would I get it to do that ? I'm looking to send multiple posts of varying lengths to the same place in a loop so need to be able to do this.
var post_data=''; //if i set my string content here rather than later on it works
var options = {
host: '127.0.0.1',
port: 8529,
path: '/_api/cursor',
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Content-Length': post_data.length
}
};
var req = http.request(options, function(res) {
res.setEncoding('utf8');
res.on('data', function (chunk) {
console.log('BODY: ' + chunk);
});
});
req.on('error', function(e) {
console.log('problem with request: ' + e.message);
});
post_data = 'a variable length string goes here';//the change in length to post_data is not //recognised
req.write(post_data);
req.end();
'Content-Length': post_data.length
You ran this before setting post_data.
If you want to set post_data after creating the object, you'll need to set it manually later:
options.headers['Content-Length'] = post_data.length;
Note that you must set that before calling http.request().
Posting data is a matter of sending a query string (just like the way you would send it with an URL after the ?) as the request body.
This also requires to declare Content-Type and Content-Length values so the server knows how to interpret the data.
var querystring = require('querystring');
var data = querystring.stringify({
username: yourUsernameValue,
password: yourPasswordValue
});
var options = {
host: 'my.url',
port: 80,
path: '/login',
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': data.length
}
};
var req = http.request(options, function(res) {
res.setEncoding('utf8');
res.on('data', function (chunk) {
console.log("body: " + chunk);
});
});
req.write(data);
req.end();
You need to replace:
'Content-Length': post_data.length
for:
'Content-Length': Buffer.byteLength(post_data, 'utf-8')
See https://github.com/strongloop/express/issues/1870

Google Closure Compiler moved ?? It's giving a 302 error

I'm using nodejs 0.4.7 to make the request, this is my code:
var post_data = JSON.stringify({
'compilation_level' : 'ADVANCED_OPTIMIZATIONS',
'output_format': 'json',
'warning_level' : 'QUIET',
'js_code' : code
});
var post_options = {
host: 'closure-compiler.appspot.com',
port: '80',
path: 'compile',
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': post_data.length
}
};
var post_req = http.request(post_options, function(res) {
res.setEncoding('utf8');
res.on('data', function (chunk) {
console.log('Response: ' + chunk);
});
});
post_req.write(post_data);
post_req.end();
And the response I get is
Response: <HTML><HEAD><meta http-equiv="content-type" content="text/html;charset=utf-8">
<TITLE>302 Moved</TITLE></HEAD><BODY>
<H1>302 Moved</H1>
The document has moved
here.
</BODY></HTML>
Why is this happening ? What am I doing wrong ? In the tutorial it says I'm suposed to make the POST request to http://closure-compiler.appspot.com/compile...
You're trying to send json data:
var post_data = JSON.stringify({
'compilation_level' : 'ADVANCED_OPTIMIZATIONS',
'output_format': 'json',
'warning_level' : 'QUIET',
'js_code' : code
});
Google Closure Compiler API wants standard form data, so you want to use querystring instead. Also you need to indicate the output format you want (compiled code I assume), as specified by their documentation:
var post_data = querystring.stringify({
'compilation_level' : 'ADVANCED_OPTIMIZATIONS',
'output_format': 'json',
'output_info': 'compiled_code',
'warning_level' : 'QUIET',
'js_code' : code
});
Path is better declared like so:
path: '/compile',
Here is the full proof of concept code:
var http = require('http');
var querystring = require('querystring');
var code ="// ADD YOUR CODE HERE\n" +
"function hello(name) {\n" +
" alert('Hello, ' + name);\n" +
"}\n" +
"hello('New user');\n";
var post_data = querystring.stringify({
'compilation_level' : 'ADVANCED_OPTIMIZATIONS',
'output_format': 'json',
'output_info': 'compiled_code',
'warning_level' : 'QUIET',
'js_code' : code
});
var post_options = {
host: 'closure-compiler.appspot.com',
port: '80',
path: '/compile',
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': post_data.length
}
};
var post_req = http.request(post_options, function(res) {
res.setEncoding('utf8');
res.on('data', function (chunk) {
console.log('Response: ' + chunk);
});
});
post_req.write(post_data);
post_req.end();
Running it with node.js produces the following:
$ node test.js
Response: {"compiledCode":"alert(\"Hello, New user\");"}

Resources