AWS X-Ray w/ HAPI - No logs in Console - node.js

Question

Trying out AWS new service X-RAY for my microservice deployment on Kubernetes on EC2. Here are the logs I am seeing:
root#internal-reporting-test-1680758663-3gweh:/usr/src/app# cat AWSXRay.log
2016-12-06 04:36:23.5840 +00:00 [INFO] UDP message sent: {"trace_id":"1-58464048-03fbcdf5c861f95fc5204ec6","id":"105107e6c9df4bfe","start_time":1480998983.553,"name":"localhost","http":{"request":{"method":"GET","user_agent":"curl/7.38.0","client_ip":"127.0.0.1","url":"http://localhost"},"response":{"status":404,"content_length":0}},"service":{"version":"1.0.0"},"error":true,"end_time":1480998983.574}
2016-12-06 04:36:33.2560 +00:00 [INFO] UDP message sent: {"trace_id":"1-58464051-3e3799a566c0933093f51b7a","id":"94d4267609d2ba40","start_time":1480998993.205,"name":"localhost","http":{"request":{"method":"GET","user_agent":"curl/7.38.0","client_ip":"127.0.0.1","url":"http://localhost"},"response":{"status":200,"content_length":0}},"service":{"version":"1.0.0"},"end_time":1480998993.252}
2016-12-06 04:36:35.5810 +00:00 [INFO] UDP message sent: {"trace_id":"1-58464054-5df1e9ec46ef2b26b2d29965","id":"113fd36cddb5de95","start_time":1480998995.552,"name":"localhost","http":{"request":{"method":"GET","user_agent":"curl/7.38.0","client_ip":"127.0.0.1","url":"http://localhost"},"response":{"status":200,"content_length":0}},"service":{"version":"1.0.0"},"end_time":1480998995.579}
2016-12-06 04:36:41.3350 +00:00 [INFO] UDP message sent: {"trace_id":"1-58464059-0dfd2e274ca20499c6bce17f","id":"1dfac818142b2d28","start_time":1480999001.304,"name":"localhost","http":{"request":{"method":"GET","user_agent":"curl/7.38.0","client_ip":"127.0.0.1","url":"http://localhost"},"response":{"status":200,"content_length":0}},"service":{"version":"1.0.0"},"end_time":1480999001.332}
Curl I am using:
# curl localhost/v1/internal/report/users
Supervisor Conf:
[supervisord]
nodaemon=true
[program:xray]
command=/usr/src/app/xray
[program:npm]
command=npm start
Mods to express middleware to use with hapi:
/**
* Express_mw module.
*
* Exposes middleware functions to enable automated data capturing on a web service. To enable on a Node.js/Express application,
* use 'app.use(AWSXRay.express.openSegment())' before defining your routes. After your routes, before any extra error
* handling middleware, use 'app.use(AWSXRay.express.closeSegment())'.
* Use AWSXRay.getSegment() to access the current segment/subsegment.
* Otherwise, for manual mode, this appends the Segment object to the request object as req.segment.
* #module express_mw
*/
var _ = require('underscore');
var CLSUtils = require('../utils').CLSUtils;
var MWUtils = require('./mw_utils');
var Local = require('../segments/attributes/local');
var Segment = require('../segments/segment');
var getCauseType = require('../utils').getCauseTypeFromHttpStatus;
var XRAY_HEADER = 'x-amzn-trace-id';
var NAME = process.env.XRAY_TRACING_NAME;
var DEFAULT_NAME = process.env.XRAY_TRACING_DEFAULT_NAME;
/**
* Use 'app.use(AWSXRay.express.openSegment())' before defining your routes.
* Use AWSXRay.getSegment() to access the current segment/subsegment.
* Otherwise, for manual mode, this appends the Segment object to the request object as req.segment.
* #alias module:express_mw.openSegment
* #returns {function}
*/
module.exports.openSegment = function openSegment() {
if (_.isUndefined(DEFAULT_NAME) && _.isUndefined(MWUtils.defaultName)) {
throw new Error('Default segment name must be set using "process.env.XRAY_TRACING_DEFAULT_NAME"' +
' or "setDefaultName()".');
}
return function open(request, next) {
/*
* Get the true HTTP objects
*/
req = request.raw.req;
res = request.raw.res;
var amznTraceHeader = processHeaders(req);
var segment = new Segment(resolveName(req.headers.host), amznTraceHeader.Root, amznTraceHeader.Parent);
resolveSampling(amznTraceHeader, segment, res);
segment.addAttribute('http', new Local(req));
MWUtils.populate(segment);
res.on('finish', function () {
if (this.statusCode === 429)
segment.addThrottle();
else if (getCauseType(this.statusCode))
segment[getCauseType(this.statusCode)] = true;
segment.http.close(this);
segment.close();
});
if (CLSUtils.isCLSMode()) {
var ns = CLSUtils.getNamespace();
ns.bindEmitter(req);
ns.bindEmitter(res);
ns.run(function () {
CLSUtils.setSegment(segment);
if (next) { next(); }
});
} else {
req.segment = segment;
if (next) { next(); }
}
};
};
/**
* After your routes, before any extra error handling middleware, use 'app.use(AWSXRay.express.closeSegment())'.
* #alias module:express_mw.closeSegment
* #returns {function}
*/
module.exports.closeSegment = function closeSegment() {
return function close(request, next) {
req = request.raw.req;
res = request.raw.res;
var segment = CLSUtils.isCLSMode() ? CLSUtils.getSegment() : segment = req.segment;
if (segment && err) {
segment.counter = 0;
segment.close(err);
} else if (segment) {
segment.close();
}
if (next)
next(err);
};
};
function processHeaders(req) {
var amznTraceHeader = {};
if (req && req.headers && req.headers[XRAY_HEADER]) {
_.each(req.headers[XRAY_HEADER].replace(/ /g,'').split(';'), function (header) {
var pair = header.split('=');
this[pair[0]] = pair[1];
}, amznTraceHeader);
}
return amznTraceHeader;
}
function resolveName(hostName) {
if (NAME)
return NAME;
var regex = new RegExp('(?:[0-9]{1,3}\.){3}[0-9]{1,3}', 'g');
var hostIp = hostName.match(regex) || _.isEmpty(hostName);
return !hostIp ? hostName : (DEFAULT_NAME ? DEFAULT_NAME : MWUtils.defaultName);
}
function resolveSampling(amznTraceHeader, segment, res) {
var isSampled;
if (amznTraceHeader.Sampled === '1')
isSampled = true;
else if (amznTraceHeader.Sampled === '0')
isSampled = false;
isSampled = !_.isUndefined(isSampled) ? isSampled : MWUtils.shouldSample();
if (amznTraceHeader.Sampled === '?')
res.header[XRAY_HEADER] = 'Root=' + amznTraceHeader.Root + '; Sampled=' + (isSampled ? '1' : '0');
if (!isSampled)
segment.notTraced = true;
}
The changes are relatively simple while playing with this service. Just pretty much forcing the raw node HTTP objects to the middleware.
Logs are showing the UDP requests to the xray service are working as expected. Yet in the console I see no results.
Anyone have any ideas?