How to send traces from rust + tracing_opentelemetry to Honeycomb? [duplicate] - rust

Question

I'm trying to use opentelemetry and open telemetry-otlp to provide observability data over OTLP to Honeycomb.
I'm using something like this as a proof of concept (extracted out into this repo if you want to run it: https://github.com/timfpark/honeycomb-rust-poc)
fn init_tracer(metadata: &MetadataMap) -> Result<sdktrace::Tracer, TraceError> {
let opentelemetry_endpoint =
env::var("OTEL_ENDPOINT").unwrap_or_else(|_| "https://api.honeycomb.io".to_owned());
let opentelemetry_endpoint =
Url::parse(&opentelemetry_endpoint).expect("OTEL_ENDPOINT is not a valid url");
opentelemetry_otlp::new_pipeline()
.tracing()
.with_exporter(
opentelemetry_otlp::new_exporter()
.tonic()
.with_endpoint(opentelemetry_endpoint.as_str())
.with_metadata(metadata.clone())
.with_tls_config(
ClientTlsConfig::new().domain_name(
opentelemetry_endpoint
.host_str()
.expect("OTEL_ENDPOINTshould have a valid host"),
),
),
)
.install_batch(opentelemetry::runtime::Tokio)
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let mut metadata = MetadataMap::with_capacity(2);
metadata.insert(
"x-honeycomb-team",
"...honeycomb api key...".parse().unwrap(),
);
metadata.insert("x-honeycomb-dataset", "my-api".parse().unwrap());
let tracer = init_tracer(&metadata).expect("failed to instantiate opentelemetry tracing");
tracing_subscriber::registry()
.with(tracing_subscriber::EnvFilter::from_default_env())
.with(tracing_opentelemetry::layer().with_tracer(tracer))
.with(tracing_subscriber::fmt::layer())
.try_init()
.expect("failed to register tracer with registry");
let tracer = global::tracer("ex.com/basic");
but I am getting:
2022-11-02T17:01:01.088429Z DEBUG hyper::client::connect::http: connecting to 52.5.162.226:443
2022-11-02T17:01:01.170767Z DEBUG hyper::client::connect::http: connected to 52.5.162.226:443
2022-11-02T17:01:01.171870Z DEBUG rustls::client::hs: No cached session for DnsName(DnsName(DnsName("api.honeycomb.io")))
2022-11-02T17:01:01.172555Z DEBUG rustls::client::hs: Not resuming any session
2022-11-02T17:01:01.269218Z DEBUG rustls::client::hs: ALPN protocol is Some(b"h2")
2022-11-02T17:01:01.269398Z DEBUG rustls::client::hs: Using ciphersuite TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
2022-11-02T17:01:01.269504Z DEBUG rustls::client::tls12::server_hello: Server supports tickets
2022-11-02T17:01:01.269766Z DEBUG rustls::client::tls12: ECDHE curve is ECParameters { curve_type: NamedCurve, named_group: secp256r1 }
2022-11-02T17:01:01.269843Z DEBUG rustls::client::tls12: Server DNS name is DnsName(DnsName(DnsName("api.honeycomb.io")))
2022-11-02T17:01:01.271123Z WARN rustls::conn: Sending fatal alert BadCertificate
2022-11-02T17:01:01.271861Z DEBUG tonic::transport::service::reconnect: reconnect::poll_ready: hyper::Error(Connect, Custom { kind: InvalidData, error: InvalidCertificateData("invalid peer certificate: UnknownIssuer") })
2022-11-02T17:01:01.271967Z DEBUG tower::buffer::worker: service.ready=true processing request
2022-11-02T17:01:01.272169Z DEBUG tonic::transport::service::reconnect: error: error trying to connect: invalid peer certificate contents: invalid peer certificate: UnknownIssuer
OpenTelemetry trace error occurred. Exporter otlp encountered the following error(s): the grpc server returns error (The service is currently unavailable): , detailed error message: error trying to connect: invalid peer certificate contents: invalid peer certificate: UnknownIssuer
which seems to indicate that something about my TLS setup is not correct... Does anyone have a snippet of opentelemetry code in Rust that works with Honeycomb?

Answer 1

The problem is: you need to give ClientTlsConfig a root certificate, that the target site (api.honeycomb.io) chains back to.
I found a suitable root cert installed in my container, and then made the program load it in.
Here's the code:
let pem = tokio::fs::read("/etc/ssl/certs/Starfield_Services_Root_Certificate_Authority_-_G2.pem").await.expect("read the cert file");
let cert = Certificate::from_pem(pem);
let mut metadata = MetadataMap::with_capacity(1);
metadata.insert("x-honeycomb-team", honeycomb_api_key.parse().unwrap());
let opentelemetry_endpoint =
env::var("OTEL_ENDPOINT").unwrap_or_else(|_| "https://api.honeycomb.io".to_owned());
let opentelemetry_endpoint =
Url::parse(&opentelemetry_endpoint).expect("OTEL_ENDPOINT is not a valid url");
opentelemetry_otlp::new_pipeline()
.tracing()
.with_exporter(
opentelemetry_otlp::new_exporter()
.tonic()
.with_endpoint(opentelemetry_endpoint.as_str())
.with_metadata(metadata.clone())
.with_tls_config(
ClientTlsConfig::new().ca_certificate(cert)
),
)
.install_batch(opentelemetry::runtime::Tokio)
}
The first two lines are new; they load a root certificate from the filesystem.
Then use that to configure the ClientTlsConfig.
I chose that root certificate file based on the output of the certificate details:
openssl s_client -connect api.honeycomb.io:443 -servername localhost
The last entry in the certificate chain resembled the filename. It included: /CN=Starfield Services Root Certificate Authority - G2

Answer 2

For reference, a complete solution to Rust/Honeycomb/Tracing/OpenTelemetry implementation is now available here: https://github.com/Dhghomon/rust_opentelemetry_honeycomb.
It does not need to use certs.
UPDATE: Expanding a little since this is the accepted answer now.
The core tracer initialization code is as follows:
fn init_tracer() -> Result<sdktrace::Tracer, TraceError> {
opentelemetry_otlp::new_pipeline()
.tracing()
.with_exporter(
opentelemetry_otlp::new_exporter()
.http()
.with_endpoint("https://api.honeycomb.io/v1/traces")
.with_http_client(reqwest::Client::default())
.with_headers(HashMap::from([
("x-honeycomb-dataset".into(), DATASET_NAME.into()),
("x-honeycomb-team".into(), API_KEY.into()),
]))
.with_timeout(std::time::Duration::from_secs(2)),
) // Replace with runtime::Tokio if using async main
.install_batch(opentelemetry::runtime::TokioCurrentThread)
}
The subscriber is configured like this:
let tracer = init_tracer().unwrap();
let telemetry = tracing_opentelemetry::layer().with_tracer(tracer);
let subscriber = tracing_subscriber::Registry::default().with(telemetry);
tracing::subscriber::set_global_default(subscriber).unwrap();

Answer 3

In addition to #jessitron's answer I was also able to get Honeycomb working with a Rust service using a separate OpenTelemetry collector service. The instructions for how to configure that collector within a Kubernetes cluster are here, also by #jessitron.
With that, then you can use a simpler configuration for reporting traces to the collector, Ala:
use opentelemetry::sdk::trace as sdktrace;
use opentelemetry::trace::{TraceContextExt, TraceError, Tracer};
use opentelemetry::{global, Key};
use opentelemetry_otlp::WithExportConfig;
use std::env;
use tonic::metadata::MetadataMap;
use tonic::transport::ClientTlsConfig;
use tracing_subscriber::prelude::*;
use url::Url;
fn init_tracer() -> Result<sdktrace::Tracer, TraceError> {
// Configure OTEL_ENDPOINT to point at your collector
let opentelemetry_endpoint =
env::var("OTEL_ENDPOINT").unwrap_or_else(|_| "http://localhost:4317".to_owned());
let opentelemetry_endpoint =
Url::parse(&opentelemetry_endpoint).expect("OTEL_ENDPOINT is not a valid url");
opentelemetry_otlp::new_pipeline()
.tracing()
.with_exporter(
opentelemetry_otlp::new_exporter()
.tonic()
.with_endpoint(opentelemetry_endpoint.as_str()),
)
.install_batch(opentelemetry::runtime::Tokio)
}
#[tokio::main]
async fn main() {
let tracer = init_tracer().expect("failed to instantiate opentelemetry tracing");
tracing_subscriber::registry()
.with(tracing_subscriber::EnvFilter::from_default_env())
.with(tracing_opentelemetry::layer().with_tracer(tracer))
.with(tracing_subscriber::fmt::layer())
.try_init()
.expect("failed to register tracer with registry");
const LEMONS_KEY: Key = Key::from_static_str("lemons");
const ANOTHER_KEY: Key = Key::from_static_str("ex.com/another");
let tracer = global::tracer("ex.com/basic");
tracer.in_span("operation", |cx| {
let span = cx.span();
span.add_event(
"Nice operation!".to_string(),
vec![Key::new("bogons").i64(100)],
);
span.set_attribute(ANOTHER_KEY.string("yes"));
tracer.in_span("Sub operation...", |cx| {
let span = cx.span();
span.set_attribute(LEMONS_KEY.string("five"));
span.add_event("Sub span event", vec![]);
});
});
loop {
tracing::info!("just sleeping, press ctrl-c to exit");
tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
}
}
As one final help, if you use the Helm chart to deploy the collector, you will want to use a configuration that looks something like this:
values:
mode: deployment
config:
exporters:
otlp/honeycomb:
endpoint: api.honeycomb.io:443
headers:
"x-honeycomb-team": "{{{honeycomb-api-key}}}"
service:
pipelines:
traces:
receivers:
- otlp
exporters:
- otlp/honeycomb
Thanks again to #jessitron for her clutch help. :)