use serde::{Deserialize, Serialize};
use std::time::Duration;
use kafka::producer::{Producer, Record, RequiredAcks};
use opentelemetry::{global, KeyValue};
#[tokio::main]
async fn main() {
let mut producer =
Producer::from_hosts(vec!("localhost:29092".to_owned()))
.with_ack_timeout(Duration::from_secs(1))
.with_required_acks(RequiredAcks::One)
.create()
.unwrap();
let meter = global::meter("test");
let counter = meter.u64_counter("my_counter").init();
counter.add(1, &[KeyValue::new("cpu", "80")]);
counter.add(1, &[KeyValue::new("cpu", "90")]);
let mut buf = String::with_capacity(2);
for _ in 0..10 {
// producer.send(&Record::from_value("KubeStatistics", serde_json::to_string("test").unwrap())).unwrap();
/* How to pass metrics data over here */
buf.clear();
}
}
In the above Rust code, I am trying to create a counter metrics using opentelemetry and sending it to my kafka.
I have created a "counter", how am I supposed to pass this counter to the Kafka. Am I supposed to write a new exporter for Kafka ? If yes, is there any references available ? OR can I get an example snippet ?
Generally, it's preferable to export OpenTelemetry data via OTLP (see Rust docs here) to some intermediary like an OpenTelemetry Collector, then use the Kafka Exporter to write that data to a Kafka topic.
Related
my goal/use case:
Subscribe to a datafeed
Publish to internal subscribers
Example use case: subscribe to stock prices, consume from multiple different bots running on different threads within the same app.
In other languages, I'd be using an RX Subject and simply subscribe to that from anywhere else and I can choose which thread to observe the values on (threadpool or same thread, etc).
Here is my attempt using a simulated data feed:
Code:
async fn test_observable() -> Receiver<Decimal> {
let (x, response) = mpsc::channel::<Decimal>(100);
tokio::spawn(async move {
for i in 0..10 {
sleep(Duration::from_secs(1)).await;
x.send(Decimal::from(i)).await;
}
});
response
}
#[tokio::main]
async fn main() {
let mut o = test_observable().await;
while let Some(x) = o.recv().await {
println!("{}", x);
}
}
Questions:
Is this the right approach? I normally use RX in other languages but it is too complicated in Rust so I resorted to using Rust channels. RX for Rust
I think this approach won't work if I have multiple receivers. How do I work around that? I just want something like an RX observable, it should not be difficult to achieve that.
Is this creating any threads?
I have a tokio TcpStream. I want to pass some type T over this stream. This type T implement Serialize and Deserialize. How can I obtain a Sink<T> and a Stream<T>?
I found the crates tokio_util and tokio_serde, but I can't figure out how to use them to do what I want.
I don't know your code structure or the codec you're planning on using, but I've figured out how to glue everything together into a workable example.
Your Sink<T> and Stream<Item=T> are going to be provided by the Framed type in tokio-serde. This layer deals with passing your messages through serde. This type takes four generic parameters: Transport, Item (the stream item), SinkItem, and Codec. Codec is a wrapper for the specific serializer and deserializer you want to use. You can view the provided options here. Item and SinkItem are just going to be your message type which must implement Serialize and Deserialize. Transport needs to be a Sink<SinkItem> and/or Stream<Item=Item> itself in order for the frame to implement any useful traits. This is where tokio-util comes in. It provides various Framed* types which allow you to convert things implementing AsyncRead/AsyncWrite into streams and sinks respectively. In order to construct these frames, you need to specify a codec which delimits frames from the wire. For simplicity in my example I just used the LengthDelimitedCodec, but there are other options provided as well.
Without further adieu, here's an example of how you can take a tokio::net::TcpStream and split it into an Sink<T> and Stream<Item=T>. Note that T is a result on the stream side because the serde layer can fail if the message is malformed.
use futures::{SinkExt, StreamExt};
use serde::{Deserialize, Serialize};
use tokio::net::{
tcp::{OwnedReadHalf, OwnedWriteHalf},
TcpListener,
TcpStream,
};
use tokio_serde::{formats::Json, Framed};
use tokio_util::codec::{FramedRead, FramedWrite, LengthDelimitedCodec};
#[derive(Serialize, Deserialize, Debug)]
struct MyMessage {
field: String,
}
type WrappedStream = FramedRead<OwnedReadHalf, LengthDelimitedCodec>;
type WrappedSink = FramedWrite<OwnedWriteHalf, LengthDelimitedCodec>;
// We use the unit type in place of the message types since we're
// only dealing with one half of the IO
type SerStream = Framed<WrappedStream, MyMessage, (), Json<MyMessage, ()>>;
type DeSink = Framed<WrappedSink, (), MyMessage, Json<(), MyMessage>>;
fn wrap_stream(stream: TcpStream) -> (SerStream, DeSink) {
let (read, write) = stream.into_split();
let stream = WrappedStream::new(read, LengthDelimitedCodec::new());
let sink = WrappedSink::new(write, LengthDelimitedCodec::new());
(
SerStream::new(stream, Json::default()),
DeSink::new(sink, Json::default()),
)
}
#[tokio::main]
async fn main() {
let listener = TcpListener::bind("0.0.0.0:8080")
.await
.expect("Failed to bind server to addr");
tokio::task::spawn(async move {
let (stream, _) = listener
.accept()
.await
.expect("Failed to accept incoming connection");
let (mut stream, mut sink) = wrap_stream(stream);
println!(
"Server received: {:?}",
stream
.next()
.await
.expect("No data in stream")
.expect("Failed to parse ping")
);
sink.send(MyMessage {
field: "pong".to_owned(),
})
.await
.expect("Failed to send pong");
});
let stream = TcpStream::connect("127.0.0.1:8080")
.await
.expect("Failed to connect to server");
let (mut stream, mut sink) = wrap_stream(stream);
sink.send(MyMessage {
field: "ping".to_owned(),
})
.await
.expect("Failed to send ping to server");
println!(
"Client received: {:?}",
stream
.next()
.await
.expect("No data in stream")
.expect("Failed to parse pong")
);
}
Running this example yields:
Server received: MyMessage { field: "ping" }
Client received: MyMessage { field: "pong" }
Note that it's not required that you split the stream. You could instead construct a tokio_util::codec::Framed out of the TcpStream, and construct a tokio_serde::Framed with a tokio_serde::formats::SymmetricalJson<MyMessage>, and then that Framed would implement Sink and Stream accordingly. Also a lot of the functionality in this example is feature-gated, so be sure to enable the appropriate features according to the docs.
I'm trying to make a server in Rust using tcp protocol. I can make a normal server, through the language documentation, but I don't want that whenever a new connection is made, a new thread is created, nor do I want to use a thread pool, because the tcp connections will be persistent, that is, they will last a long time (around 30min-2h). So, I looped over all the connections and, with a 1 millisecond timeout, I try to read if there are any new packets. However, something tells me this is not the right thing to do. Any idea?
Thanks in advance.
You are probably looking for some asynchronous runtime. Like most runtimes, tokio can be customized to work with a single thread, if you don't have many connections you centainly don't need more than one. If we translate the example #Benjamin Boortz provided:
use tokio::io::*;
use tokio::net::{TcpListener, TcpStream};
#[tokio::main(flavor = "current_thread")]
async fn main() {
let listener = TcpListener::bind("127.0.0.1:7878").await.unwrap();
while let Ok((stream, _address)) = listener.accept().await {
// this is similar to spawning a new thread.
tokio::spawn(handle_connection(stream));
}
}
async fn handle_connection(mut stream: TcpStream) {
let mut buffer = [0; 1024];
stream.read(&mut buffer).await.unwrap();
println!("Request: {}", String::from_utf8_lossy(&buffer[..]));
}
This code is concurrent, yet single threaded, which seems to be what you want. I recommend you check the tokio tutorial. It is a really good resource if you are unfamiliar with asynchronous programming in Rust.
not sure if this is what you need but you can find in "The Rust Programming Language" Book an example for a single threaded TCP Server. https://doc.rust-lang.org/book/ch20-01-single-threaded.html
Please refer to Listing 20-2:
use std::io::prelude::*;
use std::net::TcpListener;
use std::net::TcpStream;
fn main() {
let listener = TcpListener::bind("127.0.0.1:7878").unwrap();
for stream in listener.incoming() {
let stream = stream.unwrap();
handle_connection(stream);
}
}
fn handle_connection(mut stream: TcpStream) {
let mut buffer = [0; 1024];
stream.read(&mut buffer).unwrap();
println!("Request: {}", String::from_utf8_lossy(&buffer[..]));
}
Ref: https://doc.rust-lang.org/book/ch20-01-single-threaded.html#reading-the-request
I'm trying to write a simple Rust program that reads Docker stats using shiplift and exposes them as Prometheus metrics using rust-prometheus.
The shiplift stats example runs correctly on its own, and I'm trying to integrate it in the server as
fn handle(_req: Request<Body>) -> Response<Body> {
let docker = Docker::new();
let containers = docker.containers();
let id = "my-id";
let stats = containers
.get(&id)
.stats().take(1).wait();
for s in stats {
println!("{:?}", s);
}
// ...
}
// in main
let make_service = || {
service_fn_ok(handle)
};
let server = Server::bind(&addr)
.serve(make_service);
but it appears that the stream hangs forever (I cannot produce any error message).
I've also tried the same refactor (using take and wait instead of tokio::run) in the shiplift example, but in that case I get the error executor failed to spawn task: tokio::spawn failed (is a tokio runtime running this future?). Is tokio somehow required by shiplift?
EDIT:
If I've understood correctly, my attempt does not work because wait will block tokio executor and stats will never produce results.
shiplift's API is asynchronous, meaning wait() and other functions return a Future, instead of blocking the main thread until a result is ready. A Future won't actually do any I/O until it is passed to an executor. You need to pass the Future to tokio::run as in the example you linked to. You should read the tokio docs to get a better understanding of how to write asynchronous code in rust.
There were quite a few mistakes in my understanding of how hyper works. Basically:
if a service should handle futures, do not use service_fn_ok to create it (it is meant for synchronous services): use service_fn;
do not use wait: all futures use the same executor, the execution will just hang forever (there is a warning in the docs but oh well...);
as ecstaticm0rse notices, hyper::rt::spawn could be used to read stats asynchronously, instead of doing it in the service
Is tokio somehow required by shiplift?
Yes. It uses hyper, which throws executor failed to spawn task if the default tokio executor is not available (working with futures nearly always requires an executor anyway).
Here is a minimal version of what I ended up with (tokio 0.1.20 and hyper 0.12):
use std::net::SocketAddr;
use std::time::{Duration, Instant};
use tokio::prelude::*;
use tokio::timer::Interval;
use hyper::{
Body, Response, service::service_fn_ok,
Server, rt::{spawn, run}
};
fn init_background_task(swarm_name: String) -> impl Future<Item = (), Error = ()> {
Interval::new(Instant::now(), Duration::from_secs(1))
.map_err(|e| panic!(e))
.for_each(move |_instant| {
futures::future::ok(()) // unimplemented: call shiplift here
})
}
fn init_server(address: SocketAddr) -> impl Future<Item = (), Error = ()> {
let service = move || {
service_fn_ok(|_request| Response::new(Body::from("unimplemented")))
};
Server::bind(&address)
.serve(service)
.map_err(|e| panic!("Server error: {}", e))
}
fn main() {
let background_task = init_background_task("swarm_name".to_string());
let server = init_server(([127, 0, 0, 1], 9898).into());
run(hyper::rt::lazy(move || {
spawn(background_task);
spawn(server);
Ok(())
}));
}
I want to download large files (500mb) with hyper, and be able to resume if the download fails.
Is there any way with hyper to run some function for each chunk of data received? The send() method returns a Result<Response>, but I can't find any methods on Response that return an iterator over chunks. Ideally I'd be able to do something like:
client.get(&url.to_string())
.send()
.map(|mut res| {
let mut chunk = String::new();
// write this chunk to disk
});
Is this possible, or will map only be called once hyper has downloaded the entire file?
Is there any way with hyper to run some function for each chunk of data received?
Hyper's Response implements Read. It means that Response is a stream and you can read arbitrary chunks of data from it as you would usually do with a stream.
For what it's worth, here's a piece of code I use to download large files from ICECat. I'm using the Read interface in order to display the download progress in the terminal.
The variable response here is an instance of Hyper's Response.
{
let mut file = try_s!(fs::File::create(&tmp_path));
let mut deflate = try_s!(GzDecoder::new(response));
let mut buf = [0; 128 * 1024];
let mut written = 0;
loop {
status_line! ("icecat_fetch] " (url) ": " (written / 1024 / 1024) " MiB.");
let len = match deflate.read(&mut buf) {
Ok(0) => break, // EOF.
Ok(len) => len,
Err(ref err) if err.kind() == io::ErrorKind::Interrupted => continue,
Err(err) => return ERR!("{}: Download failed: {}", url, err),
};
try_s!(file.write_all(&buf[..len]));
written += len;
}
}
try_s!(fs::rename(tmp_path, target_path));
status_line_clear();
I want to download large files (500mb) with hyper, and be able to resume if the download fails.
This is usually implemented with the HTTP "Range" header (cf. RFC 7233).
Not every server out there supports the "Range" header. I've seen a lot of servers with a custom HTTP stack and without the proper "Range" support, or with the "Range" header disabled for some reason. So skipping the Hyper's Response chunks might be a necessary fallback.
But if you want to speed things up and save traffic then the primary means of resuming a stopped download should be by using the "Range" header.
P.S. With Hyper 0.12 the response body returned by the Hyper is a Stream and to run some function for each chunk of data received we can use the for_each stream combinator:
extern crate futures;
extern crate futures_cpupool;
extern crate hyper; // 0.12
extern crate hyper_rustls;
use futures::Future;
use futures_cpupool::CpuPool;
use hyper::rt::Stream;
use hyper::{Body, Client, Request};
use hyper_rustls::HttpsConnector;
use std::thread;
use std::time::Duration;
fn main() {
let url = "https://steemitimages.com/DQmYWcEumaw1ajSge5PcGpgPpXydTkTcqe1daF4Ro3sRLDi/IMG_20130103_103123.jpg";
// In real life we'd want an asynchronous reactor, such as the tokio_core, but for a short example the `CpuPool` should do.
let pool = CpuPool::new(1);
let https = HttpsConnector::new(1);
let client = Client::builder().executor(pool.clone()).build(https);
// `unwrap` is used because there are different ways (and/or libraries) to handle the errors and you should pick one yourself.
// Also to keep this example simple.
let req = Request::builder().uri(url).body(Body::empty()).unwrap();
let fut = client.request(req);
// Rebinding (shadowing) the `fut` variable allows us (in smart IDEs) to more easily examine the gradual weaving of the types.
let fut = fut.then(move |res| {
let res = res.unwrap();
println!("Status: {:?}.", res.status());
let body = res.into_body();
// `for_each` returns a `Future` that we must embed into our chain of futures in order to execute it.
body.for_each(move |chunk| {println!("Got a chunk of {} bytes.", chunk.len()); Ok(())})
});
// Handle the errors: we need error-free futures for `spawn`.
let fut = fut.then(move |r| -> Result<(), ()> {r.unwrap(); Ok(())});
// Spawning the future onto a runtime starts executing it in background.
// If not spawned onto a runtime the future will be executed in `wait`.
//
// Note that we should keep the future around.
// To save resources most implementations would *cancel* the dropped futures.
let _fut = pool.spawn(fut);
thread::sleep (Duration::from_secs (1)); // or `_fut.wait()`.
}