I'm trying to implement a retry in a client built with Hyper v0.11, but I can't find a way to reuse a request for different attempts:
#[macro_use]
extern crate hyper;
extern crate futures;
extern crate tokio_core;
use futures::Future;
use hyper::{Client, Body, Uri, StatusCode};
use hyper::server::{Request, Response};
use hyper::client::HttpConnector;
use hyper::Get;
use tokio_core::reactor::Core;
fn main() {
let mut core = Core::new().expect("Event Loop");
let handle = core.handle();
let client = Client::new(&handle.clone());
// Request
let json = r#"{"user":"Peter"}"#;
let mut req: Request<Body> = Request::new(Post, "http://localhost:8080/create/user".parse().unwrap());
req.headers_mut().set(ContentType::json());
req.headers_mut().set(ContentLength(json.len() as u64));
req.set_body(json);
dispatch_request(&client, req, 2);
}
fn clone_req(req: &Request) -> Request {
let mut new_req = Request::new(req.method().clone(), req.uri().clone());
new_req.headers_mut().extend(req.headers().iter());
new_req.set_body(req.body()); // <------- here the error occur!
new_req
}
fn dispatch_request(
client: &Client<HttpConnector, Body>,
req: Request<Body>,
n_retry: u32,
) -> Box<Future<Error = hyper::Error, Item = Response>> {
println!("Attemp {}", n_retry);
let max_retry = 3;
let client_clone = client.clone();
let clone_req = clone_req(&req);
let resp = client.request(req).then(move |result| match result {
Ok(client_resp) => {
if client_resp.status() == hyper::StatusCode::Ok {
Box::new(futures::future::ok(client_resp))
} else if n_retry < max_retry {
dispatch_request(&client_clone, clone_req, max_retry + 1)
} else {
Box::new(futures::future::ok(
Response::new().with_status(StatusCode::ServiceUnavailable),
))
}
}
Err(e) => {
println!("Connection error: {:?}", &e);
Box::new(futures::future::ok(
Response::new().with_status(StatusCode::ServiceUnavailable),
))
}
});
Box::new(resp)
}
This is the compilation error:
error[E0507]: cannot move out of borrowed content
--> src/main.rs:28:22
|
28 | new_req.set_body(req.body());
| ^^^ cannot move out of borrowed content
The error is clear, but I don't know how to fix it.
An option is to use the tokio-retry crate. I only tried with hyper v0.12 though.
Why not put retry in main loop? Note you also need to do core.run somewhere.
loop {
let req = Request::new(Get, "http://www.google.com".parse().unwrap());
let resp = dispatch_request(&client, req, );
if let Ok(_) = resp.wait() {
break
}
}
Related
I'd like to both read and process messages from two channels and construct another message and send this message via another channel.
Messages from the two channels are received at different frequencies (as per sleep).
Example: "foo1" and "bar1" are received, so we process them and form "foo1bar1". "foo2" is received ("bar2" will be received in 2sec), so we will process it as "foo2bar1". "foo3" is received, so "foo3bar1" is constructed. When "bar2" is received, then we get "foo4bar2" and so on.
In the current implementation, since the two tasks don't communicate with one another, I cannot do the "fooNbarM" construction.
use std::time::Duration;
use tokio;
use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender};
use tokio::time::sleep;
use futures::future::join_all;
async fn message_sender(msg: &'static str, foo_tx: UnboundedSender<Result<&str, Box<dyn std::error::Error + Send>>>) {
loop {
match foo_tx.send(Ok(msg)) {
Ok(()) => {
if msg == "foo" {
sleep(Duration::from_millis(1000)).await;
} else {
sleep(Duration::from_millis(3000)).await;
}
}
Err(_) => {
println!("failed to send foo");
break;
}
}
}
}
#[tokio::main]
async fn main() {
let result: Vec<&str> = vec![];
let (foo_tx, mut foo_rx): (
UnboundedSender<Result<&str, Box<dyn std::error::Error + Send>>>,
UnboundedReceiver<Result<&str, Box<dyn std::error::Error + Send>>>,
) = tokio::sync::mpsc::unbounded_channel();
let (bar_tx, mut bar_rx): (
UnboundedSender<Result<&str, Box<dyn std::error::Error + Send>>>,
UnboundedReceiver<Result<&str, Box<dyn std::error::Error + Send>>>,
) = tokio::sync::mpsc::unbounded_channel();
let foo_sender_handle = tokio::spawn(async move {
message_sender("foo", foo_tx).await;
});
let foo_handle = tokio::spawn(async move {
while let Some(v) = foo_rx.recv().await {
println!("{:?}", v);
}
});
let bar_sender_handle = tokio::spawn(async move {
message_sender("bar", bar_tx).await;
});
let bar_handle = tokio::spawn(async move {
while let Some(v) = bar_rx.recv().await {
println!("{:?}", v);
}
});
let handles = vec![foo_sender_handle, foo_handle, bar_sender_handle, bar_handle];
join_all(handles.into_iter()).await;
}
Cargo.toml
[package]
name = "play"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
tokio = { version = "1.16.1", features = ["full"] }
futures = "0.3.21"
Use tokio::select to wait for either channel to become ready:
use futures::future; // 0.3.19
use std::time::Duration;
use tokio::{
sync::mpsc::{self, UnboundedSender},
time,
}; // 1.16.1
async fn message_sender(msg: &'static str, foo_tx: UnboundedSender<String>) {
for count in 0.. {
let message = format!("{msg}{count}");
foo_tx.send(message).unwrap();
if msg == "foo" {
time::sleep(Duration::from_millis(100)).await;
} else {
time::sleep(Duration::from_millis(300)).await;
}
}
}
#[tokio::main]
async fn main() {
let (foo_tx, mut foo_rx) = mpsc::unbounded_channel();
let (bar_tx, mut bar_rx) = mpsc::unbounded_channel();
let foo_sender_handle = tokio::spawn(message_sender("foo", foo_tx));
let bar_sender_handle = tokio::spawn(message_sender("bar", bar_tx));
let receive_handle = tokio::spawn(async move {
let mut foo = None;
let mut bar = None;
loop {
tokio::select! {
f = foo_rx.recv() => foo = f,
b = bar_rx.recv() => bar = b,
}
if let (Some(foo), Some(bar)) = (&foo, &bar) {
println!("{foo}{bar}");
}
}
});
future::join_all([foo_sender_handle, bar_sender_handle, receive_handle]).await;
}
You also have to handle the case where only one message has been received yet, so Option comes in useful.
I'm trying to create a counter in a Hyper web server that counts the number of requests it has received. I'm using a Arc<Mutex<u64>> to hold onto count. However, I haven't been able to figure out the right combination of move and .clone() to satisfy the types of the closures. Here's some code that compiles, but resets the counter on each request:
extern crate hyper;
use hyper::rt::Future;
use hyper::service::service_fn_ok;
use hyper::{Body, Response, Server};
use std::sync::{Arc, Mutex};
fn main() {
let addr = "0.0.0.0:3000".parse().unwrap();
// FIXME want to create the counter here, not below
let server = Server::bind(&addr)
.serve(|| {
service_fn_ok(|_req| {
let counter = Arc::new(Mutex::new(0));
use_counter(counter)
})
})
.map_err(|e| eprintln!("Error: {}", e));
hyper::rt::run(server)
}
fn use_counter(counter: Arc<Mutex<u64>>) -> Response<Body> {
let mut data = counter.lock().unwrap();
*data += 1;
Response::new(Body::from(format!("Counter: {}\n", data)))
}
It turns out I was pretty close, and looking at a few other examples helped me realize the problem. Since there are two layers of closures at play here, I need to move the counter into the outer closure, clone it, and then move that clone into the inner closure and clone there again. To wit:
extern crate hyper; // 0.12.10
use hyper::rt::Future;
use hyper::service::service_fn_ok;
use hyper::{Body, Response, Server};
use std::sync::{Arc, Mutex};
fn main() {
let addr = "0.0.0.0:3000".parse().unwrap();
let counter = Arc::new(Mutex::new(0));
let server = Server::bind(&addr)
.serve(move || {
let counter = counter.clone();
service_fn_ok(move |_req| use_counter(counter.clone()))
})
.map_err(|e| eprintln!("Error: {}", e));
hyper::rt::run(server)
}
fn use_counter(counter: Arc<Mutex<u64>>) -> Response<Body> {
let mut data = counter.lock().unwrap();
*data += 1;
Response::new(Body::from(format!("Counter: {}\n", data)))
}
Update February 2020 Here's a version using hyper 0.13:
use hyper::{Body, Response, Server, Request};
use std::sync::{Arc, Mutex};
use hyper::service::{make_service_fn, service_fn};
use std::convert::Infallible;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let addr = "0.0.0.0:3000".parse()?;
let counter = Arc::new(Mutex::new(0));
let make_service = make_service_fn(move |_conn| {
let counter = counter.clone();
async move {
Ok::<_, Infallible>(service_fn(move |_req: Request<Body>| {
let counter = counter.clone();
async move {
Ok::<_, Infallible>(use_counter(counter))
}
}))
}
});
Server::bind(&addr).serve(make_service).await?;
Ok(())
}
fn use_counter(counter: Arc<Mutex<u64>>) -> Response<Body> {
let mut data = counter.lock().unwrap();
*data += 1;
Response::new(Body::from(format!("Counter: {}\n", data)))
}
I can't seem to figure out how to get Rust to accept a client and proxied client in the same variable. While I am still new to Rust, I have a basic understanding of programming. So far I have tried structs (but no impl's though), type casting, uninitialized variables, but nothing is working.
extern crate futures;
extern crate hyper;
extern crate hyper_proxy;
extern crate stopwatch;
extern crate tokio_core;
use futures::{Future, Stream};
use hyper::client::HttpConnector;
use hyper::Client;
use hyper_proxy::{Intercept, Proxy, ProxyConnector};
use tokio_core::reactor::Core;
fn main() {
let use_proxy = true;
let proxy_uri: Option<String> = Some("http://localhost:8118".to_owned());
let mut core = Core::new().unwrap();
let handle = core.handle();
let mut proxy = None;
// looking for polymorphic variable that works with both proxyed and unproxyed hyper clients
let mut client: hyper::Client<hyper::client::HttpConnector, hyper::Body>;
if use_proxy && proxy_uri.is_some() {
println!("Using proxy: {}", proxy_uri.unwrap().as_str());
proxy = Some({
let proxy_uri = proxy_uri.unwrap().parse().unwrap();
let mut proxy = Proxy::new(Intercept::All, proxy_uri);
let connector = HttpConnector::new(4, &handle);
let proxy_connector = ProxyConnector::from_proxy(connector, proxy).unwrap();
proxy_connector
});
client = Client::configure()
.connector(proxy.clone().unwrap())
.build(&handle);
} else {
client = Client::configure()
.connector(HttpConnector::new(4, &handle))
.build(&handle);
}
// use hyper client below
}
[dependencies]
futures = "0.1.21"
hyper = "0.11.27"
tokio-core = "0.1.17"
hyper-proxy = "0.4.1"
stopwatch = "0.0.7"
I have made a GitHub repo of all the files.
I get this error when trying to compile:
error[E0308]: mismatched types
--> src/main.rs:32:18
|
32 | client = Client::configure()
| __________________^
33 | | .connector(proxy.clone().unwrap())
34 | | .build(&handle);
| |___________________________^ expected struct `hyper::client::HttpConnector`, found struct `hyper_proxy::ProxyConnector`
|
= note: expected type `hyper::Client<hyper::client::HttpConnector, _>`
found type `hyper::Client<hyper_proxy::ProxyConnector<hyper::client::HttpConnector>, _>`
If there is a better approach to this, I would also like to know about it.
This solution is not pretty, but it does work.
We start by creating an enum to handle the two cases:
enum ProxyOrNotConnector {
Proxy(ProxyConnector<HttpConnector>),
Not(HttpConnector),
}
This enum can be a single type representing both cases. Constructing it is straightforward with a match statement:
let http_connector = HttpConnector::new(4, &handle);
let connector = match (proxy_uri, use_proxy) {
(Some(proxy_uri), true) => {
println!("Using proxy: {}", proxy_uri);
let proxy_uri = proxy_uri.parse().unwrap();
let mut proxy = Proxy::new(Intercept::All, proxy_uri);
let proxy_connector = ProxyConnector::from_proxy(http_connector, proxy).unwrap();
ProxyOrNotConnector::Proxy(proxy_connector)
}
_ => ProxyOrNotConnector::Not(http_connector),
};
We can then create a Client using this connector:
let client = Config::default().connector(connector).build(&handle);
This won't work until we've implemented Connect for our enum. There's a blanket implementation of Connect for any type that implements Service in the correct manner, so we go that route:
impl Service for ProxyOrNotConnector {
type Request = Uri;
type Response = Box<AsyncRw>;
type Error = io::Error;
type Future = Box<Future<Item = Self::Response, Error = Self::Error>>;
fn call(&self, req: Self::Request) -> Self::Future {
match self {
ProxyOrNotConnector::Proxy(p) => {
let x = p.call(req);
let y = x.map(|y| Box::new(y) as Box<AsyncRw>);
Box::new(y)
}
ProxyOrNotConnector::Not(n) => {
let x = n.call(req);
let y = x.map(|y| Box::new(y) as Box<AsyncRw>);
Box::new(y)
}
}
}
}
We use multiple trait objects to perform runtime polymorphism: one for the future returned by connecting and another for each value yielded by that future.
Complete code:
extern crate futures;
extern crate hyper;
extern crate hyper_proxy;
extern crate tokio_core;
extern crate tokio_io;
use futures::Future;
use hyper::{
client::{Config, HttpConnector, Service},
Uri,
};
use hyper_proxy::{Intercept, Proxy, ProxyConnector};
use std::io;
use tokio_core::reactor::Core;
use tokio_io::{AsyncRead, AsyncWrite};
trait AsyncRw: AsyncWrite + AsyncRead {}
impl<T> AsyncRw for T where T: AsyncWrite + AsyncRead {}
enum ProxyOrNotConnector {
Proxy(ProxyConnector<HttpConnector>),
Not(HttpConnector),
}
impl Service for ProxyOrNotConnector {
type Request = Uri;
type Response = Box<AsyncRw>;
type Error = io::Error;
type Future = Box<Future<Item = Self::Response, Error = Self::Error>>;
fn call(&self, req: Self::Request) -> Self::Future {
match self {
ProxyOrNotConnector::Proxy(p) => {
let x = p.call(req);
let y = x.map(|y| Box::new(y) as Box<AsyncRw>);
Box::new(y)
}
ProxyOrNotConnector::Not(n) => {
let x = n.call(req);
let y = x.map(|y| Box::new(y) as Box<AsyncRw>);
Box::new(y)
}
}
}
}
fn main() {
let mut core = Core::new().unwrap();
let handle = core.handle();
let proxy_uri = Some("http://127.0.0.1");
let use_proxy = true;
let http_connector = HttpConnector::new(4, &handle);
let connector = match (proxy_uri, use_proxy) {
(Some(proxy_uri), true) => {
println!("Using proxy: {}", proxy_uri);
let proxy_uri = proxy_uri.parse().unwrap();
let mut proxy = Proxy::new(Intercept::All, proxy_uri);
let proxy_connector = ProxyConnector::from_proxy(http_connector, proxy).unwrap();
ProxyOrNotConnector::Proxy(proxy_connector)
}
_ => ProxyOrNotConnector::Not(http_connector),
};
let client = Config::default().connector(connector).build(&handle);
let g = client.get("http://127.0.0.1/".parse().unwrap());
let x = core.run(g).unwrap();
println!("{:?}", x);
}
I don't actually have a proxy lying around to test with, but it does compile and report a reasonable error about not being able to connect.
Building on #Shepmaster's answer, the code below has been tested using Privoxy.
We use the double if let clauses to first extract the type hidden by ProxyOrNotConnector and then modify the http request to use the proxy.
if let ProxyOrNotConnector::Proxy(x) = connector.clone() {
if let Some(headers) = x.http_headers(&uri) {
req.headers_mut().extend(headers.iter());
req.set_proxy(true);
}
}
Complete code:
extern crate futures;
extern crate hyper;
extern crate hyper_proxy;
extern crate tokio_core;
extern crate tokio_io;
use futures::{Future, Stream};
use hyper::{Chunk, Method, Request, Uri, client::{Config, HttpConnector, Service}};
use hyper_proxy::{Intercept, Proxy, ProxyConnector};
use std::io;
use tokio_core::reactor::Core;
use tokio_io::{AsyncRead, AsyncWrite};
trait AsyncRw: AsyncWrite + AsyncRead {}
impl<T> AsyncRw for T
where
T: AsyncWrite + AsyncRead,
{
}
#[derive(Clone)]
enum ProxyOrNotConnector {
Proxy(ProxyConnector<HttpConnector>),
Not(HttpConnector),
}
impl Service for ProxyOrNotConnector {
type Request = Uri;
type Response = Box<AsyncRw>;
type Error = io::Error;
type Future = Box<Future<Item = Self::Response, Error = Self::Error>>;
fn call(&self, req: Self::Request) -> Self::Future {
match self {
ProxyOrNotConnector::Proxy(p) => {
let x = p.call(req);
let y = x.map(|y| Box::new(y) as Box<AsyncRw>);
Box::new(y)
}
ProxyOrNotConnector::Not(n) => {
let x = n.call(req);
let y = x.map(|y| Box::new(y) as Box<AsyncRw>);
Box::new(y)
}
}
}
}
fn main() {
let proxy_uri = Some("http://localhost:8118");
let use_proxy = true;
let uri: Uri = "http://httpbin.org/ip".parse().unwrap();
let mut core = Core::new().unwrap();
let handle = core.handle();
let http_connector = HttpConnector::new(4, &handle);
let connector = match (proxy_uri, use_proxy) {
(Some(proxy_uri), true) => {
println!("Using proxy: {}", proxy_uri);
let proxy_uri = proxy_uri.parse().unwrap();
let proxy = Some(Proxy::new(Intercept::All, proxy_uri));
let proxy_connector =
ProxyConnector::from_proxy(http_connector, proxy.unwrap()).unwrap();
ProxyOrNotConnector::Proxy(proxy_connector)
}
_ => ProxyOrNotConnector::Not(http_connector),
};
let client = Config::default().connector(connector.clone()).build(&handle);
let mut req: hyper::Request;
match use_proxy {
true => {
req = Request::new(Method::Get, uri.clone());
if let ProxyOrNotConnector::Proxy(x) = connector.clone() {
if let Some(headers) = x.http_headers(&uri) {
req.headers_mut().extend(headers.iter());
req.set_proxy(true);
}
}
}
false => req = Request::new(Method::Get, uri.clone()),
}
let future_http = client
.request(req)
.and_then(|res| res.body().concat2())
.map(move |body: Chunk| ::std::str::from_utf8(&body).unwrap().to_string());
let x = core.run(future_http).unwrap();
println!("{:?}", x);
}
I am experiencing a bug where my Hyper HTTP response is being truncated to a specific size (7829 bytes). Making the same request with cURL works fine.
The request queries a JSON endpoint for data. The response struct is then shuffled around a lot, because a relatively complex rate-limiting procedure is used to make a number of these requests at once. However, if only one request is made, the response is still truncated.
Before implementing rate-limiting and doing some heavy refactoring, the program made these responses properly.
I made the minimal example below, but it fails to reproduce the problem. At this point I'm not sure where to look. The codebase is moderately complicated and iteratively expanding the reproduction example is difficult, especially when I don't know what might possibly cause this.
What are some ways that Hyper's Response body might get truncated? The response body is acquired as in the handle function below.
#![feature(use_nested_groups)]
extern crate futures;
extern crate hyper;
extern crate hyper_tls;
extern crate tokio_core;
use futures::{Future, Stream};
use hyper::{Body, Chunk, Client, Method, Request, Response};
use hyper_tls::HttpsConnector;
use tokio_core::reactor::Core;
use std::env;
fn main() {
let mut core = Core::new().unwrap();
let client = Client::configure()
.connector(HttpsConnector::new(4, &core.handle()).unwrap())
.build(&core.handle());
fn handle(response: Response<Body>) -> Box<Future<Item = usize, Error = hyper::Error>> {
Box::new(
response
.body()
.concat2()
.map(move |body: Chunk| -> usize { body.len() }),
)
}
let args: Vec<String> = env::args().collect();
let uri = &args[1];
let req = Request::new(Method::Get, uri.parse().unwrap());
let response_body_length = {
let future = Box::new(client.request(req).map(handle).flatten());
core.run(future).unwrap()
};
println!("response body length: {}", response_body_length);
}
Offending code:
extern crate serde;
extern crate serde_json;
use futures::{future, stream, Future, Stream};
use hyper;
use hyper::{client, Body, Chunk, Client, Headers, Method, Request, Response, header::Accept,
header::Date as DateHeader, header::RetryAfter};
use hyper_tls::HttpsConnector;
use tokio_core::reactor::Core;
use models::Bucket;
use std::thread;
use std::time::{Duration, UNIX_EPOCH};
use std::str;
header! { (XRateLimitRemaining, "x-ratelimit-remaining") => [String] }
#[derive(Debug)]
struct Uri(pub String);
const MAX_REQ_SIZE: u32 = 500;
fn make_uri(symbol: &str, page_ix: u32) -> Uri {
Uri(format!(
"https://www.bitmex.com/api/v1/trade/bucketed?\
symbol={symbol}&\
columns={columns}&\
partial=false&\
reverse=true&\
binSize={bin_size}&\
count={count}&\
start={start}",
symbol = symbol,
columns = "close,timestamp",
bin_size = "5m",
count = MAX_REQ_SIZE,
start = 0 + MAX_REQ_SIZE * page_ix
))
}
#[derive(Debug)]
struct RateLimitInfo {
remaining_reqs: u32,
retry_after: Option<Duration>,
}
impl RateLimitInfo {
fn default() -> RateLimitInfo {
RateLimitInfo {
remaining_reqs: 1,
retry_after: None,
}
}
fn from<T>(resp: &Response<T>) -> RateLimitInfo {
let headers = resp.headers();
let remaining_reqs = headers
.get::<XRateLimitRemaining>()
.unwrap_or_else(|| panic!("x-ratelimit-remaining not on request."))
.parse()
.unwrap();
let retry_after = match headers.get::<RetryAfter>() {
Some(RetryAfter::Delay(duration)) => Some(*duration),
_ => None,
};
RateLimitInfo {
remaining_reqs,
retry_after,
}
}
}
fn resp_dated_later<'a>(a: &'a Response<Body>, b: &'a Response<Body>) -> &'a Response<Body> {
let get_date = |resp: &Response<Body>| {
let headers: &Headers = resp.headers();
**headers.get::<DateHeader>().unwrap()
};
if get_date(&a) > get_date(&b) {
a
} else {
b
}
}
#[derive(Debug)]
struct Query {
uri: Uri,
response: Option<Response<Body>>,
}
impl Query {
fn from_uri(uri: Uri) -> Query {
Query {
uri: uri,
response: None,
}
}
}
fn query_good(q: &Query) -> bool {
match &q.response {
Some(response) => response.status().is_success(),
_ => false,
}
}
type HttpsClient = hyper::Client<HttpsConnector<client::HttpConnector>>;
type FutureQuery = Box<Future<Item = Query, Error = hyper::Error>>;
fn to_future(x: Query) -> FutureQuery {
Box::new(future::ok(x))
}
fn exec_if_needed(client: &HttpsClient, query: Query) -> FutureQuery {
fn exec(client: &HttpsClient, q: Query) -> FutureQuery {
println!("exec: {:?}", q);
let uri = q.uri;
let req = {
let mut req = Request::new(Method::Get, uri.0.parse().unwrap());
req.headers_mut().set(Accept::json());
req
};
Box::new(
client
.request(req)
.inspect(|resp| println!("HTTP {}", resp.status()))
.map(|resp| Query {
uri: uri,
response: Some(resp),
}),
)
}
if query_good(&query) {
to_future(query)
} else {
exec(client, query)
}
}
type BoxedFuture<T> = Box<Future<Item = T, Error = hyper::Error>>;
fn do_batch(client: &HttpsClient, queries: Vec<Query>) -> BoxedFuture<Vec<Query>> {
println!("do_batch() {} queries", queries.len());
let exec_if_needed = |q| exec_if_needed(client, q);
let futures = queries.into_iter().map(exec_if_needed);
println!("do_batch() futures {:?}", futures);
Box::new(
stream::futures_ordered(futures).collect(), //future::join_all(futures)
)
}
fn take<T>(right: &mut Vec<T>, suggested_n: usize) -> Vec<T> {
let n: usize = if right.len() < suggested_n {
right.len()
} else {
suggested_n
};
let left = right.drain(0..n);
return left.collect();
}
type BoxedResponses = Box<Vec<Response<Body>>>;
fn batched_throttle(uris: Vec<Uri>) -> BoxedResponses {
println!("batched_throttle({} uris)", uris.len());
let mut core = Core::new().unwrap();
let client = Client::configure()
.connector(HttpsConnector::new(4, &core.handle()).unwrap())
.build(&core.handle());
let mut rate_limit_info = RateLimitInfo::default();
let mut queries_right: Vec<Query> = uris.into_iter().map(Query::from_uri).collect();
loop {
let mut queries_left: Vec<Query> = Vec::with_capacity(queries_right.len());
println!("batched_throttle: starting inner loop");
loop {
// throttle program during testing
thread::sleep(Duration::from_millis(800));
println!("batched_throttle: {:?}", rate_limit_info);
if let Some(retry_after) = rate_limit_info.retry_after {
println!("batched_throttle: retrying after {:?}", retry_after);
thread::sleep(retry_after)
}
if queries_right.is_empty() {
break;
}
let mut queries_mid = {
let ri_count = rate_limit_info.remaining_reqs;
let iter_req_count = if ri_count == 0 { 1 } else { ri_count };
println!("batched_throttle: iter_req_count {}", iter_req_count);
take(&mut queries_right, iter_req_count as usize)
};
println!(
"batched_throttle: \
queries_right.len() {}, \
queries_left.len() {}, \
queries_mid.len() {})",
queries_right.len(),
queries_left.len(),
queries_mid.len()
);
if queries_mid.iter().all(query_good) {
println!("batched_throttle: queries_mid.iter().all(query_good)");
continue;
}
queries_mid = { core.run(do_batch(&client, queries_mid)).unwrap() };
rate_limit_info = {
let create_very_old_response =
|| Response::new().with_header(DateHeader(UNIX_EPOCH.into()));
let very_old_response = create_very_old_response();
let last_resp = queries_mid
.iter()
.map(|q| match &q.response {
Some(r) => r,
_ => panic!("Impossible"),
})
.fold(&very_old_response, resp_dated_later);
RateLimitInfo::from(&last_resp)
};
&queries_left.append(&mut queries_mid);
}
queries_right = queries_left;
if queries_right.iter().all(query_good) {
break;
}
}
println!(
"batched_throttle: finishing. queries_right.len() {}",
queries_right.len()
);
Box::new(
queries_right
.into_iter()
.map(|q| q.response.unwrap())
.collect(),
)
}
fn bucket_count_to_req_count(bucket_count: u32) -> u32 {
let needed_req_count = (bucket_count as f32 / MAX_REQ_SIZE as f32).ceil() as u32;
return needed_req_count;
}
type BoxedBuckets = Box<Vec<Bucket>>;
fn response_to_buckets(response: Response<Body>) -> BoxedFuture<Vec<Bucket>> {
Box::new(response.body().concat2().map(|body: Chunk| -> Vec<Bucket> {
println!("body.len(): {}", body.len());
println!("JSON: {}", str::from_utf8(&body).unwrap());
serde_json::from_slice(&body).unwrap()
}))
}
pub fn get_n_last(symbol: &str, bucket_count: u32) -> BoxedBuckets {
let req_count = bucket_count_to_req_count(bucket_count);
let uris = (0..req_count)
.map(|page_ix| make_uri(symbol, page_ix))
.collect();
let responses = batched_throttle(uris);
let mut core = Core::new().unwrap();
let boxed_buckets = {
let futures = responses.into_iter().map(response_to_buckets);
let future = stream::futures_ordered(futures).collect();
let groups_of_buckets = core.run(future).unwrap();
Box::new(
groups_of_buckets
.into_iter()
.flat_map(|bs| bs.into_iter())
.rev()
.collect(),
)
};
return boxed_buckets;
}
You first create a Core and start lots of requests and gather the Response "results".
After you got all the Responses you start a new Core and try to start reading the data from those Responses - but the server probably closed them long ago due to write timeouts, and you only get partial data.
You shouldn't keep the server waiting; start reading the Responses as soon as possible.
I make a client request inside an Iron handler. How can I reuse Tokio's Core and Hyper's Client? I'm using hyper 0.11.0 and tokio-core 0.1.
fn get_result(req: &mut Request) -> IronResult<Response> {
let mut payload = String::new();
req.body.read_to_string(&mut payload).unwrap();
// can we re-use core and client somehow. Making then global with lazy_static!() does not work.
let mut core = tokio_core::reactor::Core::new().unwrap();
let client = Client::new(&core.handle());
let uri = "http://host:port/getResult".parse().unwrap();
let mut req: hyper::Request = hyper::Request::new(hyper::Method::Post, uri);
req.headers_mut().set(ContentType::json());
req.headers_mut().set(ContentLength(payload.len() as u64));
req.set_body(payload);
let mut results: Vec<RequestFormat> = Vec::new();
let work = client.request(req).and_then(|res| {
res.body().for_each(|chunk| {
let re: ResultFormat = serde_json::from_slice(&chunk).unwrap();
results.push(re);
Ok(())
})
});
Ok(Response::with(
(iron::status::Ok, serde_json::to_string(&results).unwrap()),
))
}
I created a Downloader class that wraps client and core. Below is snippet.
use hyper;
use tokio_core;
use std::sync::{mpsc};
use std::thread;
use futures::Future;
use futures::stream::Stream;
use std::time::Duration;
use std::io::{self, Write};
use time::precise_time_ns;
use hyper::Client;
pub struct Downloader {
sender : mpsc::Sender<(hyper::Request, mpsc::Sender<hyper::Chunk>)>,
#[allow(dead_code)]
tr : thread::JoinHandle<hyper::Request>,
}
impl Downloader {
pub fn new() -> Downloader {
let (sender, receiver) = mpsc::channel::<(hyper::Request,mpsc::Sender<hyper::Chunk>)>();
let tr = thread::spawn(move||{
let mut core = tokio_core::reactor::Core::new().unwrap();
let client = Client::new(&core.handle());
loop {
let (req , sender) = receiver.recv().unwrap();
let begin = precise_time_ns();
let work = client.request(req)
.and_then(|res| {
res.body().for_each(|chunk| {
sender.send(chunk)
.map_err(|e|{
//io::sink().write(&chunk).unwrap();
io::Error::new(io::ErrorKind::Other, e)
})?;
Ok(())
})
//sender.close();
//res.body().concat2()
});
core.run(work).map_err(|e|{println!("Error Is {:?}", e);});
//This time prints same as all request processing time.
debug!("Time taken In Download {:?} ms", (precise_time_ns() - begin) / 1000000);
}
});
Downloader{sender,
tr,
}
}
pub fn download(&self, req : hyper::Request, results: mpsc::Sender<Vec<u8>>){
self.sender.send((req, results)).unwrap();
}
}
Now client of this class can have a static variable.
lazy_static!{
static ref DOWNLOADER : Mutex<downloader::Downloader> =
Mutex::new(downloader::Downloader::new());
}
let (sender, receiver) = mpsc::channel();
DOWNLOADER.lock().unwrap().download(payload, sender);
and then read through receive channel.
One may need to close sender channel using sender.drop()