I have this method:
fn parse_http_request_headers<'b>(sp: &'b mut std::str::Split<&str>) -> HashMap<String, String> {
let mut headers: HashMap<String, String> = HashMap::new();
let mut iter = sp.peekable();
while iter.peek() != None {
let next = iter.next();
if let Some(value_to_parse) = next {
let parts = value_to_parse.split(": ").collect::<Vec<&str>>();
let key = parts.get(0);
if parts.len() == 2 {
headers.insert(
(*key.expect(&format!("Error getting the header definition: {:?}", &key)))
.to_string(),
(*parts.get(1)
.expect(&format!("Error getting the header value from: {:?}", &parts)))
.to_string()
);
}
} else { iter.next(); }
}
headers
}
that I refactored for curiosity about playing with the Interior mutability pattern, to try to solve some doubts about struct's fields that contains references.
But, ok, let's go with the code:
fn parse_http_request_headers<'b>(sp: Rc<RefCell<&'b mut std::str::Split<&str>>>) -> HashMap<String, String> {
let mut headers: HashMap<String, String> = HashMap::new();
let mut borrow = sp.borrow_mut(); // let binding
let mut iter = borrow.peekable();
while iter.peek() != None {
let next = iter.next();
if let Some(value_to_parse) = next {
let parts = value_to_parse.split(": ").collect::<Vec<&str>>();
let key = parts.get(0);
if parts.len() == 2 {
headers.insert(
(*key.expect(&format!("Error getting the header definition: {:?}", &key)))
.to_string(),
(*parts.get(1)
.expect(&format!("Error getting the header value from: {:?}", &parts)))
.to_string()
);
}
} else { iter.next(); }
}
headers
}
But, for my surprise, there's no iterations over the peekable peeked: None.
Why is causing this?
The first version works perfect parsing the http headers, this is the output:
HttpRequest { verb: GET, uri: Uri { uri: "/" }, http_version: V1_1, headers: {"Sec-Fetch-Mode": "navigate", "Connection": "keep-alive", "DNT": "1", "Sec-Fetch-Site": "none", "Host": "127.0.0.1:7878", "Sec-GPC": "1", "Sec-Fetch-User": "?1", "Upgrade-Insecure-Requests": "1", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0", "Accept-Language": "en-US,en;q=0.5", "Accept-Encoding": "gzip, deflate, br", "Sec-Fetch-Dest": "document"}, body: None }
The calling code:
let rc: Rc<RefCell<&mut Split<&str>>> = Rc::new(RefCell::new(sp));
let (verb, uri, version) = Self::parse_verb_uri_version(Rc::clone(&rc));
let body = Self::parse_http_request_body(Rc::clone(&rc).borrow_mut().last().unwrap());
let headers = Self::parse_http_request_headers(Rc::clone(&rc));
where sp:
sp: &'a mut Split<&'a str>
Thanks.
You use .last() in the proceeding line which will consume all elements from the iterator leaving nothing behind to iterate in parse_http_request_headers().
Related
I send an http get request to a server and receive a response:
let resp = reqwest::blocking::get(req)?.text()?;
resp holds a String like this:
<?xml version=\"1.0\" encoding=\"UTF-8\">\n<Document xmlns=...
<datetime>202207102300</datetime>\n\t\t\t\t\t\t<value>320.08</value>\n\t\t\t\t\t<datetime>202207110000</datetime>\n\t\t\t\t\t\t<value>278.00</value>
...</Document>
What is the best way to get this text parsed into a vector containing tuple elements, as follows:
let mut tuple_items: (chrono::DateTime, f32)
This is my code that I created with the quickxml crate:
use chrono::NaiveDateTime;
use quick_xml::events::Event;
use quick_xml::Reader;
pub struct DatetimeValue {
pub dt: NaiveDateTime,
pub val: f32,
}
pub fn parse_xml_string(&self, xml_string: String) -> Vec<DatetimeValue> {
let mut response_vector: Vec<DatetimeValue> = vec![];
let mut reader = Reader::from_str(&xml_string[..]);
reader.trim_text(true);
let mut val_flag = false;
let mut dt_flag = false;
let mut buf = Vec::new();
let mut count = 0;
let mut actual_dt: NaiveDateTime;
loop {
match reader.read_event(&mut buf) {
Ok(Event::Start(ref e)) => {
if let b"value" = e.name() { val_flag = true }
else if let b"datetime" = e.name() { dt_flag = true }
}
Ok(Event::Text(e)) => {
if dt_flag {
actual_dt = NaiveDateTime::parse_from_str(e
.unescape_and_decode(&reader)
.unwrap(), "%Y%m%d%H%M").unwrap();
dt_flag = false;
}
else if val_flag {
response_vector.push(DatetimeValue {
dt: actual_dt,
val: e
.unescape_and_decode(&reader)
.unwrap()
.parse::<f32>()
.unwrap(),
});
val_flag = false;
}
}
Ok(Event::Eof) => break,
Err(e) => panic!("Error at position {}: {:?}", reader.buffer_position(), e),
_ => (),
}
buf.clear();
}
response_vector
}
I have created a gRPC server and client in Rust and it was working, but then I made changes to the .proto file, then updated all the type names etc accordingly but now I run into an "unimplemented" error.
The error occurs when checking that the response is okay, but the root cause must be something when actually send the request. The server never prints "got a request..."
I suspect it's something really stupid.
ERROR MESSAGE
Ok(Err(Status { code: Unimplemented, metadata: MetadataMap { headers: {"date": "Wed, 27 Jul 2022 09:20:55 GMT", "content-type": "application/grpc", "content-length": "0"} }, source: None }))
thread 'main' panicked at 'assertion failed: resp.unwrap().is_ok()', src/Rust/client.rs:52:9
PROTO BUFFER
syntax = "proto3";
package Labels;
service Labelling {
rpc storeLabel (StoreLabelRequest) returns (StoreLabelResponse);
rpc getLabel (GetLabelRequest) returns (GetLabelResponse);
}
message StoreLabelRequest{
string header = 1;
}
message StoreLabelResponse{
bool successful = 1;
string new_id = 2;
}
message GetLabelRequest{
string label_id = 1;
}
message GetLabelResponse{
string header = 1;
}
SERVER
pub mod documents{
tonic::include_proto!("labels");
}
#[derive(Debug, Default)]
pub struct LabelService {}
#[tonic::async_trait]
impl Labelling for LabelService {
async fn store_label(
&self,
request: Request<StoreLabelRequest>,
) -> Result<Response<StoreLabelResponse>, Status> {
println!("Got a request: {:?}", request);
let req = request.into_inner();
let reply = StoreLabelResponse {
successful: true,
new_id: format!("Document received with header: {}", req.header).into()
};
Ok(Response::new(reply))
}
async fn get_label(
&self,
request: tonic::Request<GetLabelRequest>,
) -> Result<tonic::Response<GetLabelResponse>, tonic::Status>{
let req = request.into_inner();
let reply = GetLabelResponse {
header: format!("Document received with header: {}", req.label_id).into()
};
Ok(Response::new(reply))
}
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let args: Vec<String> = env::args().collect();
let addr: SocketAddr = args[1].parse()?;
let service = LabelService::default();
Server::builder()
.add_service(LabellingServer::new(service))
.serve(addr)
.await?;
Ok(())
}
CLIENT
pub mod documents{
tonic::include_proto!("label");
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let args: Vec<String> = env::args().collect();
let message_count: usize = args[1].parse().unwrap();
let address: String = args[2].parse().unwrap();
let full_address: String = "http://".to_string() + address.as_str();
let client = LabellingClient::connect(full_address).await?;
let mut responses = Vec::with_capacity(message_count);
for _ in 0..message_count {
let mut client = client.clone();
let header = "Hello"; // This will be replaced with a file reader but is simplified for the question
let request = tonic::Request::new(StoreLabelRequest {
header: header.to_string()});
responses.push(spawn(async move {
client.store_label(request).await
}));
}
for resp in responses {
let resp = resp.await;
assert!(resp.is_ok());
assert!(resp.unwrap().is_ok()); // <-- This is where the Panic occurs
}
Ok(())
}
As pointed out, the included protos were not lined up, which is not caught at compile-time.
I am trying to extract the factory closures into it's own function.
So instead of this
let server = HttpServer::new(|| App::new().wrap(Logger::default()))
.bind("127.0.0.1:8080")?
.run();
I'd like to move App::new()... into a new function called new_app()
let server = HttpServer::new(|| new_app())
.bind("127.0.0.1:8080")?.run();
// todo
fn new_app() { todo!() }
I was unable to use Clion IDE or VSCode do it automatically as they are unable to figure out the return type of App::new().wrap()..
However I figured the return type is something like this
pub fn new_app() -> App<
impl ServiceFactory<
ServiceRequest,
Config = (),
Response = ServiceResponse<
actix_web::middleware::logger::StreamLog<actix_web::body::AnyBody>,
>,
Error = Error,
InitError = (),
>,
actix_web::middleware::logger::StreamLog<actix_web::body::AnyBody>,
> {
App::new().wrap(Logger::default())
}
However this can not be right because the module actix_web::middleware::logger is private.
So I tried with a more "generic" type of the generic type parameters,
pub fn new_app() -> App<
impl ServiceFactory<ServiceRequest>,
impl MessageBody,
> {
App::new().wrap(Logger::default())
}
However this also doesn't compile with this error
error[E0277]: the trait bound `App<impl ServiceFactory<ServiceRequest>, impl MessageBody>: actix_service::IntoServiceFactory<_, Request>` is not satisfied
--> src/app.rs:79:78
|
79 | let server = HttpServer::new(|| new_app_2()).bind("127.0.0.1:8080")?.run();
| ^^^ the trait `actix_service::IntoServiceFactory<_, Request>` is not implemented for `App<impl ServiceFactory<ServiceRequest>, impl MessageBody>`
|
= help: the following implementations were found:
<App<T, B> as actix_service::IntoServiceFactory<actix_web::app_service::AppInit<T, B>, Request>>
It seems like a trivial work to extract some code into a function but I am not sure how to fix this. Would you help me?
I solved similar issue by implementing custom logging middleware.
Then we can use actix_web::dev::Body instead of actix_web::middleware::logger::StreamLog<actix_web::body::AnyBody>.
use actix_service::ServiceFactory;
fn app() -> App<
impl ServiceFactory<
Request = actix_web::dev::ServiceRequest,
Config = (),
Response = actix_web::dev::ServiceResponse,
Error = actix_web::Error,
InitError = (),
>,
actix_web::dev::Body, // <- This
> {
App::new()
.wrap(middlewares::logging::Logging)
.service(index)
}
As a side note,
You can implement middleware referring official examples https://github.com/actix/examples/tree/master/basics/middleware/src or my code below that includes request & response log.
#![allow(unused_imports)]
use std::pin::Pin;
use std::task::{Context, Poll};
use std::rc::Rc;
use std::cell::RefCell;
use futures::{future::{ok, Future, Ready}, stream::StreamExt};
use std::collections::HashMap;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use actix_service::{Service, Transform};
use actix_web::{
dev::{ServiceRequest, ServiceResponse},
web::{Bytes, BytesMut},
web,
body::{Body, BodySize, MessageBody, ResponseBody},
http::Version,
Error, HttpMessage,
};
use actix_http::h1::{Payload};
use chrono::{Utc, DateTime, Date, NaiveDateTime};
// There are two steps in middleware processing.
// 1. Middleware initialization, middleware factory gets called with
// next service in chain as parameter.
// 2. Middleware's call method gets called with normal request.
pub struct Logging;
// Middleware factory is `Transform` trait from actix-service crate
// `S` - type of the next service
// `B` - type of response's body
impl<S, B> Transform<S> for Logging
where
S: Service<Request = ServiceRequest, Response = ServiceResponse<B>, Error = Error> + 'static,
S::Future: 'static,
B: MessageBody + Unpin + 'static,
{
type Request = ServiceRequest;
type Response = ServiceResponse<B>;
type Error = Error;
type InitError = ();
type Transform = LoggingMiddleware<S>;
type Future = Ready<Result<Self::Transform, Self::InitError>>;
fn new_transform(&self, service: S) -> Self::Future {
ok(LoggingMiddleware {
service: Rc::new(RefCell::new(service))
})
}
}
pub struct LoggingMiddleware<S> {
service: Rc<RefCell<S>>,
}
impl<S, B> Service for LoggingMiddleware<S>
where
S: Service<Request = ServiceRequest, Response = ServiceResponse<B>, Error = Error> + 'static,
S::Future: 'static,
B: MessageBody + Unpin + 'static,
{
type Request = ServiceRequest;
type Response = ServiceResponse<B>;
type Error = Error;
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>>>>;
fn poll_ready(&mut self, cx: &mut Context) -> Poll<Result<(), Self::Error>> {
self.service.poll_ready(cx)
}
fn call(&mut self, mut req: ServiceRequest) -> Self::Future {
let mut svc = self.service.clone();
Box::pin(async move {
let begin = Utc::now();
let path = req.path().to_string();
let method = req.method().as_str().to_string();
let queries = req.query_string().to_string();
let ip = req.head().peer_addr;
let protocol = match req.version() {
Version::HTTP_09 => "http/0.9",
Version::HTTP_10 => "http/1.0",
Version::HTTP_11 => "http/1.1",
Version::HTTP_2 => "http/2.0",
Version::HTTP_3 => "http/3.0",
_ => "UNKNOWN",
};
// Request headers
let mut domain = String::new();
let mut user_agent = String::new();
let mut headers: HashMap<&str, &str> = HashMap::new();
for (k, v) in req.headers().iter() {
if let Ok(inner) = v.to_str() {
let key = k.as_str();
headers.insert(key, inner);
match key {
"host" => { domain = inner.to_string() },
"user-agent" => { user_agent = inner.to_string() },
_ => {},
}
}
}
let req_headers = json!(headers).to_string();
let req_body = get_request_body(&mut req).await;
let mut parsed = parse_body(req_body.unwrap());
let req_body: Option<String> = if ! parsed.is_object() {
None
} else {
/*
// my code
// Mask some words for security, like `password`
for k in vec!["password"] {
let obj = parsed.as_object_mut().unwrap();
if let Some(p) = obj.get_mut(k) {
*p = json!("MASKED_FOR_SECURITY");
}
}
*/
Some(parsed.to_string())
};
// DbPool
/*
// my code
let pool = req.app_data::<web::Data<DbPool>>().map(|p| p.clone());
*/
// Exec main function and wait response generated
let mut res = svc.call(req).await?;
let duration = (Utc::now() - begin).num_microseconds();
let status_code = res.status();
// Response headers
let mut headers: HashMap<&str, &str> = HashMap::new();
for (k, v) in res.headers().iter() {
if let Ok(inner) = v.to_str() {
headers.insert(k.as_str(), inner);
}
}
let res_headers = json!(headers).to_string();
// Get response body
let mut res_body = BytesMut::new();
let mut stream = res.take_body();
while let Some(chunk) = stream.next().await {
res_body.extend_from_slice(&chunk?);
}
// Logging
println!("req.domain : {:?}", domain);
println!("req.user_agent : {:?}", user_agent);
println!("req.ip : {:?}", ip);
println!("req.path : {:?}", path);
println!("req.method : {:?}", method);
println!("req.headers: {:?}", req_headers);
println!("req.query : {:?}", queries);
println!("req.body : {:?}", req_body);
println!("duration : {:?}", duration);
println!("res.status : {:?}", status_code);
println!("res.headers: {:?}", res_headers);
println!("res.body : {:?}", res_body);
/*
// my code
let a = AccessLog {
id: None,
protocol: Some(protocol.to_string()).into_iter().find(|v| v != ""),
domain: Some(domain).into_iter().find(|v| v != ""),
ip: ip.map(|inner| inner.to_string()).into_iter().find(|v| v != ""),
method: Some(method).into_iter().find(|v| v != ""),
path: Some(path.to_string()).into_iter().find(|v| v != ""),
query: Some(queries).into_iter().find(|v| v != ""),
user_agent: Some(user_agent),
req_headers: Some(req_headers).into_iter().find(|v| v != ""),
req_body: req_body,
duration: duration.map(|inner| inner as i32),
status_code: Some(status_code.as_u16() as i32),
res_headers: Some(res_headers).into_iter().find(|v| v != ""),
res_body: String::from_utf8(res_body.clone().to_vec()).into_iter().find(|v| v != ""),
others: None,
requested_at: Some(begin.with_timezone(&*TIMEZONE).naive_local()),
created_at: None,
};
if let Some(pool) = pool {
if let Ok(conn) = pool.get() {
if let Err(e) = a.create(&conn) {
eprintln!("database err: {:?}", e);
};
}
}
*/
// return original response body
Ok(res.map_body(|_, _b| ResponseBody::Other(Body::from(res_body))))
})
}
}
#[pin_project::pin_project(PinnedDrop)]
pub struct BodyLogger<B> {
#[pin]
body: ResponseBody<B>,
body_accum: BytesMut,
}
#[pin_project::pinned_drop]
impl<B> PinnedDrop for BodyLogger<B> {
fn drop(self: Pin<&mut Self>) {
println!("response body: {:?}", self.body_accum);
}
}
impl<B: MessageBody> MessageBody for BodyLogger<B> {
fn size(&self) -> BodySize {
self.body.size()
}
fn poll_next(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
) -> Poll<Option<Result<Bytes, Error>>> {
let this = self.project();
match this.body.poll_next(cx) {
Poll::Ready(Some(Ok(chunk))) => {
this.body_accum.extend_from_slice(&chunk);
Poll::Ready(Some(Ok(chunk)))
}
Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))),
Poll::Ready(None) => Poll::Ready(None),
Poll::Pending => Poll::Pending,
}
}
}
async fn get_request_body(req: &mut ServiceRequest) -> Result<BytesMut, Error> {
// Get body as bytes
let mut bytes: BytesMut = BytesMut::new();
let mut body = req.take_payload();
while let Some(chunk) = body.next().await {
bytes.extend_from_slice(&chunk?);
}
// Set body again
let (_, mut payload) = Payload::create(true);
payload.unread_data(web::Bytes::from(bytes.clone()));
req.set_payload(payload.into());
Ok(bytes)
}
#[derive(Debug, Deserialize, Serialize)]
struct Password {
password: String,
}
fn parse_body(body: BytesMut) -> Value {
let json_parsed = serde_json::from_slice::<Value>(&body);
if let Ok(b) = json_parsed {
return b
}
// let query_parsed = serde_qs::from_bytes::<Password>(&body);
json!(null)
}
Here is my simplified code. I need to change the pdp_state in a function. but the state remain 'a'. I don't figure out why cloning Rc does not work. I also checked this cloning out side a structure an it worked.
#[derive(Clone,Copy)]
enum PDPStatus{
a,
b
}
struct network{
pdp_state:Rc<RefCell<PDPStatus>>,
}
impl network{
fn set(&mut self){
let mut t = *self.pdp_state.clone().borrow_mut();
match t {
a => {let m1 = self.pdp_state.clone();
let mut a = (*m1).borrow_mut() ;
*a = PDPStatus::b;
println!("a");},
b=> {let m1 = self.pdp_state.clone();m1.replace( PDPStatus::a);
println!("b");},
};
}
}
fn main() {
let mut network1 = network::new();
network1.set();
network1.set();
network1.set();
network1.set();
}
Update:
My set function would look like this. I need two closure that have access to pdp_state. I pass these closures as callbacks. I am sure the these closure wouldn't call together.
fn set(&mut self){
let borrowed_pdp_status = self.pdp_state.borrow().clone();
match borrowed_pdp_status {
PDPStatus::a => {
let mut state = self.pdp_state.clone();
let mut closuree = || state = Rc::new(RefCell::new(PDPStatus::b));
let mut state1 = self.pdp_state.clone();
let mut closuree1 = || state1 = Rc::new(RefCell::new(PDPStatus::b));
closuree();
closuree1();
println!("a");
},
PDPStatus::b => {
let mut closuree = || self.pdp_state = Rc::new(RefCell::new(PDPStatus::a));
closuree();
println!("b");
},
};
}
In the set method, you need to borrow self.pdp_state and then clone() it in a variable, and then match the variable where you cloned it.
Replace the set method with this:
fn set(&mut self) {
let borrowed_pdp_status = self.pdp_state.borrow().clone();
match borrowed_pdp_status {
PDPStatus::a => {
self.pdp_state = Rc::new(RefCell::new(PDPStatus::b));
println!("a");
},
PDPStatus::b => {
self.pdp_state = Rc::new(RefCell::new(PDPStatus::a));
println!("b");
},
};
}
Playground link - https://play.rust-lang.org/?version=stable&mode=release&edition=2018&gist=426d4cb7db9a92ee4ddcd4f36dbc12f7
This answer was posted after the question was updated:
EDIT
You can use the replace() method from RefCell
fn set(&mut self) {
let borrowed_pdp_status = self.pdp_state.borrow().clone();
match borrowed_pdp_status {
PDPStatus::a => {
let mut closuree = || {
self.pdp_state.replace(PDPStatus::b);
};
let mut closuree1 = || {
self.pdp_state.replace(PDPStatus::b);
};
closuree();
closuree1();
println!("a");
}
PDPStatus::b => {
let mut closuree = || {
self.pdp_state.replace(PDPStatus::a);
};
closuree();
println!("b");
}
};
}
Playground link - https://play.rust-lang.org/?version=stable&mode=release&edition=2018&gist=4af02228d58b2f2c865a525e3f70d6a0
OLD
You could just make the closures take &mut self.pdp_state as an argument, and then update it.
fn set(&mut self) {
let borrowed_pdp_status = self.pdp_state.borrow().clone();
match borrowed_pdp_status {
PDPStatus::a => {
let mut closuree = |local_pdp_state: &mut Rc<RefCell<PDPStatus>>| {
*local_pdp_state = Rc::new(RefCell::new(PDPStatus::b))
};
let mut closuree1 = |local_pdp_state: &mut Rc<RefCell<PDPStatus>>| {
*local_pdp_state = Rc::new(RefCell::new(PDPStatus::b))
};
closuree(&mut self.pdp_state);
closuree1(&mut self.pdp_state);
println!("a");
}
PDPStatus::b => {
let mut closuree = |local_pdp_state: &mut Rc<RefCell<PDPStatus>>| {
*local_pdp_state = Rc::new(RefCell::new(PDPStatus::a))
};
closuree(&mut self.pdp_state);
println!("b");
}
};
}
Playground link - https://play.rust-lang.org/?version=stable&mode=release&edition=2018&gist=4af96385b0446082afdb7d615bb8eecb
I am experiencing a bug where my Hyper HTTP response is being truncated to a specific size (7829 bytes). Making the same request with cURL works fine.
The request queries a JSON endpoint for data. The response struct is then shuffled around a lot, because a relatively complex rate-limiting procedure is used to make a number of these requests at once. However, if only one request is made, the response is still truncated.
Before implementing rate-limiting and doing some heavy refactoring, the program made these responses properly.
I made the minimal example below, but it fails to reproduce the problem. At this point I'm not sure where to look. The codebase is moderately complicated and iteratively expanding the reproduction example is difficult, especially when I don't know what might possibly cause this.
What are some ways that Hyper's Response body might get truncated? The response body is acquired as in the handle function below.
#![feature(use_nested_groups)]
extern crate futures;
extern crate hyper;
extern crate hyper_tls;
extern crate tokio_core;
use futures::{Future, Stream};
use hyper::{Body, Chunk, Client, Method, Request, Response};
use hyper_tls::HttpsConnector;
use tokio_core::reactor::Core;
use std::env;
fn main() {
let mut core = Core::new().unwrap();
let client = Client::configure()
.connector(HttpsConnector::new(4, &core.handle()).unwrap())
.build(&core.handle());
fn handle(response: Response<Body>) -> Box<Future<Item = usize, Error = hyper::Error>> {
Box::new(
response
.body()
.concat2()
.map(move |body: Chunk| -> usize { body.len() }),
)
}
let args: Vec<String> = env::args().collect();
let uri = &args[1];
let req = Request::new(Method::Get, uri.parse().unwrap());
let response_body_length = {
let future = Box::new(client.request(req).map(handle).flatten());
core.run(future).unwrap()
};
println!("response body length: {}", response_body_length);
}
Offending code:
extern crate serde;
extern crate serde_json;
use futures::{future, stream, Future, Stream};
use hyper;
use hyper::{client, Body, Chunk, Client, Headers, Method, Request, Response, header::Accept,
header::Date as DateHeader, header::RetryAfter};
use hyper_tls::HttpsConnector;
use tokio_core::reactor::Core;
use models::Bucket;
use std::thread;
use std::time::{Duration, UNIX_EPOCH};
use std::str;
header! { (XRateLimitRemaining, "x-ratelimit-remaining") => [String] }
#[derive(Debug)]
struct Uri(pub String);
const MAX_REQ_SIZE: u32 = 500;
fn make_uri(symbol: &str, page_ix: u32) -> Uri {
Uri(format!(
"https://www.bitmex.com/api/v1/trade/bucketed?\
symbol={symbol}&\
columns={columns}&\
partial=false&\
reverse=true&\
binSize={bin_size}&\
count={count}&\
start={start}",
symbol = symbol,
columns = "close,timestamp",
bin_size = "5m",
count = MAX_REQ_SIZE,
start = 0 + MAX_REQ_SIZE * page_ix
))
}
#[derive(Debug)]
struct RateLimitInfo {
remaining_reqs: u32,
retry_after: Option<Duration>,
}
impl RateLimitInfo {
fn default() -> RateLimitInfo {
RateLimitInfo {
remaining_reqs: 1,
retry_after: None,
}
}
fn from<T>(resp: &Response<T>) -> RateLimitInfo {
let headers = resp.headers();
let remaining_reqs = headers
.get::<XRateLimitRemaining>()
.unwrap_or_else(|| panic!("x-ratelimit-remaining not on request."))
.parse()
.unwrap();
let retry_after = match headers.get::<RetryAfter>() {
Some(RetryAfter::Delay(duration)) => Some(*duration),
_ => None,
};
RateLimitInfo {
remaining_reqs,
retry_after,
}
}
}
fn resp_dated_later<'a>(a: &'a Response<Body>, b: &'a Response<Body>) -> &'a Response<Body> {
let get_date = |resp: &Response<Body>| {
let headers: &Headers = resp.headers();
**headers.get::<DateHeader>().unwrap()
};
if get_date(&a) > get_date(&b) {
a
} else {
b
}
}
#[derive(Debug)]
struct Query {
uri: Uri,
response: Option<Response<Body>>,
}
impl Query {
fn from_uri(uri: Uri) -> Query {
Query {
uri: uri,
response: None,
}
}
}
fn query_good(q: &Query) -> bool {
match &q.response {
Some(response) => response.status().is_success(),
_ => false,
}
}
type HttpsClient = hyper::Client<HttpsConnector<client::HttpConnector>>;
type FutureQuery = Box<Future<Item = Query, Error = hyper::Error>>;
fn to_future(x: Query) -> FutureQuery {
Box::new(future::ok(x))
}
fn exec_if_needed(client: &HttpsClient, query: Query) -> FutureQuery {
fn exec(client: &HttpsClient, q: Query) -> FutureQuery {
println!("exec: {:?}", q);
let uri = q.uri;
let req = {
let mut req = Request::new(Method::Get, uri.0.parse().unwrap());
req.headers_mut().set(Accept::json());
req
};
Box::new(
client
.request(req)
.inspect(|resp| println!("HTTP {}", resp.status()))
.map(|resp| Query {
uri: uri,
response: Some(resp),
}),
)
}
if query_good(&query) {
to_future(query)
} else {
exec(client, query)
}
}
type BoxedFuture<T> = Box<Future<Item = T, Error = hyper::Error>>;
fn do_batch(client: &HttpsClient, queries: Vec<Query>) -> BoxedFuture<Vec<Query>> {
println!("do_batch() {} queries", queries.len());
let exec_if_needed = |q| exec_if_needed(client, q);
let futures = queries.into_iter().map(exec_if_needed);
println!("do_batch() futures {:?}", futures);
Box::new(
stream::futures_ordered(futures).collect(), //future::join_all(futures)
)
}
fn take<T>(right: &mut Vec<T>, suggested_n: usize) -> Vec<T> {
let n: usize = if right.len() < suggested_n {
right.len()
} else {
suggested_n
};
let left = right.drain(0..n);
return left.collect();
}
type BoxedResponses = Box<Vec<Response<Body>>>;
fn batched_throttle(uris: Vec<Uri>) -> BoxedResponses {
println!("batched_throttle({} uris)", uris.len());
let mut core = Core::new().unwrap();
let client = Client::configure()
.connector(HttpsConnector::new(4, &core.handle()).unwrap())
.build(&core.handle());
let mut rate_limit_info = RateLimitInfo::default();
let mut queries_right: Vec<Query> = uris.into_iter().map(Query::from_uri).collect();
loop {
let mut queries_left: Vec<Query> = Vec::with_capacity(queries_right.len());
println!("batched_throttle: starting inner loop");
loop {
// throttle program during testing
thread::sleep(Duration::from_millis(800));
println!("batched_throttle: {:?}", rate_limit_info);
if let Some(retry_after) = rate_limit_info.retry_after {
println!("batched_throttle: retrying after {:?}", retry_after);
thread::sleep(retry_after)
}
if queries_right.is_empty() {
break;
}
let mut queries_mid = {
let ri_count = rate_limit_info.remaining_reqs;
let iter_req_count = if ri_count == 0 { 1 } else { ri_count };
println!("batched_throttle: iter_req_count {}", iter_req_count);
take(&mut queries_right, iter_req_count as usize)
};
println!(
"batched_throttle: \
queries_right.len() {}, \
queries_left.len() {}, \
queries_mid.len() {})",
queries_right.len(),
queries_left.len(),
queries_mid.len()
);
if queries_mid.iter().all(query_good) {
println!("batched_throttle: queries_mid.iter().all(query_good)");
continue;
}
queries_mid = { core.run(do_batch(&client, queries_mid)).unwrap() };
rate_limit_info = {
let create_very_old_response =
|| Response::new().with_header(DateHeader(UNIX_EPOCH.into()));
let very_old_response = create_very_old_response();
let last_resp = queries_mid
.iter()
.map(|q| match &q.response {
Some(r) => r,
_ => panic!("Impossible"),
})
.fold(&very_old_response, resp_dated_later);
RateLimitInfo::from(&last_resp)
};
&queries_left.append(&mut queries_mid);
}
queries_right = queries_left;
if queries_right.iter().all(query_good) {
break;
}
}
println!(
"batched_throttle: finishing. queries_right.len() {}",
queries_right.len()
);
Box::new(
queries_right
.into_iter()
.map(|q| q.response.unwrap())
.collect(),
)
}
fn bucket_count_to_req_count(bucket_count: u32) -> u32 {
let needed_req_count = (bucket_count as f32 / MAX_REQ_SIZE as f32).ceil() as u32;
return needed_req_count;
}
type BoxedBuckets = Box<Vec<Bucket>>;
fn response_to_buckets(response: Response<Body>) -> BoxedFuture<Vec<Bucket>> {
Box::new(response.body().concat2().map(|body: Chunk| -> Vec<Bucket> {
println!("body.len(): {}", body.len());
println!("JSON: {}", str::from_utf8(&body).unwrap());
serde_json::from_slice(&body).unwrap()
}))
}
pub fn get_n_last(symbol: &str, bucket_count: u32) -> BoxedBuckets {
let req_count = bucket_count_to_req_count(bucket_count);
let uris = (0..req_count)
.map(|page_ix| make_uri(symbol, page_ix))
.collect();
let responses = batched_throttle(uris);
let mut core = Core::new().unwrap();
let boxed_buckets = {
let futures = responses.into_iter().map(response_to_buckets);
let future = stream::futures_ordered(futures).collect();
let groups_of_buckets = core.run(future).unwrap();
Box::new(
groups_of_buckets
.into_iter()
.flat_map(|bs| bs.into_iter())
.rev()
.collect(),
)
};
return boxed_buckets;
}
You first create a Core and start lots of requests and gather the Response "results".
After you got all the Responses you start a new Core and try to start reading the data from those Responses - but the server probably closed them long ago due to write timeouts, and you only get partial data.
You shouldn't keep the server waiting; start reading the Responses as soon as possible.