color_quant::NeuQuant compiled to WebAssembly outputs zero values - rust

I am trying to load an image in the browser and use the NewQuant algorithm to quantize my image buffer it in Rust via WebAssembly. However, the NewQuant output contains zero values, regardless of what PNG I try to feed it.
I expose two Rust methods to WASM:
alloc for allocating a byte buffer
read_img which will read and process the img buffer
I know that I get zero values because I imported a JavaScript method called log_nr for logging simple u8 numbers. The buffer seems to contain valid pixel values.
extern crate color_quant;
extern crate image;
use color_quant::NeuQuant;
use image::{DynamicImage, GenericImage, Pixel, Rgb};
use std::collections::BTreeMap;
use std::mem;
use std::os::raw::c_void;
static NQ_SAMPLE_FACTION: i32 = 10;
static NQ_PALETTE_SIZE: usize = 256;
extern "C" {
fn log(s: &str, len: usize);
fn log_nr(nr: u8);
}
fn get_pixels(img: DynamicImage) -> Vec<u8> {
let mut pixels = Vec::new();
for (_, _, px) in img.pixels() {
let rgba = px.to_rgba();
for channel in px.channels() {
pixels.push(*channel);
}
}
pixels
}
#[no_mangle]
pub extern "C" fn alloc(size: usize) -> *mut c_void {
let mut buf = Vec::with_capacity(size);
let ptr = buf.as_mut_ptr();
mem::forget(buf);
return ptr as *mut c_void;
}
fn process_img(img: DynamicImage) {
let pixels: Vec<u8> = get_pixels(img);
let quantized = NeuQuant::new(NQ_SAMPLE_FACTION, NQ_PALETTE_SIZE, &pixels);
let q = quantized.color_map_rgb();
for c in &q {
unsafe {
log_nr(*c);
}
}
}
#[no_mangle]
pub extern "C" fn read_img(buff_ptr: *mut u8, buff_len: usize) {
let mut img: Vec<u8> = unsafe { Vec::from_raw_parts(buff_ptr, buff_len, buff_len) };
return match image::load_from_memory(&img) {
Ok(img) => {
process_img(img);
}
Err(err) => {
let err_msg: String = err.to_string().to_owned();
let mut ns: String = "[load_from_memory] ".to_owned();
ns.push_str(&err_msg);
unsafe {
log(&ns, ns.len());
}
}
};
}
fn main() {
println!("Hello from rust 2");
}
The JavaScript code is the following:
run('sample.png');
function run(img) {
return compile().then(m => {
return loadImgIntoMem(img, m.instance.exports.memory, m.instance.exports.alloc).then(r => {
return m.instance.exports.read_img(r.imgPtr, r.len);
});
})
}
function compile(wasmFile = 'distil_wasm.gc.wasm') {
return fetch(wasmFile)
.then(r => r.arrayBuffer())
.then(r => {
let module = new WebAssembly.Module(r);
let importObject = {}
for (let imp of WebAssembly.Module.imports(module)) {
if (typeof importObject[imp.module] === "undefined")
importObject[imp.module] = {};
switch (imp.kind) {
case "function": importObject[imp.module][imp.name] = () => {}; break;
case "table": importObject[imp.module][imp.name] = new WebAssembly.Table({ initial: 256, maximum: 256, element: "anyfunc" }); break;
case "memory": importObject[imp.module][imp.name] = new WebAssembly.Memory({ initial: 256 }); break;
case "global": importObject[imp.module][imp.name] = 0; break;
}
}
importObject.env = Object.assign({}, importObject.env, {
log: (ptr, len) => console.log(ptrToStr(ptr, len)),
log_nr: (nr) => console.log(nr),
});
return WebAssembly.instantiate(r, importObject);
});
}
function loadImgIntoMemEmscripten(img) {
return new Promise(resolve => {
fetch(img)
.then(r => r.arrayBuffer())
.then(buff => {
const imgPtr = Module._malloc(buff.byteLength);
const imgHeap = new Uint8Array(Module.HEAPU8.buffer, imgPtr, buff.byteLength);
imgHeap.set(new Uint8Array(buff));
resolve({ imgPtr });
});
});
}

Related

How to add partition in kafka rust configuratgion

I want to config this file to add a number of partition option here as by default it is creating only 1 partition , but I need 10 for my data .
I dont have much knowledge of rdkafka library in rust , as I am directly using this plugin file
Can anyone guide me where can I find solution to this or what direction .
Thanks
use rdkafka::error::{KafkaError};
use rdkafka::{ClientConfig};
use rdkafka::producer::{FutureProducer, FutureRecord};
use std::fmt::Error;
use std::os::raw::{c_char, c_int, c_void};
use std::sync::mpsc::TrySendError;
use suricata::conf::ConfNode;
use suricata::{SCLogError, SCLogNotice};
const DEFAULT_BUFFER_SIZE: &str = "65535";
const DEFAULT_CLIENT_ID: &str = "rdkafka";
#[derive(Debug, Clone)]
struct ProducerConfig {
brokers: String,
topic: String,
client_id: String,
buffer: usize,
}
impl ProducerConfig {
fn new(conf: &ConfNode) -> Result<Self,Error> {
let brokers = if let Some(val) = conf.get_child_value("brokers"){
val.to_string()
}else {
SCLogError!("brokers parameter required!");
panic!();
};
let topic = if let Some(val) = conf.get_child_value("topic"){
val.to_string()
}else {
SCLogError!("topic parameter required!");
panic!();
};
let client_id = conf.get_child_value("client-id").unwrap_or(DEFAULT_CLIENT_ID);
let buffer_size = match conf
.get_child_value("buffer-size")
.unwrap_or(DEFAULT_BUFFER_SIZE)
.parse::<usize>()
{
Ok(size) => size,
Err(_) => {
SCLogError!("invalid buffer-size!");
panic!();
},
};
let config = ProducerConfig {
brokers: brokers.into(),
topic: topic.into(),
client_id: client_id.into(),
buffer: buffer_size,
};
Ok(config)
}
}
struct KafkaProducer {
producer: FutureProducer,
config: ProducerConfig,
rx: std::sync::mpsc::Receiver<String>,
count: usize,
}
impl KafkaProducer {
fn new(
config: ProducerConfig,
rx: std::sync::mpsc::Receiver<String>,
) -> Result<Self,KafkaError> {
let producer: FutureProducer = ClientConfig::new()
.set("bootstrap.servers", &config.brokers)
.set("client.id",&config.client_id)
.set("message.timeout.ms", "5000")
.create()?;
Ok(Self {
config,
producer,
rx,
count: 0,
})
}
fn run(&mut self) {
// Get a peekable iterator from the incoming channel. This allows us to
// get the next message from the channel without removing it, we can
// then remove it once its been sent to the server without error.
//
// Not sure how this will work with pipe-lining tho, will probably have
// to do some buffering here, or just accept that any log records
// in-flight will be lost.
let mut iter = self.rx.iter().peekable();
loop {
if let Some(buf) = iter.peek() {
self.count += 1;
if let Err(err) = self.producer.send_result(
FutureRecord::to(&self.config.topic)
.key("")
.payload(&buf),
) {
SCLogError!("Failed to send event to Kafka: {:?}", err);
break;
} else {
// Successfully sent. Pop it off the channel.
let _ = iter.next();
}
} else {
break;
}
}
SCLogNotice!("Producer finished: count={}", self.count,);
}
}
struct Context {
tx: std::sync::mpsc::SyncSender<String>,
count: usize,
dropped: usize,
}
unsafe extern "C" fn output_open(conf: *const c_void, init_data: *mut *mut c_void) -> c_int {
// Load configuration.
let config = ProducerConfig::new(&ConfNode::wrap(conf)).unwrap();
let (tx, rx) = std::sync::mpsc::sync_channel(config.buffer);
let mut kafka_producer = match KafkaProducer::new(config, rx) {
Ok(producer) => {
SCLogNotice!(
"KafKa Producer initialize success with brokers:{:?} | topic: {:?} | client_id: {:?} | buffer-size: {:?}",
producer.config.brokers,
producer.config.topic,
producer.config.client_id,
producer.config.buffer
);
producer
}
Err(err) => {
SCLogError!("Failed to initialize Kafka Producer: {:?}", err);
panic!()
}
};
let context = Context {
tx,
count: 0,
dropped: 0,
};
std::thread::spawn(move || {kafka_producer.run()});
// kafka_producer.run();
*init_data = Box::into_raw(Box::new(context)) as *mut _;
0
}
unsafe extern "C" fn output_close(init_data: *const c_void) {
let context = Box::from_raw(init_data as *mut Context);
SCLogNotice!(
"Kafka produce finished: count={}, dropped={}",
context.count,
context.dropped
);
std::mem::drop(context);
}
unsafe extern "C" fn output_write(
buffer: *const c_char,
buffer_len: c_int,
init_data: *const c_void,
) -> c_int {
let context = &mut *(init_data as *mut Context);
let buf = if let Ok(buf) = ffi::str_from_c_parts(buffer, buffer_len) {
buf
} else {
return -1;
};
context.count += 1;
if let Err(err) = context.tx.try_send(buf.to_string()) {
context.dropped += 1;
match err {
TrySendError::Full(_) => {
SCLogError!("Eve record lost due to full buffer");
}
TrySendError::Disconnected(_) => {
SCLogError!("Eve record lost due to broken channel{}",err);
}
}
}
00
}
unsafe extern "C" fn init_plugin() {
let file_type =
ffi::SCPluginFileType::new("kafka", output_open, output_close, output_write);
ffi::SCPluginRegisterFileType(file_type);
}
#[no_mangle]
extern "C" fn SCPluginRegister() -> *const ffi::SCPlugin {
// Rust plugins need to initialize some Suricata internals so stuff like logging works.
suricata::plugin::init();
// Register our plugin.
ffi::SCPlugin::new("Kafka Eve Filetype", "GPL-2.0", "1z3r0", init_plugin)
}

Get a raw vec with field names of any struct with a custom derive macro in Rust

I am trying to write some code that could be able to write a method that returns me a Vec with the names of the fields of a struct.
Code snippet below:
# Don't forget about dependencies if you try to reproduce this on local
use proc_macro2::{Span, Ident};
use quote::quote;
use syn::{
punctuated::Punctuated, token::Comma, Attribute, DeriveInput, Fields, Meta, NestedMeta,
Variant, Visibility,
};
#[proc_macro_derive(StructFieldNames, attributes(struct_field_names))]
pub fn derive_field_names(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
let ast: DeriveInput = syn::parse(input).unwrap();
let (vis, ty, generics) = (&ast.vis, &ast.ident, &ast.generics);
let names_struct_ident = Ident::new(&(ty.to_string() + "FieldStaticStr"), Span::call_site());
let fields = filter_fields(match ast.data {
syn::Data::Struct(ref s) => &s.fields,
_ => panic!("FieldNames can only be derived for structs"),
});
let names_struct_fields = fields.iter().map(|(vis, ident)| {
quote! {
#vis #ident: &'static str
}
});
let mut vec_fields: Vec<String> = Vec::new();
let names_const_fields = fields.iter().map(|(_vis, ident)| {
let ident_name = ident.to_string();
vec_fields.push(ident_name);
quote! {
#vis #ident: -
}
});
let names_const_fields_as_vec = fields.iter().map(|(_vis, ident)| {
let ident_name = ident.to_string();
// vec_fields.push(ident_name)
});
let (impl_generics, ty_generics, where_clause) = generics.split_for_impl();
let tokens = quote! {
#[derive(Debug)]
#vis struct #names_struct_ident {
#(#names_struct_fields),*
}
impl #impl_generics #ty #ty_generics
#where_clause
{
#vis fn get_field_names() -> &'static str {
// stringify!(
[ #(#vec_fields),* ]
.map( |s| s.to_string())
.collect()
// )
}
}
};
tokens.into()
}
fn filter_fields(fields: &Fields) -> Vec<(Visibility, Ident)> {
fields
.iter()
.filter_map(|field| {
if field
.attrs
.iter()
.find(|attr| has_skip_attr(attr, "struct_field_names"))
.is_none()
&& field.ident.is_some()
{
let field_vis = field.vis.clone();
let field_ident = field.ident.as_ref().unwrap().clone();
Some((field_vis, field_ident))
} else {
None
}
})
.collect::<Vec<_>>()
}
const ATTR_META_SKIP: &'static str = "skip";
fn has_skip_attr(attr: &Attribute, path: &'static str) -> bool {
if let Ok(Meta::List(meta_list)) = attr.parse_meta() {
if meta_list.path.is_ident(path) {
for nested_item in meta_list.nested.iter() {
if let NestedMeta::Meta(Meta::Path(path)) = nested_item {
if path.is_ident(ATTR_META_SKIP) {
return true;
}
}
}
}
}
false
}
The code it's taken from here. Basically I just want to get those values as a String, and not to access them via Foo::FIELD_NAMES.some_random_field, because I need them for another process.
How can I achieve that?
Thanks

Can't get image::load_from_memory() to work when compiled to WebAssembly

I am trying to load an image from JavaScript to WebAssembly with Rust using the image crate.
I have the following Rust code:
extern crate image;
extern crate libc;
use libc::c_void;
use std::mem;
#[no_mangle]
pub extern "C" fn alloc(size: usize) -> *mut c_void {
let mut buf = Vec::with_capacity(size);
let ptr = buf.as_mut_ptr();
mem::forget(buf);
return ptr as *mut c_void;
}
#[no_mangle]
pub extern "C" fn read_img(buff_ptr: *mut u8, buff_len: usize) -> *mut i32 {
let mut img: Vec<u8> = unsafe { Vec::from_raw_parts(buff_ptr, buff_len, buff_len) };
let ok = Box::new([333]);
let err = Box::new([331]);
return match image::load_from_memory(&img) {
Ok(img) => Box::into_raw(ok) as *mut i32,
Err(_) => Box::into_raw(err) as *mut i32,
};
}
fn main() {}
which I compile using the following tools:
cargo +nightly build --target wasm32-unknown-unknown --release
In the read_img() function, I naively handle errors via two vectors: [333] for OK and [331] for any error. I read these vectors on the JavaScript side to know if the image was loaded successfully.
The load_from_memory method fails because I get the [331] vector. If I replace the load_from_memory method with the guess_format method, I get the [333] vector. I also did some pattern matching for PNG and JPG and it reads the buffer correctly.
I couldn't find how can I more thoroughly debug such behaviour.
On the JavaScript part, I simply load up the image's arrayBuffer into WASM's shared memory.
Here is what I am doing on the JavaScript side:
function compile(wasmFile = 'distil_wasm.gc.wasm') {
return fetch(wasmFile)
.then(r => r.arrayBuffer())
.then(r => {
let module = new WebAssembly.Module(r);
let importObject = {}
for (let imp of WebAssembly.Module.imports(module)) {
if (typeof importObject[imp.module] === "undefined")
importObject[imp.module] = {};
switch (imp.kind) {
case "function": importObject[imp.module][imp.name] = () => {}; break;
case "table": importObject[imp.module][imp.name] = new WebAssembly.Table({ initial: 256, maximum: 256, element: "anyfunc" }); break;
case "memory": importObject[imp.module][imp.name] = new WebAssembly.Memory({ initial: 256 }); break;
case "global": importObject[imp.module][imp.name] = 0; break;
}
}
return WebAssembly.instantiate(r, importObject);
});
}
function loadImgIntoMem(img, memory, alloc) {
return new Promise(resolve => {
fetch(img)
.then(r => r.arrayBuffer())
.then(buff => {
const imgPtr = alloc(buff.byteLength);
const imgHeap = new Uint8Array(memory.buffer, imgPtr, buff.byteLength);
imgHeap.set(new Uint8Array(buff));
resolve({ imgPtr, len: buff.byteLength });
});
});
}
function run(img) {
return compile().then(m => {
return loadImgIntoMem(img, m.instance.exports.memory, m.instance.exports.alloc).then(r => {
window.WASM = m;
return m.instance.exports.read_img(r.imgPtr, r.len);
});
});
}
run('img-2.jpg')
.then(ptr => console.log(new Int32Array(WASM.instance.exports.memory.buffer, ptr, 1)))
This console logs:
Int32Array [ 331 ]
It's basically impossible to debug things without access to a debugger or the ability to print out messages. Because of this, I ported your code to use wasm-bindgen, purely for the ability to access the console from inside Rust code:
#![feature(proc_macro, wasm_custom_section, wasm_import_module)]
extern crate wasm_bindgen;
extern crate image;
use wasm_bindgen::prelude::*;
use std::mem;
pub mod console {
use wasm_bindgen::prelude::*;
#[wasm_bindgen]
extern {
#[wasm_bindgen(js_namespace = console)]
pub fn log(s: &str);
}
}
#[wasm_bindgen]
pub fn alloc(len: usize) -> *mut u8 {
let mut buf = Vec::with_capacity(len);
let ptr = buf.as_mut_ptr();
mem::forget(buf);
ptr
}
#[wasm_bindgen]
pub fn read_img(ptr: *mut u8, len: usize) {
let img = unsafe { Vec::from_raw_parts(ptr, len, len) };
if let Err(e) = image::load_from_memory(&img) {
console::log(&e.to_string());
}
}
The updated JavaScript:
const js = import("./imaj_bg");
async function loadImgIntoMem(img, { alloc, memory }) {
const resp = await fetch(img);
const buf = await resp.arrayBuffer();
const len = buf.byteLength;
const ptr = alloc(len);
const imgArray = new Uint8Array(memory.buffer, ptr, len);
imgArray.set(new Uint8Array(buf));
return { ptr, len };
}
async function go(js) {
const { ptr, len } = await loadImgIntoMem('cat.jpg', js);
js.read_img(ptr, len);
};
js.then(go);
Building and serving the code:
$ cargo build --target wasm32-unknown-unknown --release
$ wasm-bindgen target/wasm32-unknown-unknown/release/imaj.wasm --out-dir=.
$ yarn serve
Accessing the page and reviewing the console log shows this anticlimactic message:
operation not supported on wasm yet
The truth is that there's large parts of the Rust standard library that don't exist yet in WebAssembly. Many of these are stubbed out to return this error.
I don't know exactly which platform support that is missing for your code. The most obvious one is threading, required by the jpeg_rayon and hdr feature, but turning off all of image's features except jpeg still reports same error. It's likely there's something else needed.
However, it does seem to be specific to a given image codec. If you try the same code but load a PNG image, it's successful:
pub fn read_img(ptr: *mut u8, len: usize) {
let img = unsafe { Vec::from_raw_parts(ptr, len, len) };
let img = match image::load_from_memory(&img) {
Ok(i) => i,
Err(e) => {
console::log(&e.to_string());
return;
}
};
console::log(&format!("{:?}", img.to_rgba()));
}
ImageBuffer { width: 305, height: 314, _phantom: PhantomData, data: [255, 255, 255, 0 /* remaining pixels skipped */
This indicates that the JPEG code does not yet work with WASM. A given codec may or may not work yet; it's probably best to file issues with the upstream maintainers.

What might cause a difficult-to-reproduce truncation of a Hyper HTTP response?

I am experiencing a bug where my Hyper HTTP response is being truncated to a specific size (7829 bytes). Making the same request with cURL works fine.
The request queries a JSON endpoint for data. The response struct is then shuffled around a lot, because a relatively complex rate-limiting procedure is used to make a number of these requests at once. However, if only one request is made, the response is still truncated.
Before implementing rate-limiting and doing some heavy refactoring, the program made these responses properly.
I made the minimal example below, but it fails to reproduce the problem. At this point I'm not sure where to look. The codebase is moderately complicated and iteratively expanding the reproduction example is difficult, especially when I don't know what might possibly cause this.
What are some ways that Hyper's Response body might get truncated? The response body is acquired as in the handle function below.
#![feature(use_nested_groups)]
extern crate futures;
extern crate hyper;
extern crate hyper_tls;
extern crate tokio_core;
use futures::{Future, Stream};
use hyper::{Body, Chunk, Client, Method, Request, Response};
use hyper_tls::HttpsConnector;
use tokio_core::reactor::Core;
use std::env;
fn main() {
let mut core = Core::new().unwrap();
let client = Client::configure()
.connector(HttpsConnector::new(4, &core.handle()).unwrap())
.build(&core.handle());
fn handle(response: Response<Body>) -> Box<Future<Item = usize, Error = hyper::Error>> {
Box::new(
response
.body()
.concat2()
.map(move |body: Chunk| -> usize { body.len() }),
)
}
let args: Vec<String> = env::args().collect();
let uri = &args[1];
let req = Request::new(Method::Get, uri.parse().unwrap());
let response_body_length = {
let future = Box::new(client.request(req).map(handle).flatten());
core.run(future).unwrap()
};
println!("response body length: {}", response_body_length);
}
Offending code:
extern crate serde;
extern crate serde_json;
use futures::{future, stream, Future, Stream};
use hyper;
use hyper::{client, Body, Chunk, Client, Headers, Method, Request, Response, header::Accept,
header::Date as DateHeader, header::RetryAfter};
use hyper_tls::HttpsConnector;
use tokio_core::reactor::Core;
use models::Bucket;
use std::thread;
use std::time::{Duration, UNIX_EPOCH};
use std::str;
header! { (XRateLimitRemaining, "x-ratelimit-remaining") => [String] }
#[derive(Debug)]
struct Uri(pub String);
const MAX_REQ_SIZE: u32 = 500;
fn make_uri(symbol: &str, page_ix: u32) -> Uri {
Uri(format!(
"https://www.bitmex.com/api/v1/trade/bucketed?\
symbol={symbol}&\
columns={columns}&\
partial=false&\
reverse=true&\
binSize={bin_size}&\
count={count}&\
start={start}",
symbol = symbol,
columns = "close,timestamp",
bin_size = "5m",
count = MAX_REQ_SIZE,
start = 0 + MAX_REQ_SIZE * page_ix
))
}
#[derive(Debug)]
struct RateLimitInfo {
remaining_reqs: u32,
retry_after: Option<Duration>,
}
impl RateLimitInfo {
fn default() -> RateLimitInfo {
RateLimitInfo {
remaining_reqs: 1,
retry_after: None,
}
}
fn from<T>(resp: &Response<T>) -> RateLimitInfo {
let headers = resp.headers();
let remaining_reqs = headers
.get::<XRateLimitRemaining>()
.unwrap_or_else(|| panic!("x-ratelimit-remaining not on request."))
.parse()
.unwrap();
let retry_after = match headers.get::<RetryAfter>() {
Some(RetryAfter::Delay(duration)) => Some(*duration),
_ => None,
};
RateLimitInfo {
remaining_reqs,
retry_after,
}
}
}
fn resp_dated_later<'a>(a: &'a Response<Body>, b: &'a Response<Body>) -> &'a Response<Body> {
let get_date = |resp: &Response<Body>| {
let headers: &Headers = resp.headers();
**headers.get::<DateHeader>().unwrap()
};
if get_date(&a) > get_date(&b) {
a
} else {
b
}
}
#[derive(Debug)]
struct Query {
uri: Uri,
response: Option<Response<Body>>,
}
impl Query {
fn from_uri(uri: Uri) -> Query {
Query {
uri: uri,
response: None,
}
}
}
fn query_good(q: &Query) -> bool {
match &q.response {
Some(response) => response.status().is_success(),
_ => false,
}
}
type HttpsClient = hyper::Client<HttpsConnector<client::HttpConnector>>;
type FutureQuery = Box<Future<Item = Query, Error = hyper::Error>>;
fn to_future(x: Query) -> FutureQuery {
Box::new(future::ok(x))
}
fn exec_if_needed(client: &HttpsClient, query: Query) -> FutureQuery {
fn exec(client: &HttpsClient, q: Query) -> FutureQuery {
println!("exec: {:?}", q);
let uri = q.uri;
let req = {
let mut req = Request::new(Method::Get, uri.0.parse().unwrap());
req.headers_mut().set(Accept::json());
req
};
Box::new(
client
.request(req)
.inspect(|resp| println!("HTTP {}", resp.status()))
.map(|resp| Query {
uri: uri,
response: Some(resp),
}),
)
}
if query_good(&query) {
to_future(query)
} else {
exec(client, query)
}
}
type BoxedFuture<T> = Box<Future<Item = T, Error = hyper::Error>>;
fn do_batch(client: &HttpsClient, queries: Vec<Query>) -> BoxedFuture<Vec<Query>> {
println!("do_batch() {} queries", queries.len());
let exec_if_needed = |q| exec_if_needed(client, q);
let futures = queries.into_iter().map(exec_if_needed);
println!("do_batch() futures {:?}", futures);
Box::new(
stream::futures_ordered(futures).collect(), //future::join_all(futures)
)
}
fn take<T>(right: &mut Vec<T>, suggested_n: usize) -> Vec<T> {
let n: usize = if right.len() < suggested_n {
right.len()
} else {
suggested_n
};
let left = right.drain(0..n);
return left.collect();
}
type BoxedResponses = Box<Vec<Response<Body>>>;
fn batched_throttle(uris: Vec<Uri>) -> BoxedResponses {
println!("batched_throttle({} uris)", uris.len());
let mut core = Core::new().unwrap();
let client = Client::configure()
.connector(HttpsConnector::new(4, &core.handle()).unwrap())
.build(&core.handle());
let mut rate_limit_info = RateLimitInfo::default();
let mut queries_right: Vec<Query> = uris.into_iter().map(Query::from_uri).collect();
loop {
let mut queries_left: Vec<Query> = Vec::with_capacity(queries_right.len());
println!("batched_throttle: starting inner loop");
loop {
// throttle program during testing
thread::sleep(Duration::from_millis(800));
println!("batched_throttle: {:?}", rate_limit_info);
if let Some(retry_after) = rate_limit_info.retry_after {
println!("batched_throttle: retrying after {:?}", retry_after);
thread::sleep(retry_after)
}
if queries_right.is_empty() {
break;
}
let mut queries_mid = {
let ri_count = rate_limit_info.remaining_reqs;
let iter_req_count = if ri_count == 0 { 1 } else { ri_count };
println!("batched_throttle: iter_req_count {}", iter_req_count);
take(&mut queries_right, iter_req_count as usize)
};
println!(
"batched_throttle: \
queries_right.len() {}, \
queries_left.len() {}, \
queries_mid.len() {})",
queries_right.len(),
queries_left.len(),
queries_mid.len()
);
if queries_mid.iter().all(query_good) {
println!("batched_throttle: queries_mid.iter().all(query_good)");
continue;
}
queries_mid = { core.run(do_batch(&client, queries_mid)).unwrap() };
rate_limit_info = {
let create_very_old_response =
|| Response::new().with_header(DateHeader(UNIX_EPOCH.into()));
let very_old_response = create_very_old_response();
let last_resp = queries_mid
.iter()
.map(|q| match &q.response {
Some(r) => r,
_ => panic!("Impossible"),
})
.fold(&very_old_response, resp_dated_later);
RateLimitInfo::from(&last_resp)
};
&queries_left.append(&mut queries_mid);
}
queries_right = queries_left;
if queries_right.iter().all(query_good) {
break;
}
}
println!(
"batched_throttle: finishing. queries_right.len() {}",
queries_right.len()
);
Box::new(
queries_right
.into_iter()
.map(|q| q.response.unwrap())
.collect(),
)
}
fn bucket_count_to_req_count(bucket_count: u32) -> u32 {
let needed_req_count = (bucket_count as f32 / MAX_REQ_SIZE as f32).ceil() as u32;
return needed_req_count;
}
type BoxedBuckets = Box<Vec<Bucket>>;
fn response_to_buckets(response: Response<Body>) -> BoxedFuture<Vec<Bucket>> {
Box::new(response.body().concat2().map(|body: Chunk| -> Vec<Bucket> {
println!("body.len(): {}", body.len());
println!("JSON: {}", str::from_utf8(&body).unwrap());
serde_json::from_slice(&body).unwrap()
}))
}
pub fn get_n_last(symbol: &str, bucket_count: u32) -> BoxedBuckets {
let req_count = bucket_count_to_req_count(bucket_count);
let uris = (0..req_count)
.map(|page_ix| make_uri(symbol, page_ix))
.collect();
let responses = batched_throttle(uris);
let mut core = Core::new().unwrap();
let boxed_buckets = {
let futures = responses.into_iter().map(response_to_buckets);
let future = stream::futures_ordered(futures).collect();
let groups_of_buckets = core.run(future).unwrap();
Box::new(
groups_of_buckets
.into_iter()
.flat_map(|bs| bs.into_iter())
.rev()
.collect(),
)
};
return boxed_buckets;
}
You first create a Core and start lots of requests and gather the Response "results".
After you got all the Responses you start a new Core and try to start reading the data from those Responses - but the server probably closed them long ago due to write timeouts, and you only get partial data.
You shouldn't keep the server waiting; start reading the Responses as soon as possible.

Polymorphism in Rust and trait references (trait objects?)

I'm writing a process memory scanner with a console prompt interface in Rust.
I need scanner types such as a winapi scanner or a ring0 driver scanner so I'm trying to implement polymorphism.
I have the following construction at this moment:
pub trait Scanner {
fn attach(&mut self, pid: u32) -> bool;
fn detach(&mut self);
}
pub struct WinapiScanner {
pid: u32,
hprocess: HANDLE,
addresses: Vec<usize>
}
impl WinapiScanner {
pub fn new() -> WinapiScanner {
WinapiScanner {
pid: 0,
hprocess: 0 as HANDLE,
addresses: Vec::<usize>::new()
}
}
}
impl Scanner for WinapiScanner {
fn attach(&mut self, pid: u32) -> bool {
let handle = unsafe { OpenProcess(PROCESS_ALL_ACCESS, FALSE, pid) };
if handle == 0 as HANDLE {
self.pid = pid;
self.hprocess = handle;
true
} else {
false
}
}
fn detach(&mut self) {
unsafe { CloseHandle(self.hprocess) };
self.pid = 0;
self.hprocess = 0 as HANDLE;
self.addresses.clear();
}
}
In future, I'll have some more scanner types besides WinapiScanner, so, if I understand correctly, I should use a trait reference (&Scanner) to implement polymorphism. I'm trying to create Scanner object like this (note the comments):
enum ScannerType {
Winapi
}
pub fn start() {
let mut scanner: Option<&mut Scanner> = None;
let mut scanner_type = ScannerType::Winapi;
loop {
let line = prompt();
let tokens: Vec<&str> = line.split_whitespace().collect();
match tokens[0] {
// commands
"scanner" => {
if tokens.len() != 2 {
println!("\"scanner\" command takes 1 argument")
} else {
match tokens[1] {
"list" => {
println!("Available scanners: winapi");
},
"winapi" => {
scanner_type = ScannerType::Winapi;
println!("Scanner type set to: winapi");
},
x => {
println!("Unknown scanner type: {}", x);
}
}
}
},
"attach" => {
if tokens.len() > 1 {
match tokens[1].parse::<u32>() {
Ok(pid) => {
scanner = match scanner_type {
// ----------------------
// Problem goes here.
// Object, created by WinapiScanner::new() constructor
// doesn't live long enough to borrow it here
ScannerType::Winapi => Some(&mut WinapiScanner::new())
// ----------------------
}
}
Err(_) => {
println!("Wrong pid");
}
}
}
},
x => println!("Unknown command: {}", x)
}
}
}
fn prompt() -> String {
use std::io::Write;
use std::io::BufRead;
let stdout = io::stdout();
let mut lock = stdout.lock();
let _ = lock.write(">> ".as_bytes());
let _ = lock.flush();
let stdin = io::stdin();
let mut lock = stdin.lock();
let mut buf = String::new();
let _ = lock.read_line(&mut buf);
String::from(buf.trim())
}
It's not a full program; I've pasted important parts only.
What am I doing wrong and how do I implement what I want in Rust?
Trait objects must be used behind a pointer. But references are not the only kind of pointers; Box is also a pointer!
let mut scanner: Option<Box<Scanner>> = None;
scanner = match scanner_type {
ScannerType::Winapi => Some(Box::new(WinapiScanner::new()))
}

Resources