rust async trait with coroutine - rust

I develop an update system in rust. It create patches between 2 binaries. You can download binaries or patches and install it. Patches and binaries are compressed. The client download compressed files and decompress it
I do
#[async_trait]
impl<'a> Stream for UpdatePackageStream<'a> {
type Item = Result<SharedUpdateProgress, UpdateError>;
async fn poll_next(
self: Pin<&mut Self>,
cx: &mut std::task::Context<'_>,
) -> Poll<Option<Self::Item>> {
let this = self.get_mut();
let download_poll = this.download_stream.poll_next_unpin(cx);
let apply_poll = this.apply_stream.poll_next_unpin(cx);
match (download_poll, apply_poll) {
(Poll::Ready(None), Poll::Ready(None)) => Poll::Ready(None),
(Poll::Pending, Poll::Pending) => Poll::Pending,
(Poll::Pending, Poll::Ready(None)) => Poll::Pending,
(Poll::Ready(None), Poll::Pending) => Poll::Pending,
(Poll::Ready(Some(Err(err))), _) => {
// Download errors cause the apply thread to be cancelled
this.apply_stream.cancel();
Poll::Ready(Some(Err(err)))
}
(download_poll, apply_poll) => {
let mut delta = Progression::default();
if let Poll::Ready(Some(Ok(download_progress))) = download_poll {
this.state.borrow_mut().available = download_progress.available;
let mut state = this.shared_state.lock().await;
state.downloading_operation_idx = download_progress.available.operation_idx;
delta.downloaded_files = download_progress.delta_downloaded_files;
delta.downloaded_bytes = download_progress.delta_downloaded_bytes;
this.apply_stream.notify(download_progress.available);
}
if let Poll::Ready(Some(apply_progress)) = apply_poll {
match apply_progress {
Ok(apply_progress) => {
this.state.borrow_mut().applied.operation_idx =
apply_progress.operation_idx;
let mut state = this.shared_state.lock().await; // note the await here, so the closure must be async
state.applying_operation_idx = apply_progress.operation_idx;
delta.applied_files = apply_progress.delta_applied_files;
delta.applied_input_bytes = apply_progress.delta_input_bytes;
delta.applied_output_bytes = apply_progress.delta_output_bytes;
}
Err(ApplyError::OperationFailed { path, slice, cause }) => {
warn!("{} failed: {}", path, cause);
let mut state = this.state.borrow_mut();
state.failures.push(match slice {
Some(slice) => metadata::v1::Failure::Slice { path, slice },
None => metadata::v1::Failure::Path { path },
});
delta.failed_files = 1;
}
Err(ApplyError::Cancelled) => {}
Err(ApplyError::PoisonError) => {
return Poll::Ready(Some(Err(UpdateError::PoisonError)))
}
}
}
`` {
let mut state = this.shared_state.lock().await;
state.inc_progress(delta);
}
Poll::Ready(Some(Ok(this.shared_state.clone())))
}
}
}
}
But get
error[E0195]: lifetime parameters or bounds on method `poll_next` do not match the trait declaration
--> lib/src/workspace/updater.rs:143:14
|
143 | async fn poll_next(
| ______________^
144 | | self: Pin<&mut Self>,
145 | | cx: &mut std::task::Context<'_>,
146 | | ) -> Poll<Option<Self::Item>> {
| |_____^ lifetimes do not match method in trait
Stream is from future::stream::Stream
Without async_trait and the 2 async fn, the fn and my app build without issue.
If I use std::sync::Mutex instead of tokio::sync::Mutex, I get
let state = update_state.lock();
| ----- has type `std::sync::MutexGuard<'_, UpdateProgress>` which is not `Send`
...
273 | let res = update_stream.try_for_each(|_state| future::ready(Ok(()))).await;
| ^^^^^^ await occurs here, with `state` maybe used later
...
305 | }
| - `state` is later dropped here
= note: required for the cast from `impl futures::Future<Output = Result<tonic::Response<BuildOutput>, Status>>` to the object type `dyn futures::Future<Output = Result<tonic::Response<BuildOutput>, Status>> + std::marker::Send`
UpdateProgress is a simple struct.
I don't inderstand why I get these issues and don't know how to fix them

Related

Error in Async closure: Lifetime may not live long enough; returning this value requires that `'1` must outlive `'2`

I'm trying to use a Mutex<Sender> inside an async closure but I'm not entirely sure why I'm getting the error below:
error: lifetime may not live long enough
--> src/main.rs:120:41
|
120 | io.add_method(MARK_ITSELF, move |_| async {
| ________________________________--------_^
| | | |
| | | return type of closure `impl std::future::Future<Output = Result<jsonrpc::serde_json::Value, jsonrpc_http_server::jsonrpc_core::Error>>` contains a lifetime `'2`
| | lifetime `'1` represents this closure's body
121 | | trace!("Mark itself!");
122 | | let tx = sender.lock().map_err(to_internal)?;
123 | | tx.send(Action::MarkItself)
124 | | .map_err(to_internal)
125 | | .map(|_| Value::Bool(true))
126 | | });
| |_____^ returning this value requires that `'1` must outlive `'2`
|
= note: closure implements `Fn`, so references to captured variables can't escape the closure
My main function looks like this:
use jsonrpc::serde_json::Value;
use jsonrpc::Error as ClientError;
use jsonrpc::{
serde_json::value::RawValue,
simple_http::{self, SimpleHttpTransport},
Client,
};
use jsonrpc_http_server::jsonrpc_core::{Error as ServerError, IoHandler};
use jsonrpc_http_server::ServerBuilder;
use log::{debug, error, trace};
use serde::Deserialize;
use std::rc::Rc;
use std::sync::mpsc::Receiver;
use std::sync::Mutex;
use std::thread;
use std::{
env, fmt,
net::SocketAddr,
sync::mpsc::{channel, Sender},
};
const START_ROLL_CALL: &str = "start_roll_call";
const MARK_ITSELF: &str = "mark_itself";
fn create_client(url: &str, user: &str, pass: &str) -> Result<Client, simple_http::Error> {
let t = SimpleHttpTransport::builder()
.url(url)?
.auth(user, Some(pass))
.build();
Ok(Client::with_transport(t))
}
fn spawn_worker() -> Result<Sender<Action>, failure::Error> {
let (tx, rx): (Sender<Action>, Receiver<Action>) = channel();
let next: SocketAddr = env::var("NEXT")?.parse()?;
thread::spawn(move || {
let remote = Remote::new(next).unwrap();
let mut in_roll_call = false;
for action in rx.iter() {
match action {
Action::StartRollCall => {
if !in_roll_call {
if remote.start_roll_call().is_ok() {
debug!("ON");
in_roll_call = true;
}
} else {
if remote.mark_itself().is_ok() {
debug!("OFF");
in_roll_call = false;
}
}
}
Action::MarkItself => {
if in_roll_call {
if remote.mark_itself().is_ok() {
debug!("OFF");
in_roll_call = false;
}
} else {
debug!("SKIP");
}
}
}
}
});
Ok(tx)
}
enum Action {
StartRollCall,
MarkItself,
}
struct Remote {
client: Client,
}
impl Remote {
fn new(addr: SocketAddr) -> Result<Self, simple_http::Error> {
let url = format!("http://{}", addr);
let client = create_client(&url, "", "")?;
Ok(Self { client })
}
fn call_method<T>(&self, method: &str, params: &[Box<RawValue>]) -> Result<T, ClientError>
where
T: for<'de> Deserialize<'de>,
{
let request = self.client.build_request(method, params);
self.client
.send_request(request)
.and_then(|res| res.result::<T>())
}
fn start_roll_call(&self) -> Result<bool, ClientError> {
self.call_method(START_ROLL_CALL, &[])
}
fn mark_itself(&self) -> Result<bool, ClientError> {
self.call_method(MARK_ITSELF, &[])
}
}
fn main() -> Result<(), failure::Error> {
env_logger::init();
let tx = spawn_worker()?;
let addr: SocketAddr = env::var("ADDRESS")?.parse()?;
let mut io = IoHandler::default();
let sender = Mutex::new(tx.clone());
io.add_method(START_ROLL_CALL, move |_| async move {
trace!("Starting roll call!");
let tx = sender.lock().map_err(to_internal)?;
tx.send(Action::StartRollCall)
.map_err(to_internal)
.map(|_| Value::Bool(true))
});
let sender = Mutex::new(tx.clone());
io.add_method(MARK_ITSELF, move |_| async {
trace!("Mark itself!");
let tx = sender.lock().map_err(to_internal)?;
tx.send(Action::MarkItself)
.map_err(to_internal)
.map(|_| Value::Bool(true))
});
let server = ServerBuilder::new(io).start_http(&addr)?;
Ok(server.wait())
}
fn to_internal<E: fmt::Display>(err: E) -> ServerError {
error!("Error: {}", err);
ServerError::internal_error()
}
The main idea is to pass the mspc sender to the closure so that the method can send the Action(An enum). Is there something I'm doing wrong?
The problem is add_method requires that
your closure is static, and
the Future returned from your closure is static.
Try this
let sender = Arc::new(Mutex::new(tx.clone()));
io.add_method(START_ROLL_CALL, move |_| {
let cloned_sender = sender.clone();
async move {
trace!("Starting roll call!");
let tx = cloned_sender.lock().map_err(to_internal)?;
tx.send(Action::StartRollCall)
.map_err(to_internal)
.map(|_| Value::Bool(true))
}
});
Side note: you are using sync Mutex in an async environment. This undermines the benefits of async. Consider using async Mutex such as Tokio Mutex or async-lock Mutex.

"future cannot be sent between threads safely" when pass Arc<Mutex> into tokio::spawn

I implemented TCP client using tokio. However, my code not compile because I got an error:
error: future cannot be sent between threads safely
--> src/main.rs:81:9
|
81 | tokio::spawn(async move {
| ^^^^^^^^^^^^ future created by async block is not `Send`
|
= help: within `impl Future<Output = ()>`, the trait `Send` is not implemented for `std::sync::MutexGuard<'_, Option<tokio::net::TcpStream>>`
note: future is not `Send` as this value is used across an await
--> src/main.rs:90:42
|
82 | match stream.lock().unwrap().as_mut() {
| ---------------------- has type `std::sync::MutexGuard<'_, Option<tokio::net::TcpStream>>` which is not `Send`
...
90 | stream.write(&packet).await.unwrap();
| ^^^^^^ await occurs here, with `stream.lock().unwrap()` maybe used later
...
94 | };
| - `stream.lock().unwrap()` is later dropped here
help: consider moving this into a `let` binding to create a shorter lived borrow
--> src/main.rs:82:19
|
82 | match stream.lock().unwrap().as_mut() {
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
note: required by a bound in `tokio::spawn`
--> /playground/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.19.2/src/task/spawn.rs:127:21
|
127 | T: Future + Send + 'static,
| ^^^^ required by this bound in `tokio::spawn`
This is my code where issue occurs:
async fn handle_write(&mut self) -> JoinHandle<()> {
let stream = Arc::clone(&self.stream);
let session = Arc::clone(&self.session);
let queue = Arc::clone(&self.queue);
tokio::spawn(async move {
match stream.lock().unwrap().as_mut() {
Some(stream) => {
let packet: Vec<u8> = queue.lock().unwrap().pop_front().unwrap();
let packet = match session.lock().unwrap().header_crypt.as_mut() {
Some(header_crypt) => header_crypt.encrypt(&packet),
_ => packet,
};
stream.write(&packet).await.unwrap();
stream.flush().await.unwrap();
},
_ => {},
};
})
}
and same issue here:
async fn handle_read(&mut self) -> JoinHandle<()> {
let queue = Arc::clone(&self.queue);
let stream = Arc::clone(&self.stream);
let session = Arc::clone(&self.session);
tokio::spawn(async move {
match stream.lock().unwrap().as_mut() {
Some(stream) => {
let mut buffer = [0u8; 4096];
match stream.read(&mut buffer).await {
Ok(bytes_count) => {
let raw_data = match session.lock().unwrap().header_crypt.as_mut() {
Some(header_crypt) => header_crypt.decrypt(&buffer[..bytes_count]),
_ => buffer[..bytes_count].to_vec(),
};
queue.lock().unwrap().push_back(raw_data);
},
_ => {},
};
},
_ => {},
};
})
}
Playground.
Could somebody explain, what am I doing wrong ?
P.S. just in case: I am using std::sync::{Arc, Mutex};
Finally, as decided in comments to my question, I used tokio::sync::Mutex instead of std::sync::Mutex. So, now code compiles correctly.
Playground.
In my case the problem was using ThreadRng with thread_rng() which is NOT thread safe. Just as a heads-up for anyone else banging their head against this error message. I refactored to using let mut rng = ::rand::rngs::StdRng::from_seed(OsRng.gen());

Cannot move out of an `Arc`

I have a struct called routing with the following methods.
#[derive(Clone)]
pub struct Routing {
get_routes: Node<PyFunction>,
post_routes: Node<PyFunction>,
put_routes: Node<PyFunction>,
delete_routes: Node<PyFunction>,
patch_routes: Node<PyFunction>,
head_routes: Node<PyFunction>,
options_routes: Node<PyFunction>,
connect_routes: Node<PyFunction>,
trace_routes: Node<PyFunction>,
}
impl Routing {
pub fn new(router: &Router) -> Routing {
let get_routes = Router::protocol_to_tree(&router.get_routes);
let post_routes = Router::protocol_to_tree(&router.post_routes);
let put_routes = Router::protocol_to_tree(&router.put_routes);
let delete_routes = Router::protocol_to_tree(&router.delete_routes);
let patch_routes = Router::protocol_to_tree(&router.patch_routes);
let head_routes = Router::protocol_to_tree(&router.head_routes);
let options_routes = Router::protocol_to_tree(&router.options_routes);
let connect_routes = Router::protocol_to_tree(&router.connect_routes);
let trace_routes = Router::protocol_to_tree(&router.trace_routes);
Self {
get_routes,
post_routes,
put_routes,
delete_routes,
patch_routes,
head_routes,
options_routes,
connect_routes,
trace_routes,
}
}
#[inline(always)]
fn get_relevant_map(self, route: &Method) -> Option<Node<PyFunction>> {
match route {
&Method::GET => Some(self.get_routes),
&Method::POST => Some(self.post_routes),
&Method::PUT => Some(self.put_routes),
&Method::PATCH => Some(self.patch_routes),
&Method::DELETE => Some(self.delete_routes),
&Method::HEAD => Some(self.head_routes),
&Method::OPTIONS => Some(self.options_routes),
&Method::CONNECT => Some(self.connect_routes),
&Method::TRACE => Some(self.trace_routes),
_ => None,
}
}
#[inline(always)]
pub fn get_route(self, route_method: &Method, route: &str) -> Option<PyFunction> {
println!("Hello world how are your");
let table = self.get_relevant_map(route_method)?;
let x = match table.at(route) {
Ok(res) => Some(res.value.clone()),
Err(_) => None,
};
x
}
}
and I am trying to access call the get_route method of the implementation, but I am getting an error.
async fn index(
router: web::Data<Arc<Routing>>,
headers: web::Data<Arc<Headers>>,
mut payload: web::Payload,
req: HttpRequest,
) -> impl Responder {
let x = match (&router)
.clone()
.get_route(&req.method().clone(), req.uri().path())
.clone()
{
Some(handler_function) => {
return handle_request(&handler_function, &headers, &mut payload, &req).await
}
None => {
let mut response = HttpResponse::Ok();
apply_headers(&mut response, &headers);
return response.finish();
}
};
x
}
[rustc E0507] [E] cannot move out of an `Arc`
move occurs because value has type `router::Routing`, which does not implement the `Copy` trait
Complete error message:
error[E0507]: cannot move out of an `Arc`
--> src/server.rs:160:19
|
160 | let x = match (&router)
| ___________________^
161 | | .clone()
| |________________^ move occurs because value has type `Routing`, which does not implement the `Copy` trait
error: aborting due to 2 previous errors; 2 warnings emitted
Some errors have detailed explanations: E0308, E0507.
For more information about an error, try `rustc --explain E0308`.
error: could not compile `robyn`
I have tried wrapping the properties of the struct in Arc, Locks but still am unable to figure out the solution.
Any help would be highly appreciated. Thanks in advance.

How to write an asynchronous recursive walkdir function with an asynchronous callback

I'm trying to write an async function that will traverse the filesystem tree, recursively, and calls an asynchronous callback for each file found.
This is for a learning effort, I have no real use case.
Here is what I have so far:
use async_std::{
fs::{self, *},
path::*,
prelude::*,
}; // 1.5.0, features = ["unstable"]
use futures::{
executor::block_on,
future::{BoxFuture, FutureExt},
}; // 0.3.4
use std::{marker::Sync, pin::Pin};
fn main() {
fn walkdir<F>(path: String, cb: &'static F) -> BoxFuture<'static, ()>
where
F: Fn(&DirEntry) -> BoxFuture<()> + Sync + Send,
{
async move {
let mut entries = fs::read_dir(&path).await.unwrap();
while let Some(path) = entries.next().await {
let entry = path.unwrap();
let path = entry.path().to_str().unwrap().to_string();
if entry.path().is_file().await {
cb(&entry).await
} else {
walkdir(path, cb).await
}
}
}
.boxed()
}
let foo = async {
walkdir(".".to_string(), &|entry: &DirEntry| async {
async_std::println!(">> {}\n", &entry.path().to_str().unwrap()).await
})
.await
};
block_on(foo);
}
I get this far by some sort of trial and error, but now I'm stuck on async closure callback with this error
warning: unused import: `path::*`
--> src/main.rs:3:5
|
3 | path::*,
| ^^^^^^^
|
= note: `#[warn(unused_imports)]` on by default
warning: unused import: `pin::Pin`
--> src/main.rs:10:25
|
10 | use std::{marker::Sync, pin::Pin};
| ^^^^^^^^
error[E0308]: mismatched types
--> src/main.rs:33:54
|
33 | walkdir(".".to_string(), &|entry: &DirEntry| async {
| ______________________________________________________^
34 | | async_std::println!(">> {}\n", &entry.path().to_str().unwrap()).await
35 | | })
| |_________^ expected struct `std::pin::Pin`, found opaque type
|
= note: expected struct `std::pin::Pin<std::boxed::Box<dyn core::future::future::Future<Output = ()> + std::marker::Send>>`
found opaque type `impl core::future::future::Future`
use async_std::{
fs::{self, *},
path::*,
prelude::*,
}; // 1.5.0
use futures::{future::{Future, FutureExt, LocalBoxFuture}, executor}; // 0.3.4
fn main() {
async fn walkdir<R>(path: impl AsRef<Path>, mut cb: impl FnMut(DirEntry) -> R)
where
R: Future<Output = ()>,
{
fn walkdir_inner<'a, R>(path: &'a Path, cb: &'a mut dyn FnMut(DirEntry) -> R) -> LocalBoxFuture<'a, ()>
where
R: Future<Output = ()>,
{
async move {
let mut entries = fs::read_dir(path).await.unwrap();
while let Some(path) = entries.next().await {
let entry = path.unwrap();
let path = entry.path();
if path.is_file().await {
cb(entry).await
} else {
walkdir_inner(&path, cb).await
}
}
}.boxed_local()
}
walkdir_inner(path.as_ref(), &mut cb).await
}
executor::block_on({
walkdir(".", |entry| async move {
async_std::println!(">> {}", entry.path().display()).await
})
});
}
Notable changes:
Take in AsRef<Path> instead of a String and a generic closure instead of a trait object reference
Change the closure type to be FnMut as it's more permissive
The closure returns any type that is a future.
There's an inner implementation function that hides the ugly API required for recursive async functions.
The callback takes the DirEntry by value instead of by reference.
See also:
How to asynchronously explore a directory and its sub-directories?
How to using async fn callback in rust

Lifetime issue with CpuPool

I'm writing a simple RPC server with tokio and futures-cpupool. The server holds a BTreeMap of boxed closures, with the function name as key. The current implementation is pretty straight-forward:
pub struct SlackerServiceSync<T>
where T: Send + Sync + 'static
{
functions: Arc<BTreeMap<String, RpcFnSync<T>>>,
pool: CpuPool,
}
impl<T> SlackerServiceSync<T>
where T: Send + Sync + 'static
{
pub fn new(functions: Arc<BTreeMap<String, RpcFnSync<T>>>,
threads: usize)
-> SlackerServiceSync<T> {
let pool = CpuPool::new(threads);
SlackerServiceSync { functions, pool }
}
}
impl<T> Service for SlackerServiceSync<T>
where T: Send + Sync + 'static
{
type Request = SlackerPacket<T>;
type Response = SlackerPacket<T>;
type Error = io::Error;
type Future = BoxFuture<Self::Response, Self::Error>;
fn call(&self, req: Self::Request) -> Self::Future {
match req {
SlackerPacket::Request(sreq) => {
debug!("getting request: {:?}", sreq.fname);
if let Some(f) = self.functions.get(&sreq.fname) {
self.pool
.spawn_fn(move || -> FutureResult<T, Self::Error> {
ok(f(&sreq.arguments))
})
.and_then(move |result| {
debug!("getting results");
ok(SlackerPacket::Response(SlackerResponse {
version: sreq.version,
code: RESULT_CODE_SUCCESS,
content_type: sreq.content_type,
serial_id: sreq.serial_id,
result: result,
}))
})
.map_err(|_| io::Error::new(io::ErrorKind::Other, "Oneshot canceled"))
.boxed()
} else {
let error = SlackerError {
version: sreq.version,
code: RESULT_CODE_NOT_FOUND,
serial_id: sreq.serial_id,
};
ok(SlackerPacket::Error(error)).boxed()
}
}
SlackerPacket::Ping(ref ping) => {
ok(SlackerPacket::Pong(SlackerPong { version: ping.version })).boxed()
}
_ => err(io::Error::new(io::ErrorKind::InvalidInput, "Unsupported packet")).boxed(),
}
}
}
I'm currently blocked by this lifetime issue on self.functions.get(&sreq.fname).
error[E0495]: cannot infer an appropriate lifetime for lifetime parameter in function call due to conflicting requirements
--> src/service.rs:103:49
|
103 | if let Some(f) = self.functions.get(&sreq.fname) {
| ^^^
|
note: first, the lifetime cannot outlive the anonymous lifetime #1 defined on the body at 99:55...
--> src/service.rs:99:56
|
99 | fn call(&self, req: Self::Request) -> Self::Future {
| ________________________________________________________^
100 | | match req {
101 | | SlackerPacket::Request(sreq) => {
102 | | debug!("getting request: {:?}", sreq.fname);
... |
133 | | }
134 | | }
| |_____^
note: ...so that reference does not outlive borrowed content
--> src/service.rs:103:34
|
103 | if let Some(f) = self.functions.get(&sreq.fname) {
| ^^^^^^^^^^^^^^
= note: but, the lifetime must be valid for the static lifetime...
note: ...so that the type `[closure#src/service.rs:105:35: 107:36 f:&std::boxed::Box<for<'r> std::ops::Fn(&'r std::vec::Vec<T>) -> T + std::marker::Send + std::marker::Sync>, sreq:packets::SlackerRequest<T>]` will meet its required lifetime bounds
--> src/service.rs:105:26
|
105 | .spawn_fn(move || -> FutureResult<T, Self::Error> {
| ^^^^^^^^
Similar code works without CpuPool. I cannot fully understand the error reported by compiler.
Full code is here
It turns out I need to wrap the closure into an Arc by declaring the RcpFnSync like this:
pub type RpcFnSync<T> = Arc<Fn(&Vec<T>) -> T + Send + Sync + 'static>;
Then clone it before sending to another thread:
fn call(&self, req: Self::Request) -> Self::Future {
match req {
SlackerPacket::Request(sreq) => {
debug!("getting request: {:?}", sreq.fname);
if let Some(fi) = self.functions.get(&sreq.fname) {
let f = fi.clone();
self.pool
.spawn_fn(move || -> FutureResult<Self::Response, Self::Error> {
let result = f(&sreq.arguments);
ok(SlackerPacket::Response(SlackerResponse {
version: sreq.version,
code: RESULT_CODE_SUCCESS,
content_type: sreq.content_type,
serial_id: sreq.serial_id,
result: result,
}))
})
.boxed()
} else {
let error = SlackerError {
version: sreq.version,
code: RESULT_CODE_NOT_FOUND,
serial_id: sreq.serial_id,
};
ok(SlackerPacket::Error(error)).boxed()
}
}
SlackerPacket::Ping(ref ping) => {
ok(SlackerPacket::Pong(SlackerPong { version: ping.version })).boxed()
}
_ => err(io::Error::new(io::ErrorKind::InvalidInput, "Unsupported packet")).boxed(),
}
}

Resources