I have the following code:
use tokio; // 1.7.1
use futures::future::Future;
use std::pin::Pin;
use futures::FutureExt;
use std::marker::PhantomData;
use std::marker::Send;
use std::sync::Arc;
use async_trait::async_trait;
struct Orchestrator {
worker: Box<dyn WorkerT<Box<dyn Fn(i32) -> Pin<Box<dyn Future<Output = i32> + Send>> + Send + Sync>> + Send + Sync>
}
impl Orchestrator {
async fn other_things(&self, num: i32) -> i32{
// Do some async stuff in here
num+1
}
async fn main_stuff(self: Arc<Self>) {
let func = |num: i32| {
let slf = self.clone();
async move {
slf.other_things(num).await
}.boxed()
};
self.worker.do_work(Box::new(func)).await;
}
}
#[async_trait]
trait WorkerT<F: Fn(i32) -> Pin<Box<dyn Future<Output = i32> + Send>> + Send + Sync> {
async fn do_work(& self, func: F);
}
struct Worker<F: Fn(i32) -> Pin<Box<dyn Future<Output = i32> + Send>> + Send + Sync> {
phantom: std::marker::PhantomData<F>
}
#[async_trait]
impl<F: Fn(i32) -> Pin<Box<dyn Future<Output = i32> + Send>> + Send + Sync> WorkerT<F> for Worker<F> {
async fn do_work(& self, func: F) {
for i in 0..5 {
func(i).await;
}
}
}
#[tokio::main]
async fn main() {
let orchestrator = Arc::new(Orchestrator { worker: Box::new(Worker { phantom: PhantomData }) });
orchestrator.main_stuff().await;
}
With the following error:
error[E0597]: `self` does not live long enough
--> src/main.rs:22:23
|
21 | let func = |num: i32| {
| ---------- value captured here
22 | let slf = self.clone();
| ^^^^ borrowed value does not live long enough
...
28 | self.worker.do_work(Box::new(func)).await;
| -------------- cast requires that `self` is borrowed for `'static`
29 | }
| - `self` dropped here while still borrowed
Currently, because the default lifetime of dyn WorkerT... is 'static, it requires that the borrow of self in main_stuff be 'static. I need to relax the lifetime of the worker field but I don't know how. If I change worker to be of type 'Workerinstead ofdyn WorkerT...` this works, but I would much prefer keep it as the trait.
I have tried adding a lifetime to the Orchestrator struct and then giving that lifetime to the worker field, but then it says that my created lifetime needs to outlive 'static.
https://play.rust-lang.org/?version=nightly&mode=debug&edition=2018&gist=7f34fa394d706409942659f1d8ce36c0
You need to clone the Arc before creating the closure so that you can give an Arc to the closure. What you have in your code tries to borrow self, which won't outlive the closure. Cloning the Arc solves this issue as the closure can now own its own Arc pointing at the same value.
For example:
async fn main_stuff(self: Arc<Self>) {
let self2 = self.clone();
let func = move |num: i32| {
let self3 = self2.clone();
async move {
self3.other_things(num).await
}.boxed()
};
self.worker.do_work(Box::new(func)).await;
}
The inner clone is still required so that the type of func implements Fn (otherwise it will implement FnOnce due to moving a captured variable into the future).
Solution
async fn main_stuff(self: Arc<Self>) {
let slf = self.clone();
let func = move |num: i32| {
let slf = slf.clone();
async move { slf.other_things(num).await }.boxed()
};
self.worker.do_work(Box::new(func)).await;
}
Explanation
You are passing a closure that in turns creates async routine. First, the closure it self is static, so it has to take ownership of self. We clone self to slf and add move to the closure, thus closure moved it. Then we have to clone the slf each time and let the async routine own it when it exits the closure.
Related
I'm trying to write a multi-thread TCP server that can handle multiple connections at the same time with tokio.
I want to structure it in an event-driven way, where it's possible to attach one or more closures to a specific event (like new connection, message received, client disconnected etc).
For example:
server.on_message(|msg: String, stream: &mut TcpStream| {
async move {
println!("Recieved {:?}", msg);
stream.write_all(b"Hello\n").await;
}
}).await;
Every connection will receive its own thread which should have read access to a Vec of callbacks.
pub async fn run(&mut self) {
let listener = TcpListener::bind("127.0.0.1:9090").await.unwrap();
loop {
let (mut socket, _) = listener.accept().await.unwrap();
let cb = self.on_message.clone();
tokio::spawn(async move {
Self::process(socket, cb).await;
});
}
}
Unfortunately, my understanding of Rust is still very rudimentary and I run in circles with:
finding the right type to be stored in Vec
finding the right type for function argument that takes closure callback
Whenever I feel like I make progress in one place then I realise the other got messed up. This is the best I could do but it still doesn't work.
type Callback<T> = dyn Fn(T, &mut TcpStream) -> Pin<Box<dyn Future<Output=()> + Send>> + Send + 'static;
unsafe impl<T> Send for TcpStreamCallbackList<T> {}
unsafe impl<T> Sync for TcpStreamCallbackList<T> {}
impl<T> TcpStreamCallbackList<T> {
pub fn new() -> Self {
Self { callbacks: Vec::new() }
}
pub fn push<G: Send + 'static>(&mut self, mut fun: impl Fn(T, &mut TcpStream) -> G + Send + 'static) where G: Future<Output=()> {
self.callbacks.push(Arc::new(Box::new(move |val:T, stream: &mut TcpStream| Box::pin(fun(val, stream)))));
}
pub async fn call(&self, val: T, stream: &mut TcpStream) where T: Clone {
for cb in self.callbacks.iter() {
let _cb = cb.clone();
_cb(val.clone(), stream).await; // B O O M
}
}
}
The above code doesn't compile until I remove .await on the Future returned by callback in the call function (which defeats the purpose).
error[E0277]: `dyn for<'a> Fn(String, &'a mut tokio::net::TcpStream) -> Pin<Box<dyn futures::Future<Output = ()> + std::marker::Send>> + std::marker::Send` cannot be shared between threads safely
--> src/main.rs:94:26
From what I understand the problem is that the retuned Future is not Send.
note: required by a bound in `tokio::spawn`
--> /Users/lukasz/.cargo/registry/src/github.com-1ecc6299db9ec823/tokio-1.25.0/src/task/spawn.rs:163:21
|
163 | T: Future + Send + 'static,
| ^^^^ required by this bound in `tokio::spawn`
I have no idea if my type makes sense and is thread-safe. I also don't know why the compiler thinks the return type is not Send. I'm really stuck here and would appreciate any help.
I've put together one that is slightly simpler but can be spawned (playground). The key is that Callback needs to be Sync in order for &self to be Send. I tried to use the trick mentioned in this comment, but it doesn't appear to work here, nor does making call take &mut self. I wrote more about Send and Sync on that answer.
use std::future::Future;
use std::pin::Pin;
type CallbackFuture<O> = Pin<Box<dyn Future<Output = O> + Send>>;
type Callback<T> = dyn (Fn(T) -> CallbackFuture<()>) + Send + Sync;
pub struct CallbackList<T> {
list: Vec<Box<Callback<T>>>,
}
impl<T> CallbackList<T> {
pub fn new() -> Self {
Self { list: Vec::new() }
}
pub fn push<F>(&mut self, f: F)
where
F: Fn(T) -> CallbackFuture<()>,
F: Send + Sync + 'static,
{
self.list.push(Box::new(f))
}
pub async fn call(&self, t: T)
where
T: Clone,
{
for f in &self.list {
f(t.clone()).await;
}
}
}
#[tokio::main]
async fn main() {
let mut calls = CallbackList::new();
calls.push(|i| {
Box::pin(async move {
println!("{i}");
})
});
calls.push(|i| {
Box::pin(async move {
println!("{}", i + 1);
})
});
let handle = tokio::spawn(async move {
calls.call(34).await;
});
handle.await.unwrap();
}
I have removed as many trait bounds, 'statics, and wrappers as possible, but you may need to add some back depending on what you do with it. Right now it takes T, but it should be possible to separate that into T and &mut TcpStream. If you update your question with a main function that uses all the elements, I can change mine to match. If all else fails, you can use (_, Arc<Mutex<TcpStream>>) as T.
I'm using the below code in my real project. (Obviously this is simplified and one file only for playground.)
Rust explorer playground
As you can see I'm trying to use a sqlx Transaction from one function to a closure.
But I'm stucked.
I don't even know if this (so common Golang pattern) is the best wat to do in Rust. But at least it should work now.
use std::{future::Future, pin::Pin, sync::Arc};
pub trait Trait: Send + Sync + Player + Shirt {}
impl<T: Player + Shirt> Trait for T {}
pub type Lambda<'a, ArgT, ResT> =
dyn Fn(ArgT) -> Pin<Box<dyn Future<Output = Result<ResT, String>> + Send + 'a>> + Sync + 'a;
#[async_trait::async_trait]
pub trait Player: Send + Sync {
async fn player_create<'a>(
&'a self,
_input: &PlayerInput,
lambda: &Lambda<'_, PlayerCreateLambdaArgs<'a>, DomainPlayer>,
) -> Result<DomainPlayer, String>;
}
#[async_trait::async_trait]
pub trait Shirt: Send + Sync {
async fn shirt_get_next_and_increase<'a>(
&'a self,
tx: &'a mut sqlx::PgConnection,
model: String,
) -> Result<i64, String>;
}
pub struct Repo {
pub pool: Arc<sqlx::PgPool>,
}
impl Repo {
pub fn new(pool: Arc<sqlx::PgPool>) -> Self {
Self { pool }
}
}
#[async_trait::async_trait]
impl Player for Repo {
async fn player_create<'a>(
&'a self,
_input: &PlayerInput,
lambda: &Lambda<'_, PlayerCreateLambdaArgs<'a>, DomainPlayer>,
) -> Result<DomainPlayer, String> {
let mut tx = self.pool.begin().await.unwrap();
// use _input here
let shirt_next_value = Box::new(|model: String| {
self::Shirt::shirt_get_next_and_increase(self, &mut tx, model)
});
let domain_player = lambda(PlayerCreateLambdaArgs { shirt_next_value }).await?;
let res =
sqlx::query_as::<_, DomainPlayer>("INSERT INTO player (...) VALUES (...) RETURNING *")
.bind(domain_player.id)
.bind(domain_player.shirt_number)
.fetch_one(&mut *tx)
.await
.unwrap();
Ok(res)
}
}
#[async_trait::async_trait]
impl Shirt for Repo {
async fn shirt_get_next_and_increase<'a>(
&'a self,
_tx: &'a mut sqlx::PgConnection,
_model: String,
) -> Result<i64, String> {
// Here I'm awaiting an async call for DB operations using the same DB transacion of the caller (_tx)...
// use _tx here...
let res = 123;
Ok(res)
}
}
pub struct Needs {
pub command_pg_repo: Arc<dyn Trait>,
}
#[derive(Default)]
pub struct PlayerInput {
pub id: String,
}
#[derive(Debug, Default, Clone, sqlx::FromRow)]
pub struct DomainPlayer {
pub id: String,
pub shirt_number: i64,
}
pub struct PlayerCreateLambdaArgs<'a> {
// other needed fields here
pub shirt_next_value: Box<
dyn FnMut(String) -> Pin<Box<dyn Future<Output = Result<i64, String>> + Send + 'a>>
+ Send
+ Sync
+ 'a,
>,
}
pub struct Handler {
needs: Arc<Needs>,
}
impl Handler {
pub fn new(needs: Arc<Needs>) -> Self {
Self { needs }
}
pub async fn handle(&self, input: &PlayerInput) -> Result<DomainPlayer, String> {
let res = self
.needs
.command_pg_repo
.player_create(&input, &|mut args| {
let input = input;
Box::pin(async move {
let shirt_number = (args.shirt_next_value)("player".to_string()).await?;
let o = DomainPlayer {
id: input.id.to_string(),
shirt_number,
};
Ok(o)
})
})
.await?;
Ok(res)
}
}
#[tokio::main]
async fn main() -> Result<(), String> {
let db_conn = sqlx::PgPool::connect("fake_url").await.unwrap();
let pg_repo = Arc::new(Repo::new(Arc::new(db_conn)));
let needs = Arc::new(Needs {
command_pg_repo: pg_repo,
});
let handler = Handler::new(needs);
let new_player_input = PlayerInput {
id: "abc".to_string(),
};
let player = handler.handle(&new_player_input).await?;
dbg!(player);
Ok(())
}
The error:
error: captured variable cannot escape `FnMut` closure body
--> src/main.rs:62:13
|
57 | let mut tx = self.pool.begin().await.unwrap();
| ------ variable defined here
...
61 | let shirt_next_value = Box::new(|model: String| {
| - inferred to be a `FnMut` closure
62 | self::Shirt::shirt_get_next_and_increase(self, &mut tx, model)
| ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^--^^^^^^^^
| | |
| | variable captured here
| returns a reference to a captured variable which escapes the closure body
|
= note: `FnMut` closures only have access to their captured variables while they are executing...
= note: ...therefore, they cannot allow references to captured variables to escape
error[E0597]: `tx` does not live long enough
--> src/main.rs:62:65
|
55 | lambda: &Lambda<'_, PlayerCreateLambdaArgs<'a>, DomainPlayer>,
| ------ lifetime `'1` appears in the type of `lambda`
...
61 | let shirt_next_value = Box::new(|model: String| {
| --------------- value captured here
62 | self::Shirt::shirt_get_next_and_increase(self, &mut tx, model)
| ^^ borrowed value does not live long enough
...
65 | let domain_player = lambda(PlayerCreateLambdaArgs { shirt_next_value }).await?;
| ---------------- this usage requires that `tx` is borrowed for `'1`
...
76 | }
| - `tx` dropped here while still borrowed
error[E0499]: cannot borrow `tx` as mutable more than once at a time
--> src/main.rs:71:34
|
55 | lambda: &Lambda<'_, PlayerCreateLambdaArgs<'a>, DomainPlayer>,
| ------ lifetime `'1` appears in the type of `lambda`
...
61 | let shirt_next_value = Box::new(|model: String| {
| --------------- first mutable borrow occurs here
62 | self::Shirt::shirt_get_next_and_increase(self, &mut tx, model)
| -- first borrow occurs due to use of `tx` in closure
...
65 | let domain_player = lambda(PlayerCreateLambdaArgs { shirt_next_value }).await?;
| ---------------- this usage requires that `tx` is borrowed for `'1`
...
71 | .fetch_one(&mut *tx)
| ^^ second mutable borrow occurs here
Some errors have detailed explanations: E0499, E0597.
For more information about an error, try `rustc --explain E0499`.
There are 3 main issues with the code (more like 2 and a half):
Closures cannot return references to things the closure owns, or things the closure borrowed mutably (More formally: the return type of the Fn* function/closure family of traits cannot borrow from the closure type itself). One way to fix this is to have tx be moved into the closure, and then moved into the closure's returned Future. Shared ownership can be achieved with std::sync::Arc, and cloning it and moving it when you need to give another task ownership.
Fn closures may be called concurrently, so they cannot use mutable references to things the closure owns or has borrowed, but this code tries to use a &mut tx borrowing from the outer scope inside a Fn closure. (This issue is bypassed with the solution outlined below).
If you want to access something mutably when it is shared, you need to synchronize those accesses so only one thing can actually access it at a time. One way to do this in async Rust is using tokio::sync::Mutex.
Combining the above, putting tx into a Arc<Mutex<...>>, moving .clone()s of the Arc into the tasks that need ownership, and .lock().awaiting whenever you need to access it mutably may fix your problem (it makes it compile at least).
Rust explorer
diff --git a/src/main.rs b/src/main.rs
index 89fc611..6f7d375 100644
--- a/src/main.rs
+++ b/src/main.rs
## -1,5 +1,7 ##
use std::{future::Future, pin::Pin, sync::Arc};
+use tokio::sync::Mutex;
+
pub trait Trait: Send + Sync + Player + Shirt {}
impl<T: Player + Shirt> Trait for T {}
## -42,12 +44,19 ## impl Player for Repo {
_input: &PlayerInput,
lambda: &Lambda<'_, PlayerCreateLambdaArgs<'a>, DomainPlayer>,
) -> Result<DomainPlayer, String> {
- let mut tx = self.pool.begin().await.unwrap();
+ let tx = Arc::new(Mutex::new(self.pool.begin().await.unwrap()));
// use _input here
- let shirt_next_value = Box::new(|model: String| {
- self::Shirt::shirt_get_next_and_increase(self, &mut tx, model)
+ let shirt_next_value = Box::new({
+ let tx = tx.clone();
+ move |model: String| -> Pin<Box<dyn Future<Output = Result<i64, std::string::String>> + Send>> {
+ let tx = tx.clone();
+ Box::pin(async move {
+ self::Shirt::shirt_get_next_and_increase(self, &mut *tx.lock().await, model)
+ .await
+ })
+ }
});
let domain_player = lambda(PlayerCreateLambdaArgs { shirt_next_value }).await?;
## -56,7 +65,7 ## impl Player for Repo {
sqlx::query_as::<_, DomainPlayer>("INSERT INTO player (...) VALUES (...) RETURNING *")
.bind(domain_player.id)
.bind(domain_player.shirt_number)
- .fetch_one(&mut *tx)
+ .fetch_one(&mut *tx.lock().await)
.await
.unwrap();
Note that because a &mut PgConenction is passed to shirt_get_next_and_increase inside the async block, which references the MutexGuard<Transaction<...>> returned by tx.lock().await, the MutexGuard will be held until shirt_get_next_and_increase completes, even while it yeilds (if it yields). This shouldn't be an issue in this code, since it appears to be mostly sequential, and does not access tx until shirt_get_next_and_increase has completed. If this is not what you want, (i.e. if your actual code does access tx concurrently while shirt_get_next_and_increase is running) you could instead have shirt_get_next_and_increase take a &Mutex<Transaction<...>>, and only have it hold the lock when it needs to access the connection.
An sketch of an alternate solution would be to restructure the code so that the Transaction is passed around by value in function arguments and return values, e.g.
pub type Lambda<'a, ArgT, ResT> =
dyn Fn(ArgT) -> Pin<Box<dyn Future<Output = Result<(ResT, ArgT), String>> + Send + 'a>> + Sync + 'a;
// ...
async fn shirt_get_next_and_increase<'a>(
&'a self,
mut tx: sqlx::Transaction<'static, sqlx::PgConnection>,
_model: String,
) -> Result<(i64, sqlx::Transaction<'static, sqlx::PgConnection>), String> {
// ...
Ok((value, tx))
}
This could resolve the borrowing and shared-mutability issues, but may make the API more cumbersome, and may be infeasible for other reasons.
I'm trying to implement a StateMachine in Rust, but I encountered some problems while trying to fire the callback of StateMachine in a spawn thread.
Here is my StateMachine struct. The state is a generic T because I want to use it in many different scenerios, and I use a Vec to store all the callbacks those registered into this StateMachine.
At the very begining, I didn't use the lifetime 'a, but it will run into some lifetime problems, so I add the lifetime 'a by this suggestion: Idiomatic callbacks in Rust
pub struct StateMachine<'a, T> where T:Clone+Eq+'a {
state: RwLock<T>,
listeners2: Vec<Arc<Mutex<ListenerCallback<'a, T>>>>,
}
pub type ListenerCallback<'a, T> = dyn FnMut(T) -> Result<()> + Send + Sync + 'a ;
When the state is changed, the StateMachine will fire all the callbacks, as follows.
pub async fn try_set(&mut self, new_state:T) -> Result<()> {
if (block_on(self.state.read()).deref().eq(&new_state)) {
return Ok(())
}
// todo change the state
// fire every listener in spawn
let mut fire_results = vec![];
for listener in &mut self.listeners2 {
let state = new_state.clone();
let fire_listener = listener.clone();
fire_results.push(tokio::spawn(async move {
let mut guard = fire_listener.lock().unwrap();
guard.deref_mut()(state);
}));
}
// if fire result return Err, return it
for fire_result in fire_results {
fire_result.await?;
}
Ok(())
}
But it will cause a compilation error.
error[E0521]: borrowed data escapes outside of associated function
--> src/taf/taf-core/src/execution/state_machine.rs:54:33
|
15 | impl<'a,T> StateMachine<'a,T> where T:Clone+Eq+Send {
| -- lifetime `'a` defined here
...
34 | pub async fn try_set(&mut self, new_state:T) -> Result<()> {
| --------- `self` is a reference that is only valid in the associated function body
...
54 | let fire_listener = listener.clone();
| ^^^^^^^^^^^^^^^^
| |
| `self` escapes the associated function body here
| argument requires that `'a` must outlive `'static`
##########################################################
The full code is coupled with a lot of business logic, so I rewrite 2 demos as follows, the problems is the same. The first demo fire callback synchronously and it works, the second demo try to fire callback asynchronously, it encounter the same problem: self escapes the associated function body here.
First demo(it works):
use std::alloc::alloc;
use std::ops::DerefMut;
use std::sync::{Arc, Mutex, RwLock};
use anyhow::Result;
use dashmap::DashMap;
struct StateMachine<'a,T> where T:Clone+Eq+'a {
state: T,
listeners: Vec<Box<Callback<'a, T>>>,
}
type Callback<'a, T> = dyn FnMut(T) -> Result<()> + Send + Sync + 'a;
impl<'a, T> StateMachine<'a,T> where T:Clone+Eq+'a {
pub fn new(init_state: T) -> Self {
StateMachine {
state: init_state,
listeners: vec![]
}
}
pub fn add_listener(&mut self, listener: Box<Callback<'a, T>>) -> Result<()> {
self.listeners.push(listener);
Ok(())
}
pub fn set(&mut self, new_state: T) -> Result<()> {
self.state = new_state.clone();
for listener in &mut self.listeners {
listener(new_state.clone());
}
Ok(())
}
}
#[derive(Clone, Eq, PartialEq, Hash)]
enum ExeState {
Waiting,
Running,
Finished,
Failed,
}
struct Execution<'a> {
exec_id: String,
pub state_machine: StateMachine<'a, ExeState>,
}
struct ExecManager<'a> {
all_jobs: Arc<RwLock<DashMap<String, Execution<'a>>>>,
finished_jobs: Arc<RwLock<Vec<String>>>,
}
impl<'a> ExecManager<'a> {
pub fn new() -> Self {
ExecManager {
all_jobs: Arc::new(RwLock::new(DashMap::new())),
finished_jobs: Arc::new(RwLock::new(vec![]))
}
}
fn add_job(&mut self, job_id: String) {
let mut execution = Execution {
exec_id: job_id.clone(),
state_machine: StateMachine::new(ExeState::Waiting)
};
// add listener
let callback_finished_jobs = self.finished_jobs.clone();
let callback_job_id = job_id.clone();
execution.state_machine.add_listener( Box::new(move |new_state| {
println!("listener fired!, job_id {}", callback_job_id.clone());
if new_state == ExeState::Finished || new_state == ExeState::Failed {
let mut guard = callback_finished_jobs.write().unwrap();
guard.deref_mut().push(callback_job_id.clone());
}
Ok(())
}));
let mut guard = self.all_jobs.write().unwrap();
guard.deref_mut().insert(job_id, execution);
}
fn mock_exec(&mut self, job_id: String) {
let mut guard = self.all_jobs.write().unwrap();
let mut exec = guard.deref_mut().get_mut(&job_id).unwrap();
exec.state_machine.set(ExeState::Finished);
}
}
#[test]
fn test() {
let mut manager = ExecManager::new();
manager.add_job(String::from("job_id1"));
manager.add_job(String::from("job_id2"));
manager.mock_exec(String::from("job_id1"));
manager.mock_exec(String::from("job_id2"));
}
Second demo:
use std::alloc::alloc;
use std::ops::DerefMut;
use std::sync::{Arc, Mutex, RwLock};
use anyhow::Result;
use dashmap::DashMap;
use petgraph::algo::astar;
struct StateMachine<'a,T> where T:Clone+Eq+Send+'a {
state: T,
listeners: Vec<Arc<Mutex<Box<Callback<'a, T>>>>>,
}
type Callback<'a, T> = dyn FnMut(T) -> Result<()> + Send + Sync + 'a;
impl<'a, T> StateMachine<'a,T> where T:Clone+Eq+Send+'a {
pub fn new(init_state: T) -> Self {
StateMachine {
state: init_state,
listeners: vec![]
}
}
pub fn add_listener(&mut self, listener: Box<Callback<'a, T>>) -> Result<()> {
self.listeners.push(Arc::new(Mutex::new(listener)));
Ok(())
}
pub fn set(&mut self, new_state: T) -> Result<()> {
self.state = new_state.clone();
for listener in &mut self.listeners {
let spawn_listener = listener.clone();
tokio::spawn(async move {
let mut guard = spawn_listener.lock().unwrap();
guard.deref_mut()(new_state.clone());
});
}
Ok(())
}
}
#[derive(Clone, Eq, PartialEq, Hash)]
enum ExeState {
Waiting,
Running,
Finished,
Failed,
}
struct Execution<'a> {
exec_id: String,
pub state_machine: StateMachine<'a, ExeState>,
}
struct ExecManager<'a> {
all_jobs: Arc<RwLock<DashMap<String, Execution<'a>>>>,
finished_jobs: Arc<RwLock<Vec<String>>>,
}
impl<'a> ExecManager<'a> {
pub fn new() -> Self {
ExecManager {
all_jobs: Arc::new(RwLock::new(DashMap::new())),
finished_jobs: Arc::new(RwLock::new(vec![]))
}
}
fn add_job(&mut self, job_id: String) {
let mut execution = Execution {
exec_id: job_id.clone(),
state_machine: StateMachine::new(ExeState::Waiting)
};
// add listener
let callback_finished_jobs = self.finished_jobs.clone();
let callback_job_id = job_id.clone();
execution.state_machine.add_listener( Box::new(move |new_state| {
println!("listener fired!, job_id {}", callback_job_id.clone());
if new_state == ExeState::Finished || new_state == ExeState::Failed {
let mut guard = callback_finished_jobs.write().unwrap();
guard.deref_mut().push(callback_job_id.clone());
}
Ok(())
}));
let mut guard = self.all_jobs.write().unwrap();
guard.deref_mut().insert(job_id, execution);
}
fn mock_exec(&mut self, job_id: String) {
let mut guard = self.all_jobs.write().unwrap();
let mut exec = guard.deref_mut().get_mut(&job_id).unwrap();
exec.state_machine.set(ExeState::Finished);
}
}
#[test]
fn test() {
let mut manager = ExecManager::new();
manager.add_job(String::from("job_id1"));
manager.add_job(String::from("job_id2"));
manager.mock_exec(String::from("job_id1"));
manager.mock_exec(String::from("job_id2"));
}
Compile error of second demo:
error[E0521]: borrowed data escapes outside of associated function
--> generic/src/callback2.rs:34:34
|
15 | impl<'a, T> StateMachine<'a,T> where T:Clone+Eq+Send+'a {
| -- lifetime `'a` defined here
...
29 | pub fn set(&mut self, new_state: T) -> Result<()> {
| --------- `self` is a reference that is only valid in the associated function body
...
34 | let spawn_listener = listener.clone();
| ^^^^^^^^^^^^^^^^
| |
| `self` escapes the associated function body here
| argument requires that `'a` must outlive `'static`
|
= note: requirement occurs because of the type `std::sync::Mutex<Box<dyn FnMut(T) -> Result<(), anyhow::Error> + Send + Sync>>`, which makes the generic argument `Box<dyn FnMut(T) -> Result<(), anyhow::Error> + Send + Sync>` invariant
= note: the struct `std::sync::Mutex<T>` is invariant over the parameter `T`
= help: see <https://doc.rust-lang.org/nomicon/subtyping.html> for more information about variance
Tasks spawned with tokio::spawn() cannot use borrowed data (here, the data with lifetime 'a, whatever it may be). This is because there is not currently (and likely will never be) any way to guarantee that the borrowed data reliably outlives the spawned task.
You have two choices:
Fire the notifications without spawning. You can put the notification futures into a FuturesUnordered to run them all concurrently, but they will still all have to finish before try_set() does.
Remove the lifetime parameter; stop allowing callbacks that borrow data. Put 'static on your dyn types where necessary. Change the users of the StateMachine so they do not try to use borrowed data but use Arc instead, if necessary.
pub struct StateMachine<T> where T: Clone + Eq + 'static {
state: RwLock<T>,
listeners2: Vec<Arc<Mutex<ListenerCallback<T>>>>,
}
pub type ListenerCallback<T> = dyn FnMut(T) -> Result<()> + Send + Sync + 'static;
I'm trying to refactor code by moving the hyper server out of main and allow the struct function to accept a function as below:
// main.rs
#[tokio::main]
async fn main() {
let listener = Listener::new(String::from("127.0.0.1"), String::from("3000"));
listener.start(request_handler);
}
async fn request_handler(req: Request<Body>) -> ConfigResult<Response<Body>> {
match req.method() {
&Method::GET => get(req.body()),
_ => Ok(Response::builder()
.status(StatusCode::NOT_FOUND)
.body("NOT FOUND".into())
.unwrap())
}
}
// lib.rs
pub type ConfigError = Box<dyn Error + Send + Sync>;
pub type ConfigResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
#[derive(Debug, Clone)]
pub struct Listener(pub SocketAddr);
impl<'a> Listener {
pub fn new(addr: String, port:String) -> Self {
let ip = IpAddr::from_str(addr.as_str()).unwrap();
let address = SocketAddr::new(ip, port.parse::<u16>().unwrap());
Listener(address)
}
pub async fn start<F>(&'a self, request_handler: F ) -> ConfigResult<()>
where F: Fn(Request<Body>) -> ConfigResult<Response<Body>> + Sync + 'a
{
let mut service_closure = move |req| {
async {
request_handler(req)
}
};
let mut make_service_closure = move |_| {
async {
Ok::<_, ConfigError>(service_fn(&service_closure))
}
};
let service = make_service_fn(make_service_closure);
let server = Server::bind(&self.0).serve(service);
server.await?;
Ok(())
}
}
Im getting the error:
error: implementation of `FnOnce` is not general enough
--> src/config/server.rs:36:15
|
36 | server.await?;
| ^^^^^^ implementation of `FnOnce` is not general enough
|
= note: closure with signature `fn(&'2 AddrStream) -> impl Future<Output = Result<hyper::service::util::ServiceFn<&[closure#src/config/serva.rs:23:35: 27:10], Body>, Box<dyn StdError + Send + Sync>>>` must implement `FnOnce<(&'1 AddrStream,)>`, for any lifetime `'1`...
= note: ...but it actually implements `FnOnce<(&'2 AddrStream,)>`, for some specific lifetime `'2`
Looking at the error above, my conclusion is its a lifetime error, where let make_service_closure = move |_| {...} requires lifetime of '1 but when i try let make_service_closure = move | a: &'a AddrStream| {...} i get lifetime mismatch error
expected type `for<'r> FnMut<(&'r AddrStream,)>`
found type `FnMut<(&'a AddrStream,)>`
plus the error implementation of FnOnce is not general enough. Please help understand which lifetime should i use for the hyper service function and why is FnOnce is not general enough.
In the example code below, a non-send value, Vec<T>, is moved into a function that returns something else. At this point, I no longer care about that vector. The returned object stores no reference to it, it no longer exists.
However, when I .await on the next line I get the error "captured value is not Send". Which it isn't, but since it should have been destroyed when vector_as_string exited, it doesn't need to send it across threads when the future restarts, because that variable is never used again.
use async_trait::async_trait;
async fn write_value(value: Vec<u8>) {
println!("something")
}
fn vector_as_string<T>(vec: Vec<T>) -> Vec<u8> {
Vec::new()
}
#[async_trait]
trait Writer {
async fn write_array<T>(&mut self, value: Vec<T>);
}
pub struct WriterImplementation {}
#[async_trait]
impl Writer for WriterImplementation {
async fn write_array<T>(&mut self, value: Vec<T>) {
let string = vector_as_string(value);
write_value(string).await
}
}
#[tokio::main]
async fn main() {
println!("Hi");
}
Dependencies:
[dependencies]
tokio = { version = "1.9.0", features = ["full"]}
async-trait = "0.1.51"
Error:
error: future cannot be sent between threads safely
--> src/main.rs:20:55
|
20 | async fn write_array<T>(&mut self, value: Vec<T>) {
| _______________________________________________________^
21 | | let string = vector_as_string(value);
22 | |
23 | | write_value(string).await
24 | | }
| |_____^ future created by async block is not `Send`
|
note: captured value is not `Send`
--> src/main.rs:20:40
|
20 | async fn write_array<T>(&mut self, value: Vec<T>) {
| ^^^^^ has type `Vec<T>` which is not `Send`
= note: required for the cast to the object type `dyn Future<Output = ()> + Send`
help: consider further restricting this bound
|
20 | async fn write_array<T + std::marker::Send>(&mut self, value: Vec<T>) {
| ^^^^^^^^^^^^^^^^^^^
Adding T: Send as it suggests allows it to compile, but why does T need to be Send if we aren't holding any T's across the await, as it has already been moved?
From the async_trait documentation:
Async fns get transformed into methods that return Pin<Box<dyn Future + Send + 'async>> and delegate to a private async freestanding function.
Not all async traits need futures that are dyn Future + Send. To avoid having Send and Sync bounds placed on the async trait methods, invoke the async trait macro as #[async_trait(?Send)] on both the trait and the impl blocks.
Applied to your case:
#[async_trait(?Send)]
trait Writer {
async fn write_array<T>(&mut self, value: Vec<T>);
}
#[async_trait(?Send)]
impl Writer for WriterImplementation {
async fn write_array<T>(&mut self, value: Vec<T>) {
let string = vector_as_string(value);
write_value(string).await
}
}