Transferring memory between processes - rust

I want to transfer an item between processes (specifically a TcpStream). I attempted to do this with a more complex example however failed in this. So I am trying to do this with a simpler example:
My current attempt is:
use bytes::{Buf, BytesMut};
use tokio::{
io::AsyncWriteExt,
net::{tcp::OwnedWriteHalf, TcpListener, TcpStream},
};
use tokio_stream::StreamExt;
use tokio_util::codec::{Decoder, FramedRead};
const TRANSFER_ADDRESS: &str = "127.0.0.1:8081";
#[tokio::main]
async fn main() {
if let Ok(existing_server_process_listener) =
TcpStream::connect(TRANSFER_ADDRESS).await
{
let (read, _write) = existing_server_process_listener.into_split();
let mut reader = FramedRead::new(read, DatabaseTransferDecoder);
let bytes = reader.next().await.unwrap().unwrap();
dbg!(&bytes);
let addr = usize::from_ne_bytes(bytes.try_into().unwrap());
dbg!(addr);
let ptr = addr as *mut u32;
dbg!(ptr);
let box_ptr = unsafe { Box::from_raw(ptr)};
dbg!(box_ptr);
}
else {
let update_listener = TcpListener::bind(TRANSFER_ADDRESS).await.unwrap();
let (stream, _socket) = update_listener.accept().await.unwrap();
// Splits our stream into read/write
let (_read, mut write) = stream.into_split();
let x = Box::new(4u32);
let ptr = Box::into_raw(x);
dbg!(ptr);
let addr = ptr as usize;
dbg!(addr);
let bytes = addr.to_ne_bytes();
dbg!(bytes);
write_fn(&mut write,&bytes).await;
}
}
async fn write_fn(writer: &mut OwnedWriteHalf, string: &[u8]) {
let buffer = {
let mut b = Vec::with_capacity(4 + string.len());
b.extend_from_slice(&u32::to_le_bytes(u32::try_from(string.len()).unwrap()));
b.extend_from_slice(string);
b
};
writer.write_all(&buffer).await.unwrap();
}
struct DatabaseTransferDecoder;
/// Implement `tokio_util::codec::decoder` for framed reads.
impl Decoder for DatabaseTransferDecoder {
type Error = std::io::Error;
type Item = Vec<u8>;
fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
// We use u32 for len marker, so need an initial 4 bytes
if src.len() < 4 {
return Ok(None);
}
// Get length
let length = {
let mut length_bytes = [0; 4];
length_bytes.copy_from_slice(&src[..4]);
u32::from_le_bytes(length_bytes) as usize
};
if src.len() < 4 + length {
src.reserve(4 + length - src.len());
return Ok(None);
}
// Read data
let data = src[4..4 + length].to_vec();
// Advance buffer to discard data
src.advance(4 + length);
Ok(Some(data))
}
}
Running the first process then running the second process causes the first process to output:
[src\main.rs:33] ptr = 0x000001f8b8435980
[src\main.rs:35] addr = 2167754938752
[src\main.rs:37] bytes = [
128,
89,
67,
184,
248,
1,
0,
0,
]
and the second process to output:
[src\main.rs:18] &bytes = [
128,
89,
67,
184,
248,
1,
0,
0,
]
[src\main.rs:20] addr = 2167754938752
[src\main.rs:22] ptr = 0x000001f8b8435980
[src\main.rs:24] box_ptr = error: process didn't exit successfully: `target\debug\testing-bin.exe` (exit code: 0xc0000005, STATUS_ACCESS_VIOLATION)
Why would accessing this memory cause this error?

Related

Async read_to_end results in empty buffer even though number of read bytes is > 0

I'm writing an async (tokio) program that transparently reads from regular files, gzipped files and gzipped tar archives. It works for the first two cases but I'm having trouble with reading from tar files using tokio-tar. What happens is that the number of bytes read is greater that 0 but the buffer stays empty anyways.
The actual program is quite big so here is some code that presents the problem in an isolated form.
#![feature(let_chains)]
use std::path::Path;
use async_compression::tokio::bufread::GzipDecoder;
use futures::StreamExt;
use tokio::{
fs::File,
io::{AsyncReadExt, BufReader},
};
use tokio_tar::{Archive, Entry};
#[derive(Debug)]
pub enum Handle {
File(BufReader<File>),
Gz(GzipDecoder<BufReader<File>>),
TarGz(Entry<Archive<GzipDecoder<BufReader<File>>>>),
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
// prepare the reader
let f = File::open("./archive.tar.gz").await?;
let gz = GzipDecoder::new(BufReader::new(f));
let mut archive = Archive::new(gz);
// the path of the file to read within the archive
let path_wanted = Path::new("tar/archive.txt");
let mut entries = archive.entries()?;
let mut handle: Option<Handle> = None;
// iterate through the files in the archive until the desired file is found
while let Some(entry_result) = entries.next().await {
if let Ok(entry) = entry_result
&& let Ok(path) = entry.path()
&& path == path_wanted
{
handle = Some(Handle::TarGz(entry));
}
}
// use the assigned variable to read the archive entry's contents
if let Some(mut tgz) = handle {
match tgz {
Handle::TarGz(ref mut tgz) => {
let mut vec_buf = vec![0; 10];
loop {
match tgz.take(10).read_to_end(&mut vec_buf).await {
// here the actual problem becomes visible
// while reading the entry, n is greater than 0 but nothing is written to the buffer
Ok(n) if n > 0 => {
println!("n: {} - {:?}", n, &vec_buf[..n]);
}
_ => break,
}
}
}
_ => panic!("case not covered"),
}
} else {
println!("path_wanted not found in archive");
}
Ok(())
}
The file tar/archive.txt inside the tar archive looks like:
127.0.0.1 test.com
127.0.0.1 test.de
The key point of the code snippet is that the Handle is assigned to a variable (actually a field of a struct in the real program) which is the used further down in the program.
The output looks like:
tarvester on  master [?] is 📦 v0.1.0 via 🦀 v1.66.0-nightly
❯ cargo run
Compiling tarvester v0.1.0 (/home/bob/code/gardion/tarvester)
Finished dev [unoptimized + debuginfo] target(s) in 0.54s
Running `target/debug/tarvester`
n: 10 - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
n: 10 - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
n: 10 - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
n: 7 - [0, 0, 0, 0, 0, 0, 0]
As you can see it's apparently reading the correct number of bytes but the buffer still stays empty.
A modified version of the code (which is not practical for the architecture of my program) where the assigned handle is used right after assignment to the variable kind of works:
#![feature(let_chains)]
use std::path::Path;
use async_compression::tokio::bufread::GzipDecoder;
use futures::StreamExt;
use tokio::{
fs::File,
io::{AsyncReadExt, BufReader},
};
use tokio_tar::{Archive, Entry};
#[derive(Debug)]
pub enum Handle {
File(BufReader<File>),
Gz(GzipDecoder<BufReader<File>>),
TarGz(Entry<Archive<GzipDecoder<BufReader<File>>>>),
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
let f = File::open("../harvester/archive.tar.gz").await?;
let gz = GzipDecoder::new(BufReader::new(f));
let mut archive = Archive::new(gz);
let path_wanted = Path::new("tar/archive.txt");
let mut entries = archive.entries()?;
let mut handle: Option<Handle> = None;
while let Some(entry_result) = entries.next().await {
if let Ok(entry) = entry_result
&& let Ok(path) = entry.path()
&& path == path_wanted
{
handle = Some(Handle::TarGz(entry));
// !! the handle assigned to the variable is used right away without leaving the block
match handle.as_mut().unwrap() {
Handle::TarGz(tgz) => {
let mut vec_buf = vec![0; 10];
loop {
match tgz.take(10).read_to_end(&mut vec_buf).await {
// some messed up output shows up here
Ok(n) if n > 0 => {
println!("{}", String::from_utf8_lossy(&vec_buf[..]));
},
_ => break
}
}
},
_ => panic!("case not covered"),
}
}
}
Ok(())
}
The latter code produces output which looks a bit messed up though:
127.0.0.1
127.0.0.1 test.com
1
127.0.0.1 test.com
127.0.0.1 t
127.0.0.1 test.com
127.0.0.1 test.de
Can somebody spot what I'm doing wrong here? Or is this maybe a bug in tokio-tar? If it's more likely a bug in the dependency, I'd also appreciate suggestions on how to go on with the debugging even if I need to dive further into tokio-tar.
A working example:
#![feature(let_chains)]
use std::path::Path;
use async_compression::tokio::bufread::GzipDecoder;
use futures::StreamExt;
use tokio::{
fs::File,
io::{AsyncReadExt, BufReader},
};
use tokio_tar::{Archive, Entry};
#[derive(Debug)]
pub enum Handle {
File(BufReader<File>),
Gz(GzipDecoder<BufReader<File>>),
TarGz(Entry<Archive<GzipDecoder<BufReader<File>>>>),
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
// prepare the reader
let f = File::open("./archive.tar.gz").await?;
let gz = GzipDecoder::new(BufReader::new(f));
let mut archive = Archive::new(gz);
// the path of the file to read within the archive
let path_wanted = Path::new("tar/archive.txt");
let mut entries = archive.entries()?;
let mut handle: Option<Handle> = None;
// iterate through the files in the archive until the desired file is found
while let Some(entry_result) = entries.next().await {
if let Ok(entry) = entry_result
&& let Ok(path) = entry.path()
&& path == path_wanted
{
handle = Some(Handle::TarGz(entry));
break;
}
}
// use the assigned variable to read the archive entry's contents
if let Some(mut tgz) = handle {
match tgz {
Handle::TarGz(ref mut tgz) => {
let mut vec_buf = Vec::with_capacity(10);
loop {
match tgz.take(10).read_to_end(&mut vec_buf).await {
// here the actual problem becomes visible
// while reading the entry, n is greater than 0 but nothing is written to the buffer
Ok(n) if n > 0 => {
println!("n: {} - {:?}", n, &vec_buf[..n]);
}
_ => break,
}
}
}
_ => panic!("case not covered"),
}
} else {
println!("path_wanted not found in archive");
}
Ok(())
}
There is two problem in your code.
First, the above loop might not do what you expect because the entry_result variable is actually a single variable that takes on the value of each item. When the stream terminates, the handle inner value actually point to the last item in stream. That's why I add a break there.
Second, read_to_end will actually append the data to vec_buf instead of starting at 0.
All bytes read from this source will be appended to the specified buffer buf. This function will continuously call read() to append more data to buf until read() returns Ok(0).
Or you may consider to use filter:
#![feature(let_chains)]
use std::path::Path;
use async_compression::tokio::bufread::GzipDecoder;
use futures::{future, StreamExt};
use tokio::{
fs::File,
io::{AsyncReadExt, BufReader},
};
use tokio_tar::{Archive, Entry};
#[derive(Debug)]
pub enum Handle {
File(BufReader<File>),
Gz(GzipDecoder<BufReader<File>>),
TarGz(Entry<Archive<GzipDecoder<BufReader<File>>>>),
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
// prepare the reader
let f = File::open("./archive.tar.gz").await?;
let gz = GzipDecoder::new(BufReader::new(f));
let mut archive = Archive::new(gz);
// the path of the file to read within the archive
let path_wanted = Path::new("tar/archive.txt");
let entries = archive.entries()?;
// iterate through the files in the archive until the desired file is found
let handle = entries
.filter(|entry_result| {
future::ready(if let Ok(entry) = entry_result
&& let Ok(path) = entry.path()
&& path == path_wanted {
true
} else {
false
})
})
.next()
.await
.map(|entry| Handle::TarGz(entry.unwrap()));
// use the assigned variable to read the archive entry's contents
if let Some(mut tgz) = handle {
match tgz {
Handle::TarGz(ref mut tgz) => {
let mut vec_buf = Vec::with_capacity(10);
loop {
match tgz.take(10).read_to_end(&mut vec_buf).await {
// here the actual problem becomes visible
// while reading the entry, n is greater than 0 but nothing is written to the buffer
Ok(n) if n > 0 => {
println!("n: {} - {:?}", n, &vec_buf[..n]);
}
_ => break,
}
}
}
_ => panic!("case not covered"),
}
} else {
println!("path_wanted not found in archive");
}
Ok(())
}

Use par_split on a String, process using rayon and collect result in a Vector

I am trying to read a file into a string messages defined on line #14. The file contains several blocks where each block starts with a number. After I read the file contents into the string messahes, each block is separated by newline and each line in a block is separated by __SEP__. I would like to use par_split() on the string messages, process each block using rayon and collect output from each block into a vector vec_finale.g. by calling collect() on line 54 or some similar mechanism to produce a vector that contains vec_local on line 53 produced by each block. Any pointers on how I can achieve this are highly appreciated.
My code is as follows:
fn starts_with_digit_or_at_sign(inp: &str) -> bool {
let mut at_sign_found = false;
if inp.len() > 0 {
let ch = inp.chars().next().unwrap();
if ch.is_numeric() || ch == '#' {
return true;
}
}
return false;
}
fn main() {
let filepath = "inp.log";
let data = std::fs::read_to_string(filepath).expect("file not found!");
let mut messages: String = String::from("");
let separator_char = '\n';
let separator: String = String::from("__SEP__");
let mut found_first_message = false;
let mut start_of_new_msg = false;
let mut line_num = 0;
for line in data.lines() {
line_num += 1;
if line.len() > 0 {
if starts_with_digit_or_at_sign(line) {
start_of_new_msg = true;
if !found_first_message {
found_first_message = true;
} else {
messages.push(separator_char);
}
}
if found_first_message {
if !start_of_new_msg {
messages.push_str(&separator);
}
messages.push_str(line);
if start_of_new_msg {
start_of_new_msg = false;
let mut tmp = String::from("Lnumber ");
tmp.push_str(&line_num.to_string());
messages.push_str(&separator);
messages.push_str(&tmp);
}
}
}
}
messages.par_split(separator_char).for_each(|l| {
println!(
"line: '{}' len: {}, {}",
l,
l.len(),
rayon::current_num_threads()
);
let vec_local: Vec<i32> = vec![l.len() as i32];
}); // <-- line 54
}
Output produced by the cide is as follows:
line: '1__SEP__Lnumber 1__SEP__a__SEP__b__SEP__c' len: 41, 8
line: '3__SEP__Lnumber 9__SEP__g__SEP__h__SEP__i' len: 41, 8
line: '2__SEP__Lnumber 5__SEP__d__SEP__e__SEP__f' len: 41, 8
line: '4__SEP__Lnumber 13__SEP__j__SEP__k__SEP__l' len: 42, 8
File inp.log is as follows:
1
a
b
c
2
d
e
f
3
g
h
i
4
j
k
l
I was able to resolve the issue by using par_lines() instead as follows:
let tmp: Vec<_> = messages.par_lines().map(|l| proc_len(l)).collect();
...
...
...
fn proc_len(inp: &str) -> Vec<usize> {
let vec: Vec<usize> = vec![inp.len()];
return vec;
}

Reading serial data using interrupt

I am trying to read a series of bytes from the serial pin, the interrupt is being called but I only read the first byte of the data each time.
I am using the library stm32f1xx-hal.
The idea is to store the data I receive in a buffer to use it whenever I need it later on the program. I got inspired by the example in the library.
Here is my code :
const BUFFER_LEN: usize = 20;
static mut BUFFER: &mut [u8; BUFFER_LEN] = &mut [0; BUFFER_LEN];
static mut WIDX: usize = 0;
static mut RX: Option<Rx<USART1>> = None;
static mut RX_PIN : MaybeUninit<stm32f1xx_hal::gpio::gpioa::PA10<Input<Floating>>> = MaybeUninit::uninit();
#[interrupt]
unsafe fn USART1() {
INT_COUNTER += 1;
cortex_m::interrupt::free(|_| {
if let Some(rx) = RX.as_mut() {
while rx.is_rx_not_empty() {
if let Ok(w) = nb::block!(rx.read()) {
BUFFER[WIDX] = w;
WIDX += 1;
if WIDX >= BUFFER_LEN - 1 {
WIDX = 0;
}
}
rx.listen_idle();
}
if rx.is_idle() {
rx.unlisten_idle();
WIDX = 0;
}
}
})
}
#[entry]
fn main() -> ! {
let mut dp: Peripherals = pac::Peripherals::take().unwrap();
let cp = cortex_m::Peripherals::take().unwrap();
let mut rcc: Rcc = dp.RCC.constrain();
let mut gpioc = dp.GPIOC.split();
let mut flash = dp.FLASH.constrain();
let mut gpioa = dp.GPIOA.split();
let mut afio = dp.AFIO.constrain();
let clocks_serial = rcc.cfgr.freeze(&mut flash.acr);
// USART1 on Pins A9 and A10
let pin_tx = gpioa.pa9.into_alternate_push_pull(&mut gpioa.crh);
let mut pin_rx = gpioa.pa10;
unsafe {
pac::NVIC::unmask(pac::Interrupt::USART1);
}
let serial = Serial::usart1(
dp.USART1,
(pin_tx, pin_rx),
&mut afio.mapr,
Config::default().baudrate(115200.bps()), // baud rate defined in herkulex doc : 115200
clocks_serial.clone(),
// &mut rcc.apb2,
);
// separate into tx and rx channels
let (mut tx, mut rx) = serial.split();
rx.listen();
rx.listen_idle();
cortex_m::interrupt::free(|_| unsafe {
RX.replace(rx);
});
loop {
// It is my sender, I can confirm I receive the message on the logical analyzer.
let message = servo.stat();
send_message(message, &mut tx);
cortex_m::asm::wfi(); // important
unsafe {
hprintln!("{:?}", BUFFER);
}
}
}

Segmentation fault in Vulkan on vkMapMemory call / (device.map_memory in Erupt)

I'm using Rust and the Rust-based Vulkan wrapper Erupt.
Everything in this code runs fine until the device.map_memory call. No VulkanValidation warnings or errors are raised. Just a segmentation fault. I have offsets set to zero, but this is the only memory usage of this buffer, so I'm not sure if the issue is offsets or something else.
use memoffset::offset_of;
use simple_logger::SimpleLogger;
use winit::{
event::{
Event, KeyboardInput, WindowEvent,
ElementState, StartCause, VirtualKeyCode
},
event_loop::{ControlFlow, EventLoop},
window::WindowBuilder,
window::Window
};
// use winit::event::{ElementState, StartCause, VirtualKeyCode};
use structopt::StructOpt;
use erupt::{
cstr,
utils::{self, surface},
vk, DeviceLoader, EntryLoader, InstanceLoader,
};
use std::{
ffi::{c_void, CStr, CString},
mem::*,
os::raw::c_char,
ptr,
result::Result,
result::Result::*,
string::String,
thread,
time,
};
use raw_window_handle::{HasRawWindowHandle, RawWindowHandle};
const TITLE: &str = "Peregrine Ray-Trace";
const FRAMES_IN_FLIGHT: usize = 2;
const LAYER_KHRONOS_VALIDATION: *const c_char = cstr!("VK_LAYER_KHRONOS_validation");
#[derive(Debug, StructOpt)]
struct Opt {
/// Use validation layers
#[structopt(short, long)]
validation_layers: bool,
}
unsafe extern "system" fn debug_callback(
_message_severity: vk::DebugUtilsMessageSeverityFlagBitsEXT,
_message_types: vk::DebugUtilsMessageTypeFlagsEXT,
p_callback_data: *const vk::DebugUtilsMessengerCallbackDataEXT,
_p_user_data: *mut c_void,
) -> vk::Bool32 {
eprintln!(
"Vulkan: {}",
CStr::from_ptr((*p_callback_data).p_message).to_string_lossy()
);
vk::FALSE
}
#[repr(C)]
#[derive(Debug, Clone, Copy)]
struct VertexV3 {
pos: [f32; 4],
color: [f32; 4],
}
impl VertexV3 {
fn get_binding_descriptions() -> [vk::VertexInputBindingDescription; 1] {
[vk::VertexInputBindingDescription {
binding: 0,
stride: std::mem::size_of::<Self>() as u32,
input_rate: vk::VertexInputRate::VERTEX,
}]
}
pub fn get_attribute_descriptions() -> [vk::VertexInputAttributeDescription; 2] {
[
vk::VertexInputAttributeDescription {
binding: 0,
location: 0,
format: vk::Format::R32G32B32A32_SFLOAT,
offset: offset_of!(Self, pos) as u32,
},
vk::VertexInputAttributeDescription {
binding: 0,
location: 1,
format: vk::Format::R32G32B32A32_SFLOAT,
offset: offset_of!(Self, color) as u32,
},
]
}
}
fn main() {
println!("Ray-Peregrine Lab 24::::!!!!");
let mut vulkan_output = String::from("");
println!("Ray-EE-Oct-12");
let event_loop = EventLoop::new();
let window = WindowBuilder::new()
.with_title("Peregrine Ray-Trace")
.build(&event_loop)
.unwrap();
let entry = EntryLoader::new().unwrap();
// https://vulkan-tutorial.com/Drawing_a_triangle/Setup/Instance
let application_name = CString::new("Peregrine Ray-Trace").unwrap();
let engine_name = CString::new("Vulkan Engine").unwrap();
let app_info = vk::ApplicationInfoBuilder::new()
.application_name(&application_name)
.application_version(vk::make_api_version(0, 1, 0, 0))
.engine_name(&engine_name)
.engine_version(vk::make_api_version(0, 1, 0, 0))
.api_version(vk::make_api_version(0, 1, 0, 0));
let mut instance_extensions = surface::enumerate_required_extensions(&window).unwrap();
instance_extensions.push(vk::EXT_DEBUG_UTILS_EXTENSION_NAME);
let mut instance_layers = Vec::new();
instance_layers.push(LAYER_KHRONOS_VALIDATION);
let device_extensions = vec![
vk::KHR_SWAPCHAIN_EXTENSION_NAME,
vk::KHR_RAY_TRACING_PIPELINE_EXTENSION_NAME,
vk::KHR_RAY_QUERY_EXTENSION_NAME,
vk::KHR_DEFERRED_HOST_OPERATIONS_EXTENSION_NAME,
vk::KHR_ACCELERATION_STRUCTURE_EXTENSION_NAME,
vk::KHR_SPIRV_1_4_EXTENSION_NAME,
];
let mut device_layers = Vec::new();
device_layers.push(LAYER_KHRONOS_VALIDATION);
let instance_info = vk::InstanceCreateInfoBuilder::new()
.application_info(&app_info)
.enabled_extension_names(&instance_extensions)
.enabled_layer_names(&instance_layers);
let instance = unsafe { InstanceLoader::new(&entry, &instance_info, None) }.unwrap();
// https://vulkan-tutorial.com/Drawing_a_triangle/Setup/Validation_layers
// if opt.validation_layers ...
let messenger = {
let messenger_info = vk::DebugUtilsMessengerCreateInfoEXTBuilder::new()
.message_severity(
vk::DebugUtilsMessageSeverityFlagsEXT::VERBOSE_EXT
| vk::DebugUtilsMessageSeverityFlagsEXT::WARNING_EXT
| vk::DebugUtilsMessageSeverityFlagsEXT::ERROR_EXT,
)
.message_type(
vk::DebugUtilsMessageTypeFlagsEXT::GENERAL_EXT
| vk::DebugUtilsMessageTypeFlagsEXT::VALIDATION_EXT
| vk::DebugUtilsMessageTypeFlagsEXT::PERFORMANCE_EXT,
)
.pfn_user_callback(Some(debug_callback));
unsafe { instance.create_debug_utils_messenger_ext(&messenger_info, None) }.unwrap()
};
let surface = unsafe { surface::create_surface(&instance, &window, None) }.unwrap();
let (physical_device, queue_family, format, present_mode, device_properties) =
unsafe { instance.enumerate_physical_devices(None) }
.unwrap()
.into_iter()
.filter_map(|physical_device| unsafe {
// println!("Physical Device: {:?}", physical_device);
// println!("Phyisical Device Queue Family Properties: {:?}", instance.get_physical_device_properties(physical_device));
let queue_family = match instance
.get_physical_device_queue_family_properties(physical_device, None)
.into_iter()
.enumerate()
.position(|(i, queue_family_properties)| {
queue_family_properties
.queue_flags
.contains(vk::QueueFlags::GRAPHICS)
&& instance
.get_physical_device_surface_support_khr(
physical_device,
i as u32,
surface,
)
.unwrap()
}) {
Some(queue_family) => queue_family as u32,
None => return None,
};
let formats = instance
.get_physical_device_surface_formats_khr(physical_device, surface, None)
.unwrap();
let format = match formats
.iter()
.find(|surface_format| {
surface_format.format == vk::Format::B8G8R8A8_SRGB
&& surface_format.color_space == vk::ColorSpaceKHR::SRGB_NONLINEAR_KHR
})
.or_else(|| formats.get(0))
{
Some(surface_format) => *surface_format,
None => return None,
};
let present_mode = instance
.get_physical_device_surface_present_modes_khr(physical_device, surface, None)
.unwrap()
.into_iter()
.find(|present_mode| present_mode == &vk::PresentModeKHR::MAILBOX_KHR)
.unwrap_or(vk::PresentModeKHR::FIFO_KHR);
let supported_device_extensions = instance
.enumerate_device_extension_properties(physical_device, None, None)
.unwrap();
let device_extensions_supported =
device_extensions.iter().all(|device_extension| {
let device_extension = CStr::from_ptr(*device_extension);
supported_device_extensions.iter().any(|properties| {
CStr::from_ptr(properties.extension_name.as_ptr()) == device_extension
})
});
if !device_extensions_supported {
return None;
}
let device_properties = instance.get_physical_device_properties(physical_device);
Some((
physical_device,
queue_family,
format,
present_mode,
device_properties,
))
})
.max_by_key(|(_, _, _, _, properties)| match properties.device_type {
vk::PhysicalDeviceType::DISCRETE_GPU => 2,
vk::PhysicalDeviceType::INTEGRATED_GPU => 1,
_ => 0,
})
.expect("No suitable physical device found");
//end of declaration of enum (physical_device, queue_family, format, present_mode, device_properties)
// https://vulkan-tutorial.com/Drawing_a_triangle/Setup/Logical_device_and_queues
let queue_info = vec![vk::DeviceQueueCreateInfoBuilder::new()
.queue_family_index(queue_family)
.queue_priorities(&[1.0])];
let features = vk::PhysicalDeviceFeaturesBuilder::new();
let device_info = vk::DeviceCreateInfoBuilder::new()
.queue_create_infos(&queue_info)
.enabled_features(&features)
.enabled_extension_names(&device_extensions)
.enabled_layer_names(&device_layers);
let device =
unsafe { DeviceLoader::new(&instance, physical_device, &device_info, None) }.unwrap();
// let queue2 = unsafe { device2.get_device_queue(queue_family, 0) };
let queue = unsafe { device.get_device_queue(queue_family, 0) };
println!("\n \n");
let model_path: &'static str = "assets/terrain__002__.obj";
let (models, materials) = tobj::load_obj(&model_path, &tobj::LoadOptions::default()).expect("Failed to load model object!");
let model = models[0].clone();
let materials = materials.unwrap();
let material = materials.clone().into_iter().nth(0).unwrap();
let mut vertices = vec![];
let mut indices = vec![];
let mesh = model.mesh;
let total_vertices_count = mesh.positions.len() / 3;
for i in 0..total_vertices_count {
let vertex = VertexV3 {
pos: [
mesh.positions[i * 3],
mesh.positions[i * 3 + 1],
mesh.positions[i * 3 + 2],
1.0,
],
color: [1.0, 1.0, 1.0, 1.0],
};
vertices.push(vertex);
};
indices = mesh.indices.clone();
println!("Starting buffer and memory allocation/mapping processes... \n");
let vertex_buffer_size = ::std::mem::size_of_val(&vertices) as vk::DeviceSize;
println!("vertex_buffer_size: {:?}", vertex_buffer_size);
let physical_device_memory_properties = unsafe { instance.get_physical_device_memory_properties(physical_device) };
println!("\n physical_device_memory_properties: {:?}", physical_device_memory_properties);
pretty_print(physical_device_memory_properties);
let vertex_buffer_create_info = vk::BufferCreateInfoBuilder::new()
.size(vertex_buffer_size * 8)
.usage(vk::BufferUsageFlags::VERTEX_BUFFER)
.sharing_mode(vk::SharingMode::EXCLUSIVE);
println!("\n vertex_buffer_create_info: {:?}", vertex_buffer_create_info);
let vertex_buffer = unsafe {
device
.create_buffer(&vertex_buffer_create_info, None)
.expect("Failed to create vertex buffer.")
};
let vertex_buffer_memory_reqs = unsafe {
device
.get_buffer_memory_requirements(vertex_buffer)
};
println!("\n vertex_buffer_memory_reqs: {:?}", vertex_buffer_memory_reqs);
let vertex_buffer_memory_allocate_info = unsafe {
vk::MemoryAllocateInfoBuilder::new()
.allocation_size(vertex_buffer_memory_reqs.size)
.memory_type_index(2)
.build()
};
println!("\n vertex_buffer_memory_allocate_info, {:?} \n", vertex_buffer_memory_allocate_info);
let vertex_buffer_memory = unsafe {
device
.allocate_memory(&vertex_buffer_memory_allocate_info, None)
.expect("Failed to allocate vertex buffer memory.")
};
println!("\n vertex_buffer_memory: {:?} \n", &vertex_buffer_memory);
unsafe { device.bind_buffer_memory(vertex_buffer, vertex_buffer_memory, 0) }
.expect("Error on bind buffer memory");
unsafe {
let mut pointer: *mut *mut std::ffi::c_void = std::ptr::null_mut();
device
.map_memory(
vertex_buffer_memory,
0,
vertex_buffer_memory_reqs.size,
None,
pointer,
)
.expect("failed to map 333memory.");
}
}
fn pretty_print(stuff: vk::PhysicalDeviceMemoryProperties) {
println!("\n pretty_print physical_device_memory_properties: \n");
for memory_type in stuff.memory_types {
println!("memory type: {:?}", memory_type);
}
for heap in stuff.memory_heaps {
println!("memory heap: {:?}", heap);
}
}
With help from friz64 on the Erupt GameDev board:
I was derefencing a null pointer.
The pointer needed to be *mut rather than *mut *mut, and then take a mutable reference from it.
The block around map_memory should look like this:
unsafe {
let mut pointer: *mut std::ffi::c_void = std::ptr::null_mut();
let mut ref1 = &mut pointer;
device
.map_memory(
vertex_buffer_memory,
256,
vk::WHOLE_SIZE,
None,
ref1,
)
.expect("failed to map 333memory.");
}

Future created by async block is not `Send` because of *mut u8 [duplicate]

This question already has an answer here:
How to send a pointer to another thread?
(1 answer)
Closed 5 months ago.
I was able to proceed forward to implement my asynchronous udp server. However I have this error showing up twice because my variable data has type *mut u8 which is not Send:
error: future cannot be sent between threads safely
help: within `impl std::future::Future`, the trait `std::marker::Send` is not implemented for `*mut u8`
note: captured value is not `Send`
And the code (MRE):
use std::error::Error;
use std::time::Duration;
use std::env;
use tokio::net::UdpSocket;
use tokio::{sync::mpsc, task, time}; // 1.4.0
use std::alloc::{alloc, Layout};
use std::mem;
use std::mem::MaybeUninit;
use std::net::SocketAddr;
const UDP_HEADER: usize = 8;
const IP_HEADER: usize = 20;
const AG_HEADER: usize = 4;
const MAX_DATA_LENGTH: usize = (64 * 1024 - 1) - UDP_HEADER - IP_HEADER;
const MAX_CHUNK_SIZE: usize = MAX_DATA_LENGTH - AG_HEADER;
const MAX_DATAGRAM_SIZE: usize = 0x10000;
/// A wrapper for [ptr::copy_nonoverlapping] with different argument order (same as original memcpy)
unsafe fn memcpy(dst_ptr: *mut u8, src_ptr: *const u8, len: usize) {
std::ptr::copy_nonoverlapping(src_ptr, dst_ptr, len);
}
// Different from https://doc.rust-lang.org/std/primitive.u32.html#method.next_power_of_two
// Returns the [exponent] from the smallest power of two greater than or equal to n.
const fn next_power_of_two_exponent(n: u32) -> u32 {
return 32 - (n - 1).leading_zeros();
}
async fn run_server(socket: UdpSocket) {
let mut missing_indexes: Vec<u16> = Vec::new();
let mut peer_addr = MaybeUninit::<SocketAddr>::uninit();
let mut data = std::ptr::null_mut(); // ptr for the file bytes
let mut len: usize = 0; // total len of bytes that will be written
let mut layout = MaybeUninit::<Layout>::uninit();
let mut buf = [0u8; MAX_DATA_LENGTH];
let mut start = false;
let (debounce_tx, mut debounce_rx) = mpsc::channel::<(usize, SocketAddr)>(3300);
let (network_tx, mut network_rx) = mpsc::channel::<(usize, SocketAddr)>(3300);
loop {
// Listen for events
let debouncer = task::spawn(async move {
let duration = Duration::from_millis(3300);
loop {
match time::timeout(duration, debounce_rx.recv()).await {
Ok(Some((size, peer))) => {
eprintln!("Network activity");
}
Ok(None) => {
if start == true {
eprintln!("Debounce finished");
break;
}
}
Err(_) => {
eprintln!("{:?} since network activity", duration);
}
}
}
});
// Listen for network activity
let server = task::spawn({
// async{
let debounce_tx = debounce_tx.clone();
async move {
while let Some((size, peer)) = network_rx.recv().await {
// Received a new packet
debounce_tx.send((size, peer)).await.expect("Unable to talk to debounce");
eprintln!("Received a packet {} from: {}", size, peer);
let packet_index: u16 = (buf[0] as u16) << 8 | buf[1] as u16;
if start == false { // first bytes of a new file: initialization // TODO: ADD A MUTEX to prevent many initializations
start = true;
let chunks_cnt: u32 = (buf[2] as u32) << 8 | buf[3] as u32;
let n: usize = MAX_DATAGRAM_SIZE << next_power_of_two_exponent(chunks_cnt);
unsafe {
layout.as_mut_ptr().write(Layout::from_size_align_unchecked(n, mem::align_of::<u8>()));
// /!\ data has type `*mut u8` which is not `Send`
data = alloc(layout.assume_init());
peer_addr.as_mut_ptr().write(peer);
}
let a: Vec<u16> = vec![0; chunks_cnt as usize]; //(0..chunks_cnt).map(|x| x as u16).collect(); // create a sorted vector with all the required indexes
missing_indexes = a;
}
missing_indexes[packet_index as usize] = 1;
unsafe {
let dst_ptr = data.offset((packet_index as usize * MAX_CHUNK_SIZE) as isize);
memcpy(dst_ptr, &buf[AG_HEADER], size - AG_HEADER);
};
println!("receiving packet {} from: {}", packet_index, peer);
}
}
});
// Prevent deadlocks
drop(debounce_tx);
match socket.recv_from(&mut buf).await {
Ok((size, src)) => {
network_tx.send((size, src)).await.expect("Unable to talk to network");
}
Err(e) => {
eprintln!("couldn't recieve a datagram: {}", e);
}
}
}
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
let addr = env::args().nth(1).unwrap_or_else(|| "127.0.0.1:8080".to_string());
let socket = UdpSocket::bind(&addr).await?;
println!("Listening on: {}", socket.local_addr()?);
run_server(socket);
Ok(())
}
Since I was converting from synchronous to asynchronous code I know that, potentially, multiple thread would be writing to data, and that is probably why I encounter such error. But I don't know which syntax I could use to "clone" the mut ptr and make it unique for each thread (and same for the buffer).
As suggested by user4815162342 I think the best would be
to make pointer Send by wrapping it in a struct and declaring unsafe impl Send for NewStruct {}.
Any help strongly appreciated!
PS: Full code can be found on my github repository
Short version
Thanks to the comment of user4815162342 I decided to add an implementation for the mut ptr to be able to use it with Send and Sync, which allowed me to solve this part (there are still other issues, but beyond the scope of this question):
pub struct FileBuffer {
data: *mut u8
}
unsafe impl Send for FileBuffer {}
unsafe impl Sync for FileBuffer {}
//let mut data = std::ptr::null_mut(); // ptr for the file bytes
let mut fileBuffer: FileBuffer = FileBuffer { data: std::ptr::null_mut() };
Long version
use std::error::Error;
use std::time::Duration;
use std::env;
use tokio::net::UdpSocket;
use tokio::{sync::mpsc, task, time}; // 1.4.0
use std::alloc::{alloc, Layout};
use std::mem;
use std::mem::MaybeUninit;
use std::net::SocketAddr;
const UDP_HEADER: usize = 8;
const IP_HEADER: usize = 20;
const AG_HEADER: usize = 4;
const MAX_DATA_LENGTH: usize = (64 * 1024 - 1) - UDP_HEADER - IP_HEADER;
const MAX_CHUNK_SIZE: usize = MAX_DATA_LENGTH - AG_HEADER;
const MAX_DATAGRAM_SIZE: usize = 0x10000;
/// A wrapper for [ptr::copy_nonoverlapping] with different argument order (same as original memcpy)
unsafe fn memcpy(dst_ptr: *mut u8, src_ptr: *const u8, len: usize) {
std::ptr::copy_nonoverlapping(src_ptr, dst_ptr, len);
}
// Different from https://doc.rust-lang.org/std/primitive.u32.html#method.next_power_of_two
// Returns the [exponent] from the smallest power of two greater than or equal to n.
const fn next_power_of_two_exponent(n: u32) -> u32 {
return 32 - (n - 1).leading_zeros();
}
pub struct FileBuffer {
data: *mut u8
}
unsafe impl Send for FileBuffer {}
unsafe impl Sync for FileBuffer {}
async fn run_server(socket: UdpSocket) {
let mut missing_indexes: Vec<u16> = Vec::new();
let mut peer_addr = MaybeUninit::<SocketAddr>::uninit();
//let mut data = std::ptr::null_mut(); // ptr for the file bytes
let mut fileBuffer: FileBuffer = FileBuffer { data: std::ptr::null_mut() };
let mut len: usize = 0; // total len of bytes that will be written
let mut layout = MaybeUninit::<Layout>::uninit();
let mut buf = [0u8; MAX_DATA_LENGTH];
let mut start = false;
let (debounce_tx, mut debounce_rx) = mpsc::channel::<(usize, SocketAddr)>(3300);
let (network_tx, mut network_rx) = mpsc::channel::<(usize, SocketAddr)>(3300);
loop {
// Listen for events
let debouncer = task::spawn(async move {
let duration = Duration::from_millis(3300);
loop {
match time::timeout(duration, debounce_rx.recv()).await {
Ok(Some((size, peer))) => {
eprintln!("Network activity");
}
Ok(None) => {
if start == true {
eprintln!("Debounce finished");
break;
}
}
Err(_) => {
eprintln!("{:?} since network activity", duration);
}
}
}
});
// Listen for network activity
let server = task::spawn({
// async{
let debounce_tx = debounce_tx.clone();
async move {
while let Some((size, peer)) = network_rx.recv().await {
// Received a new packet
debounce_tx.send((size, peer)).await.expect("Unable to talk to debounce");
eprintln!("Received a packet {} from: {}", size, peer);
let packet_index: u16 = (buf[0] as u16) << 8 | buf[1] as u16;
if start == false { // first bytes of a new file: initialization // TODO: ADD A MUTEX to prevent many initializations
start = true;
let chunks_cnt: u32 = (buf[2] as u32) << 8 | buf[3] as u32;
let n: usize = MAX_DATAGRAM_SIZE << next_power_of_two_exponent(chunks_cnt);
unsafe {
layout.as_mut_ptr().write(Layout::from_size_align_unchecked(n, mem::align_of::<u8>()));
// /!\ data has type `*mut u8` which is not `Send`
fileBuffer.data = alloc(layout.assume_init());
peer_addr.as_mut_ptr().write(peer);
}
let a: Vec<u16> = vec![0; chunks_cnt as usize]; //(0..chunks_cnt).map(|x| x as u16).collect(); // create a sorted vector with all the required indexes
missing_indexes = a;
}
missing_indexes[packet_index as usize] = 1;
unsafe {
let dst_ptr = fileBuffer.data.offset((packet_index as usize * MAX_CHUNK_SIZE) as isize);
memcpy(dst_ptr, &buf[AG_HEADER], size - AG_HEADER);
};
println!("receiving packet {} from: {}", packet_index, peer);
}
}
});
// Prevent deadlocks
drop(debounce_tx);
match socket.recv_from(&mut buf).await {
Ok((size, src)) => {
network_tx.send((size, src)).await.expect("Unable to talk to network");
}
Err(e) => {
eprintln!("couldn't recieve a datagram: {}", e);
}
}
}
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn Error>> {
let addr = env::args().nth(1).unwrap_or_else(|| "127.0.0.1:8080".to_string());
let socket = UdpSocket::bind(&addr).await?;
println!("Listening on: {}", socket.local_addr()?);
run_server(socket);
Ok(())
}

Resources