In C++, thread 1 only psuh_back(data) in LIST, and thread 2 only pop_front(data) from LIST. Do I need to lock empty() in thread 2? I only worry about the situation where LIST is empty but empty() returns false. Is this worry unnecessary?
list<my_strust> buffer;
static void receivedata(my_strust &data)
{
mutex.lock();
buffer.push_back(data);
mutex.unlock();
}
static void senddata()
{
while(1) {
if (buffer.empty()) {
continue;
}
mutex.lock();
buffer.pop_front();
mutex.unlock();
}
}
Related
I need to provide format-magic to a class hierarchy and usually those things are passed around via pointer. This example here works currently on master:
#include <type_traits>
#include <sstream>
#include <fmt/format.h>
#include <fmt/ostream.h>
struct A {
virtual ~A() {}
virtual std::string name() const { return "A"; }
};
struct B : A {
virtual std::string name() const { return "B"; }
};
template <typename T>
struct fmt::formatter<T, std::enable_if_t<std::is_base_of<A, std::remove_const_t<std::remove_pointer_t<T>>>::value, char>> : fmt::formatter<std::string> {
template <typename FormatCtx>
auto format(const A* a, FormatCtx& ctx) {
return fmt::formatter<std::string>::format(a->name(), ctx);
}
};
template<class... T>
[[noreturn]] void err(fmt::format_string<T...> s, T&&... args) {
throw std::logic_error(fmt::format(s, std::forward<T>(args)...));
}
int main() {
A* a = new A();
B* b = new B();
const A* x = new A();
const B* y = new B();
fmt::print("{}, {}, {}, {}\n", a, b, x, y);
std::ostringstream oss;
fmt::print(oss, "{}\n", a);
err("{}, {}, {}, {}\n", a, b, x, y);
}
However, when I'm going to the latest release 8.1.1, I'm getting:
error: static assertion failed: Formatting of non-void pointers is disallowed.
Now my question is: What's the plan here? Can I do this in future versions of fmt or is this more of an accident?
This is definitely a regression. Supporting a hierarchy of classes can be done as documented in https://fmt.dev/latest/api.html#formatting-user-defined-types using references instead of pointers:
#include <type_traits>
#include <fmt/format.h>
struct A {
virtual ~A() {}
virtual std::string name() const { return "A"; }
};
struct B : A {
virtual std::string name() const { return "B"; }
};
template <typename T>
struct fmt::formatter<T, std::enable_if_t<std::is_base_of<A, T>::value, char>> :
fmt::formatter<std::string> {
template <typename FormatCtx>
auto format(const A& a, FormatCtx& ctx) {
return fmt::formatter<std::string>::format(a.name(), ctx);
}
};
int main() {
B b;
A& a = b;
fmt::print("{}", a); // prints "B"
}
I want to build a single-producer multiple-consumer example in Rust, where the producer is bounded to have no more than 10 outstanding items. I modeled a solution in C that uses a a mutex and two condvars. One condvar is to wait the consumers when there is nothing to consume and one condvar is to wait for the producer when the unconsumed items count is greater than say 10. The C code is below.
As I understand it from the Rust docs, there must be a 1-1 connection between std::sync::Mutex and a std::sync::Condvar so I can't make an exact translation of my C solution.
Is there some other way to achieve the same end (that I cannot see) in Rust using std::sync::Mutex and std::sync::Condvar.
#define _GNU_SOURCE
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <pthread.h>
//
// This is a simple example of using a mutex and 2 condition variables to
// sync a single writer and multiple readers interacting with a bounded (fixed max size) queue
//
// in this toy example a queue is simulated by an int counter n_resource
//
int n_resource;
pthread_cond_t rdr_cvar;
pthread_cond_t wrtr_cvar;
pthread_mutex_t mutex;
void reader(void* data)
{
long id = (long)data;
for(;;) {
pthread_mutex_lock(&mutex);
while (n_resource <= 0) {
pthread_cond_wait(&rdr_cvar, &mutex);
}
printf("Reader %ld n_resource = %d\n", id, n_resource);
--n_resource;
// if there are still things to read - singla one reader
if(n_resource > 0) {
pthread_cond_signal(&rdr_cvar);
}
// if there is space for the writer to add another signal the writer
if(n_resource < 10) {
pthread_cond_signal(&wrtr_cvar);
}
pthread_mutex_unlock(&mutex);
}
}
void writer(void* data)
{
for(;;) {
pthread_mutex_lock(&mutex);
printf("Writer before while n_resource %d \n", n_resource);
while (n_resource > 10) {
pthread_cond_wait(&wrtr_cvar, &mutex);
}
printf("Writer after while n_resource %d \n", n_resource);
++n_resource;
// if there is something for a reader to read signal one of the readers.
if(n_resource > 0) {
pthread_cond_signal(&rdr_cvar);
}
pthread_mutex_unlock(&mutex);
}
}
int main()
{
pthread_t rdr_thread_1;
pthread_t rdr_thread_2;
pthread_t wrtr_thread;
pthread_mutex_init(&mutex, NULL);
pthread_cond_init(&rdr_cvar, NULL);
pthread_cond_init(&wrtr_cvar, NULL);
pthread_create(&rdr_thread_1, NULL, &reader, (void*)1L);
pthread_create(&rdr_thread_2, NULL, &reader, (void*)2L);
pthread_create(&wrtr_thread, NULL, &writer, NULL);
pthread_join(wrtr_thread, NULL);
pthread_join(rdr_thread_1, NULL);
pthread_join(rdr_thread_2, NULL);
}
While a CondVar needs to be associated with only one Mutex, it is not necessary that a Mutex is associated with only one CondVar.
For example, the following code seems to work just fine - you can run it on the playground.
use std::sync::{Arc, Condvar, Mutex};
use std::thread;
struct Q {
rdr_cvar: Condvar,
wrtr_cvar: Condvar,
mutex: Mutex<i32>,
}
impl Q {
pub fn new() -> Q {
Q {
rdr_cvar: Condvar::new(),
wrtr_cvar: Condvar::new(),
mutex: Mutex::new(0),
}
}
}
fn writer(id: i32, qq: Arc<Q>) {
let q = &*qq;
for i in 0..10 {
let guard = q.mutex.lock().unwrap();
let mut guard = q.wrtr_cvar.wait_while(guard, |n| *n > 3).unwrap();
println!("{}: Writer {} n_resource = {}\n", i, id, *guard);
*guard += 1;
if *guard > 0 {
q.rdr_cvar.notify_one();
}
if *guard < 10 {
q.wrtr_cvar.notify_one();
}
}
}
fn reader(id: i32, qq: Arc<Q>) {
let q = &*qq;
for i in 0..10 {
let guard = q.mutex.lock().unwrap();
let mut guard = q.rdr_cvar.wait_while(guard, |n| *n <= 0).unwrap();
println!("{} Reader {} n_resource = {}\n", i, id, *guard);
*guard -= 1;
if *guard > 0 {
q.rdr_cvar.notify_one();
}
if *guard < 10 {
q.wrtr_cvar.notify_one();
}
}
}
fn main() {
let data = Arc::new(Q::new());
let data2 = data.clone();
let t1 = thread::spawn(move || writer(0, data2));
let t2 = thread::spawn(move || reader(1, data));
t1.join().unwrap();
t2.join().unwrap();
}
Below code uses ~150MB in single thread but uses several GBs in 100 threads:
use std::sync::{Arc, Mutex};
use std::thread;
fn main() {
let f = Arc::new(Mutex::new(Foo::new("hello")));
let mut threads = vec![];
for i in 0..100 {
let f = f.clone();
let t = thread::spawn(move || loop {
let mut locked = f.lock().unwrap();
*locked = Foo::new("hello");
drop(locked);
println!("{} reloaded", i);
thread::yield_now();
});
threads.push(t);
}
threads.into_iter().for_each(|h| h.join().unwrap());
}
pub struct Foo {
_data: Vec<String>,
}
impl Foo {
fn new(s: &str) -> Foo {
Foo {
_data: vec![s.to_owned(); 1024 * 1024],
}
}
}
While holding the LockGuard, a thread should have exclusive access. So, new Foo should be allocated and old value should be dropped at that point. So, it doesn't make any sense to me this much memory is being used when called from multiple threads.
Can anyone please explain why this code is using this much memory?
Similar code in Java keeps memory ~200mb even with 1000 threads.
import java.util.ArrayList;
import java.util.List;
public class Foo {
private List<String> data;
public static void main(String[] args) {
Foo f = new Foo();
for (int i = 0; i < 1000; i++) {
int n = i;
new Thread(() -> {
while (true) {
f.update();
System.gc();
System.out.println(n + " updated");
}
}).start();
}
}
public synchronized void update() {
data = new ArrayList<>(1024 * 1024);
for (int i = 0; i < 1024 * 1024; i++) {
data.add(new String("hello"));
}
}
}
So the problem was in the big numbers of glibc's malloc arenas,
every arena has cache of preallocated memory. The simple way to check it is running binary with MALLOC_ARENA_MAX=2, but final solution depend on usage pattern, there are a lot variables to tune glibc's allocator: http://man7.org/linux/man-pages/man3/mallopt.3.html .
Java virtual machine is also actually affected by malloc's allocator. From my experience some time it is suitable to configure number of arenas to prevent huge memory usage of jvm inside docker.
I have a struct Foo and FooRef which has references to data from Foo:
struct Foo { /* ... */ }
struct FooRef<'foo> { /* ... */ }
impl Foo {
pub fn create_ref<'a>(&'a self) -> FooRef<'a> { /* ... */ }
}
Now Foo directly cannot be used in the logic; I need FooRef. Creating FooRef requires lots of computation, so I do it once just after creating the Foo instance. FooRef is immutable; it's only used for reading data.
Multiple threads needs to access this FooRef instance. How can I implement this? The calling threads are Java threads and this will be used with JNI. This prevents using a scoped threadpool, for example.
Another complication is that when I have to refresh the Foo instance to load new data into it. I then also need to recreate the FooRef instance as well.
How can this be achieved thread-safely and memory-safely? I tried messing around with pointers and RwLock but that resulted in a memory leak (the memory usage kept on adding on each reload). I am a Java developer that is a newbie to pointers.
The data in Foo is mostly text and about 250Mb. The FooRef is mostly strs and structs of strs borrowed from Foo.
My Java usage explanation
I use two long variables in a Java class to store pointers to Foo and FooRef. I use a static ReentrantReadWriteLock to guard these pointers.
If the data need to be updated in Foo, I acquire a write lock, drop FooRef, update Foo, create a new FooRef and update the ref pointer in Java.
If I need to read the data (i.e. when I am not updating Foo), I acquire a read lock and use the FooRef.
The memory leak is visible only when multiple Java threads are calling this code.
Rust:
use jni::objects::{JClass, JString};
use jni::sys::{jlong, jstring};
use jni::JNIEnv;
use std::collections::HashMap;
macro_rules! foo_mut_ptr {
($env: expr, $class: expr) => {
$env.get_field(*$class, "ptr", "J")
.ok()
.and_then(|j| j.j().ok())
.and_then(|ptr| {
if ptr == 0 {
None
} else {
Some(ptr as *mut Foo)
}
})
};
}
macro_rules! foo_ref_mut_ptr {
($env: expr, $class: expr) => {
$env.get_field(*$class, "ptrRef", "J")
.ok()
.and_then(|j| j.j().ok())
.and_then(|ptr| {
if ptr == 0 {
None
} else {
Some(ptr as *mut FooRef)
}
})
};
}
macro_rules! foo_mut {
($env: expr, $class: expr) => {
foo_mut_ptr!($env, $class).map(|ptr| &mut *ptr)
};
}
macro_rules! foo_ref {
($env: expr, $class: expr) => {
foo_ref_mut_ptr!($env, $class).map(|ptr| &*ptr)
};
}
#[allow(non_snake_case)]
#[no_mangle]
pub unsafe extern "system" fn Java_test_App_create(_env: JNIEnv, _class: JClass) -> jlong {
Box::into_raw(Box::new(Foo::default())) as jlong
}
#[allow(non_snake_case)]
#[no_mangle]
pub unsafe extern "system" fn Java_test_App_createRef(env: JNIEnv, class: JClass) -> jlong {
let foo = foo_mut!(env, class).expect("createRef was called on uninitialized Data");
let foo_ref = foo.create_ref();
Box::into_raw(Box::new(foo_ref)) as jlong
}
#[allow(non_snake_case)]
#[no_mangle]
pub unsafe extern "system" fn Java_test_App_reload(env: JNIEnv, class: JClass) {
let foo = foo_mut!(env, class).expect("foo must be initialized");
*foo = Foo {
data: vec!["hello".to_owned(); 1024 * 1024],
};
}
#[allow(non_snake_case)]
#[no_mangle]
pub unsafe extern "system" fn Java_test_App_destroy(env: JNIEnv, class: JClass) {
drop_ptr(foo_ref_mut_ptr!(env, class));
drop_ptr(foo_mut_ptr!(env, class));
}
#[allow(non_snake_case)]
#[no_mangle]
pub unsafe extern "system" fn Java_test_App_destroyRef(env: JNIEnv, class: JClass) {
drop_ptr(foo_ref_mut_ptr!(env, class));
}
unsafe fn drop_ptr<T>(ptr: Option<*mut T>) {
if let Some(ptr) = ptr {
let _foo = Box::from_raw(ptr);
// foo drops here
}
}
#[derive(Default)]
struct Foo {
data: Vec<String>,
}
#[derive(Default)]
struct FooRef<'a> {
data: HashMap<&'a str, Vec<&'a str>>,
}
impl Foo {
fn create_ref(&self) -> FooRef {
let mut data = HashMap::new();
for s in &self.data {
let s = &s[..];
data.insert(s, vec![s]);
}
FooRef { data }
}
}
Java:
package test;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
public class App implements AutoCloseable {
private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
private final ReadLock readLock = lock.readLock();
private final WriteLock writeLock = lock.writeLock();
private volatile long ptr;
private volatile long ptrRef;
private volatile boolean reload;
static {
System.loadLibrary("foo");
}
public static void main(String[] args) throws InterruptedException {
try (App app = new App()) {
for (int i = 0; i < 20; i++) {
new Thread(() -> {
while (true) {
app.tryReload();
}
}).start();
}
while (true) {
app.setReload();
}
}
}
public App() {
this.ptr = this.create();
}
public void setReload() {
writeLock.lock();
try {
reload = true;
} finally {
writeLock.unlock();
}
}
public void tryReload() {
readLock.lock();
debug("Got read lock");
if (reload) {
debug("Cache is expired");
readLock.unlock();
debug("Released read lock coz expired");
writeLock.lock();
debug("Got write lock");
try {
if (reload) {
fullReload();
}
readLock.lock();
debug("Got read lock inside write");
} finally {
writeLock.unlock();
debug("Released write lock");
}
}
readLock.unlock();
debug("Released read lock");
}
private void fullReload() {
destroyRef();
debug("Dropped ref");
debug("Reloading");
reload();
debug("Reloading completed");
updateRef();
debug("Created ref");
reload = false;
}
private void updateRef() {
this.ptrRef = this.createRef();
}
private native void reload();
private native long create();
private native long createRef();
private native void destroy();
private native void destroyRef();
#Override
public void close() {
writeLock.lock();
try {
this.destroy();
this.ptrRef = 0;
this.ptr = 0;
} finally {
writeLock.unlock();
}
}
private static void debug(String s) {
System.out.printf("%10s : %s%n", Thread.currentThread().getName(), s);
}
}
The problem that I was thinking as memory leak wasn't actually a memory leak. The issue was that the allocator was using thread local arenas. So, whatever thread was reloading 250MB of data was leaving the allocated space as is and not returning it to the system. This issue was not specific to JNI, but also happening in pure safe rust code. See Why multiple threads using too much memory when holding Mutex
The default number of arenas created defaults to 8 * cpu count = 64 in my case. This setting can be overridden by setting MALLOC_ARENA_MAX env variable.
So I resolved this issue by setting MALLOC_ARENA_MAX env variable to 1 . So, the approach I took is fine. It was just platform specific issue.
This issue was occurring only in Ubuntu in WSL. I also tried the same code without any tweaking on Windows 10 and it works perfectly without any issues.
Im very new to VC++. Yesterday my VC++ instructor gave us this code and asked us to make it as a exe. I have no idea of where to start and end. How to make this single file into exe. How and where to paste this working code in visual studio. If my question sounds too dumb, sorry. But i'm. Please help me out in making an exe from this single file. By the way this is Josephus circle algorithm
Code :
//////////////////Header file
#include<iostream.h>
template <class T>
class ex
{
private:
struct node
{
T data;
struct node *next;
};
struct node *head,*front,*rear;
public:
ex()
{
head=new node;
head->next=NULL;
front=rear=head;
}
void enqueue(T x);
T dequeue();
void print();
void move_next();
};
//////////////////Implementation file
#include "ex.h"
template <class T>
void ex<T>::enqueue(T x)
{
node *p;
p=new node;
p->data=x;
if(head->next==NULL)
{
front=rear=p;
head->next=p;
p->next=p;
}
else
{
rear->next=p;
p->next=front;
rear=rear->next;
}
}
template<class T>
T ex<T>::dequeue()
{
node *t;
T x;
t=front;
x=t->data;
front=front->next;
rear->next=front;
delete(t);
return x;
}
template<class T>
void ex<T>::print()
{
node *p=front;
do
{
cout<<p->data<<endl;
p=p->next;
}while(p!=rear->next);
}
template<class T>
void ex<T>::move_next()
{
front=front->next;
rear=rear->next;
}
/////////////////Application file
#include "ex.cpp"
void main()
{
ex<int> e;
int m,n,i,d;
cout<<"Enter the number of people";
cin>>n;
cout<<"Enter the number of passes";
cin>>m;
for(i=1;i<=n;i++)
e.enqueue(i);
cout<<"The players are
";
e.print();
cout<<"Eliminated in order
";
while(n>1)
{
for(i=1;i<=m;i++)
e.move_next();
d=e.dequeue();
cout<<d<<endl;
n--;
}
d=e.dequeue();
cout<<"Winning player: "<<d<<endl;
}
Make the first file named ex.h
#include<iostream.h>
template <class T>
class ex
{
private:
struct node
{
T data;
struct node *next;
};
struct node *head,*front,*rear;
public:
ex()
{
head=new node;
head->next=NULL;
front=rear=head;
}
void enqueue(T x);
T dequeue();
void print();
void move_next();
};
Second file into ex.cpp
#include "ex.h"
template <class T>
void ex<T>::enqueue(T x)
{
node *p;
p=new node;
p->data=x;
if(head->next==NULL)
{
front=rear=p;
head->next=p;
p->next=p;
}
else
{
rear->next=p;
p->next=front;
rear=rear->next;
}
}
template<class T>
T ex<T>::dequeue()
{
node *t;
T x;
t=front;
x=t->data;
front=front->next;
rear->next=front;
delete(t);
return x;
}
template<class T>
void ex<T>::print()
{
node *p=front;
do
{
cout<<p->data<<endl;
p=p->next;
}while(p!=rear->next);
}
template<class T>
void ex<T>::move_next()
{
front=front->next;
rear=rear->next;
}
And the third file into Main.cpp or something.
#include "ex.cpp"
void main()
{
ex<int> e;
int m,n,i,d;
cout<<"Enter the number of people";
cin>>n;
cout<<"Enter the number of passes";
cin>>m;
for(i=1;i<=n;i++)
e.enqueue(i);
cout<<"The players are
";
e.print();
cout<<"Eliminated in order
";
while(n>1)
{
for(i=1;i<=m;i++)
e.move_next();
d=e.dequeue();
cout<<d<<endl;
n--;
}
d=e.dequeue();
cout<<"Winning player: "<<d<<endl;
}
Then compile it. Also, it's supposed to be int main() not void main()
To add to Alex F. answer, you will also need to create an ex.h file and an ex.cpp file and paste their respective code inside