gdb: set scheduler_locking on always causes deadlock - linux

Question

I am using gdb to debug a shared object loaded by a C program using dlopen() and dlsym(). The shared object is written in NASM, and it's compiled with DWARF debug symbols. I'm running on Ubuntu 18.04.
I want each thread to stop completely after it has been created so I can have all four threads created before I continue. So I start the debug session with "set scheduler-locking on" or "set scheduler-locking step."
As I understand the gdb command "set scheduler-locking on," it should allow me to create a thread, then switch back to thread 1 (the main thread) and create another thread, and so on until all threads have been created.
In the NASM code, I set a breakpoint at the label Test_fn: (see the code below). When a thread reaches that breakpoint, it stops (which is the first breakpoint after thread creation). Then I switch to back thread 1 (the main thread) and "continue" to instantiate the next thread (the main thread is still in the label_0 section of Init_Cores). Thread 1 will execute code (if I single-step through it), but at some point before the next thread is created it deadlocks, and it always deadlocks.
I have also tried single-stepping through thread creation (avoiding "continue"), but it still deadlocks.
According to the answer from Employed Russian at How to continue one thread at a time when debugging a multithreaded program in GDB?, that can happen but he doesn't say that it will always happen. Whether I use "set scheduler-locking on" or "set scheduler-locking step" I get a deadlock.
Maybe this is because it's a shared object, but everything else works with the shared object, so I don't think that's the problem.
Here is the NASM code. The same question would apply to C or C++ code, specifically why gdb deadlocks when I reach a breakpoint on a new thread, then switch back to thread 1 to continue creating threads. Thread 2 (the first created thread) should stop when scheduler-locking is on.
; Header Section
[BITS 64]
[default rel]
global Main_Entry_fn
extern pthread_create, pthread_join, pthread_exit, pthread_self, sched_getcpu
global FreeMem_fn
extern malloc, realloc, free
extern sprintf
section .data align=16
X_ctr: dq 0
data_master_ptr: dq 0
initial_dynamic_length: dq 0
XMM_Stack: dq 0, 0, 0, 0, 0, 0, 0
ThreadID: dq 0
X_ptr: dq 0
X_length: dq 0
X: dq 0
collect_ptr: dq 0
collect_length: dq 0
collect_ctr: dq 0
even_squares_list_ptrs: dq 0, 0, 0, 0
even_squares_list_ctr: dq 0
even_squares_list_length: dq 0
Number_Of_Cores: dq 4
pthread_attr_t: dq 0
pthread_arg: dq 0
Join_Ret_Val: dq 0
tcounter: dq 0
sched_getcpu_array: times 4 dq 0
ThreadIDLocked: dq 0
spin_lock_core: times 4 dq 0
extra_test_array: times 4 dq 0
spin_lock_iter: times 4 dq 0
spin_lock_base_addr: dq 0
; __________
section .text
Init_Cores_fn:
%include "/opt/P01_SH/_Include_Utilities/Buffer_Pointer_Arrays.asm"
mov rax,[Number_Of_Cores]
mov rbx,8
mul rbx
mov [Number_Of_Cores],rax
; _____
; Create Threads
label_0:
; THREAD 1 WORKS IN THIS SECTION TO CREATE THREADS
mov rdi,ThreadID ; ThreadCount
mov rsi,pthread_attr_t ; Thread Attributes
mov rdx,Test_fn ; Function Pointer
mov rcx,pthread_arg
call pthread_create wrt ..plt
mov rdi,[ThreadID] ; id to wait on
mov rsi,Join_Ret_Val ; return value
call pthread_join wrt ..plt
mov rax,[tcounter]
add rax,8
mov [tcounter],rax
mov rbx,[Number_Of_Cores]
cmp rax,rbx
jl label_0
; _____
jmp label_900 ; All threads return here, and exit
; ______________________________________
Test_fn:
; Get the core number
call sched_getcpu wrt ..plt
mov rbx,8 ; multiply by 8
mul rbx
push rax
pop rax
mov rbx,rax
push rax
Next_Stop: ; THIS IS WHERE EACH THREAD STOPS
mov rdi,extra_test_array
mov [rdi+rbx],rbx
jmp label_899 ; ******************
;__________
label_899:
pop rax
ret
; __________
label_900:
%include "/opt/P01_SH/_Include_Utilities/Sys_Close_Include.asm"
mov rdi,extra_test_array
mov rax,rdi
ret
;__________
;Free the memory
FreeMem_fn:
;The pointer is passed back in rcx (of course)
sub rsp,40
call free wrt ..plt
add rsp,40
ret
; __________
; Main Entry
Main_Entry_fn:
push rdi
push rbp
push rbx
push r15
xor r15,r15
push r14
xor r14,r14
push r13
xor r13,r13
push r12
xor r12,r12
push r11
xor r11,r11
push r10
xor r10,r10
push r9
xor r9,r9
push r8
xor r8,r8
movsd [XMM_Stack+0],xmm13
movsd [XMM_Stack+8],xmm12
movsd [XMM_Stack+16],xmm11
movsd [XMM_Stack+24],xmm15
movsd [XMM_Stack+32],xmm14
movsd [XMM_Stack+40],xmm10
mov [X_ptr],rdi
mov [data_master_ptr],rsi
; Now assign lengths
lea rdi,[data_master_ptr]
mov rbp,[rdi]
xor rcx,rcx
movsd xmm0,qword[rbp+rcx]
cvttsd2si rax,xmm0
mov [X_length],rax
add rcx,8
; __________
; Write variables to assigned registers
mov r15,0
lea rdi,[rel collect_ptr]
mov r14,qword[rdi]
mov r13,[collect_ctr]
mov r12,[collect_length]
lea rdi,[rel X_ptr]
mov r11,qword[rdi]
mov r10,[X_length]
; __________
call Init_Cores_fn
movsd xmm10,[XMM_Stack+0]
movsd xmm14,[XMM_Stack+8]
movsd xmm15,[XMM_Stack+16]
movsd xmm11,[XMM_Stack+24]
movsd xmm12,[XMM_Stack+32]
movsd xmm13,[XMM_Stack+40]
pop r8
pop r9
pop r10
pop r11
pop r12
pop r13
pop r14
pop r15
pop rbx
pop rbp
pop rdi
ret
;__________
The NASM code is a long listing, but the focus is on label_0 (where threads are created) and Test_fn (the first breakpoint that a new thread reaches.
I would really appreciate any input to this problem with gdb scheduler locking. Thanks.

Answer 1

Your assembly program appears to do this:
pthread_create(&ThreadID, ..., Test_fn, ...);
pthread_join(ThreadID, ...);
First, this doesn't actually give you any parallelism, as you create and immediately wait for the thread you just created. You could just as easily call Test_fn() directly, with less overhead.
Second, if you stop the newly-created thread (as you do with scheduler-locking), then your main thread will block waiting to join that newly-created thread, and it will block forever. So of course your program will deadlock.
Update:
I thought I need to call pthread_join right after pthread_create so all threads will finish before the main thread exits.
That is correct. But they way to do that generally looks like this:
const int NThreads = ...;
pthread_t tids[NThreads];
for (int j = 0; j < NThreads; j++)
pthread_create(&tids[j], ...);
// All threads have started, and are now running in parallel with the main thread.
// Wait for them to finish.
for (int j = 0; j < NThreads; j++)
pthread_join(tids[j], ...);
// All done.
return 0;