Windows multicore program worked before but suddenly threads execute randomly - multithreading

Question

Below is a multithreaded (4-core) NASM 64 program. It's not minimal, but it's complete; I posted the complete code because a minimal example may not reveal the problem in this code.
This is my first multithreaded multicore program in NASM, and I was happy to see yesterday afternoon that it worked correctly (all four cores) and returned the values I was expecting. I ran it successfully several times yesterday and again several times this morning.
Suddenly, about 10 minutes after my last run, without making any changes to the code, without re-assembling the dll, I ran it again and this time the threads execute randomly -- not all threads execute, and the configuration of threads that execute varies from run to run -- sometimes thread 1 only; sometimes threads 2-4 but not thread 1, sometimes only thread 2 and 4, etc.
My code did not have ExitThread or CloseHandle calls, so I added them in (see label_899:), rebooted and ran it again. The first run after a fresh reboot still shows random execution of threads.
I thought the problem may be that CreateThread was failing, so I also returned the thread handles. Every time, all four thread handles were created even though all did not execute, so that's not it.
I have done a lot of research but I haven't seen this issue discussed at all. Even though this is NASM, the same thing could also be applicable to C or C++ multithreading. I use WaitForMultipleObjects to wait on all threads to complete.
This is a dll, and the entry point is Main_Entry_fn. The dll is called from Python using ctypes, but I doubt that makes any difference. I use ctypes frequently and have never had a problem with it.
Thanks for any ideas.
P.S. ignore the realloc code because the buffers are large enough that realloc is never called.
UPDATE: Here is how I constructed the progam flow: the entry point is Main_Entry_fn toward the bottom of the program listing (it's an export because it has to be exported for a dll call). Main_Entry_fn takes the input data from the calling function, preserves registers (because the whole program is register optimized) and allocates output buffers. Main_Entry_fn then calls Init_Cores_fn (top of program listing), where we create four arrays with data for the registers on entry to each core (remember all vars are stored in registers -- see the call to DupThreadInfo), and creates four threads in a loop. The threads all call Prime_Number_fn, and the threads are set to start immediately upon creation.
Register assignments:
r15 list_of_results_ptr (pointer to list_of_results)
r14 list_of_results_ctr (counter for list_of_results)
r13 list_of_results_length
r12 numbers_ptr (pointer to "numbers" input array)
r11 numbers_ctr
r10 numbers_length
r9 num
r8 i
xmm15 num_float
xmm14: result
; Header Section
[BITS 64]
[default rel]
extern malloc, calloc, realloc, free
global Main_Entry_fn
export Main_Entry_fn
global FreeMem_fn
export FreeMem_fn
extern CreateThread, CloseHandle, ExitThread
extern WaitForMultipleObjects, GetCurrentThread
section .data align=16
Return_Pointer_Array: dq 0, 0, 0
Input_Length_Array: dq 0, 0,
list_of_results_ptr: dq 0
list_of_results_ctr: dq 0
list_of_results_length: dq 0
data_master_ptr: dq 0
initial_dynamic_length: dq 0
internal_dynamic_length: dq 5000
XMM_Stack: dq 0, 0, 0
numbers_ptr: dq 0
numbers_length: dq 0
numbers: dq 0
list_of_results: dq 0
num_float: dq 0.0
loop_counter_401: dq 0
numbers_ctr: dq 0
num: dq 0
result: dq 0
const_1: dq 1
i: dq 0
range_loop_start_i: dq 0
range_loop_end_i: dq 0
const_0: dq 0
; New vars for threads:
ThreadCount: times 4 dq 0
ThreadInfo: times 10 dq 0
ThreadInfo2: times 10 dq 0
ThreadInfo3: times 10 dq 0
ThreadInfo4: times 10 dq 0
TestInfo: times 4 dq 0
ThreadHandles: times 4 dq 0
Return_Data_Array: times 8 dq 0 ; ptr,ptr,ptr,ptr,length,length,length,length
StartByte: dq 0
stride: dq 8
output_buffer_pointers: times 4 dq 0 ; for malloc
Division_Size: dq 0
Division_Start: dq 0
section .text
; ______________________________________
Init_Cores_fn:
; Calculate the data divisions
mov rax,r10
mov rbx,4 ;cores
xor rdx,rdx
div rbx
mov [Division_Size],rax
mov [Division_Start],rax
; Populate the ThreadInfo array with vars to pass
; ThreadInfo: length, startbyte, stride, vars into registers on entry to each core
mov rdi,ThreadInfo
mov rax,0 ;ThreadInfoLength
mov [rdi],rax ; length (number of vars into registers plus 3 elements)
mov rax,[stride]
mov [rdi+16],rax ; 8 x number of cores (32 in this example)
; Vars (these registers are populated on main entry)
mov rbp,output_buffer_pointers
mov rax,[rbp]
mov [rdi+24],rax ;for r15
mov [rdi+32],r14
mov [rdi+40],r13
mov [rdi+48],r12
mov rax,[Division_Start]
mov [rdi+56],rax
mov rax,0
mov [rdi+64],rax
call DupThreadInfo
mov rbp,rsp ; preserve caller's stack frame
sub rsp,56 ; Shadow space (was 32)
; _____
label_0:
mov rax,[StartByte]
cmp rax,0
jne sb2
mov rdi,ThreadInfo
jmp sb5
sb2:cmp rax,8
jne sb3
mov rdi,ThreadInfo2
jmp sb5
sb3:cmp rax,16
jne sb4
mov rdi,ThreadInfo3
jmp sb5
sb4:cmp rax,24
jne sb5
mov rdi,ThreadInfo4
sb5:
; _____
; Create Threads
mov rcx,0 ; lpThreadAttributes (Security Attributes)
mov rdx,0 ; dwStackSize
mov r8,PrimeNumber_fn ; lpStartAddress (function pointer)
mov r9,rdi ; lpParameter (array of data passed to each core)
mov rax,0
mov [rsp+32],rax ; use default creation flags
mov rdi,ThreadCount
mov [rsp+40],rdi ; ThreadID
call CreateThread
; Move the handle into ThreadHandles array (returned in rax)
mov rdi,ThreadHandles
mov rcx,[StartByte]
mov [rdi+rcx],rax
mov rax,[StartByte]
add rax,8
mov [StartByte],rax
mov rbx,32 ; Four cores
cmp rax,rbx
jl label_0
; _____
; Wait
mov rcx,4 ;rax ; number of handles
mov rdx,ThreadHandles ; pointer to handles array
mov r8,0 ; wait for all threads to complete
mov r9,5000 ; milliseconds to wait
call WaitForMultipleObjects
; _____
mov rsp,rbp
jmp label_900
; ______________________________________
PrimeNumber_fn:
; Populate registers
;(note: rcx is the return value for ThreadProc)
mov rdi,rcx
mov rax,[rdi]
mov r15,[rdi+24]
mov r13,[rdi+40]
mov r12,[rdi+48]
mov r10,[rdi+56]
xor r11,r11
xor r9,r9
mov r8,[rdi+8] ; start byte
pxor xmm15,xmm15
pxor xmm15,xmm14
pxor xmm15,xmm13
; Get the ThreadID based on startbyte
mov rax,[rdi+72] ; 0, 8, 16, 24 for 4-core
push rax
;______
label_401:
mov rdi,r12 ; Pointer
cmp r8,r10
jge label_899
movsd xmm0,qword[rdi+r8]
movsd xmm15,xmm0
add r8,8
;______
cvttsd2si rax,xmm15
mov r9,rax
;______
label_8010:
label_801:
cmp r9,[const_1]
jle label_401
;______
label_12010:
mov rax,2
sub rax,1
movq xmm13,rax
label_1201:
movq rcx,xmm13
movq xmm1,[const_1]
addsd xmm13,xmm1
movq rcx,xmm13
mov rdx,r9
cmp rcx,rdx
jge label_12020
;______
label_16010:
label_1601:
mov rax,r9
movq rbx,xmm13
xor rdx,rdx
div rbx
mov rax,rdx
mov rdx,0
cmp rax,rdx
jne label_16020
;______
movq xmm14,[const_0]
;______
movq rax,xmm14
cvtsi2sd xmm0,rax
movsd [r15+r14],xmm14
add r14,8
mov rax,r13
cmp r14,rax
jl next_17
;[Irrelevant code omitted]
next_17:
;______
jmp label_401
;______
label_1602:
label_16020:
;______
mov rax,r9
cvtsi2sd xmm14,rax
;______
movq rax,xmm14
cvtsi2sd xmm0,rax
movsd [r15+r14],xmm14
add r14,8
mov rax,r13
cmp r14,rax
jl next_21
;[Irrelevant code omitted]
next_21:
jmp label_1201
;______
label_1202:
label_12020:
;______
movq xmm14,[const_0]
;______
movq rax,xmm14
cvtsi2sd xmm0,rax
movsd [r15+r14],xmm14
add r14,8
mov rax,r13
cmp r14,rax
jl next_25
;[Irrelevant code omitted]
next_25:
jmp label_401
;______
label_899:
pop rax
mov rdi,Return_Data_Array
mov [rdi+rax],r15
mov [rdi+rax+32],r14 ; 32 = four cores
call ExitThread
call GetCurrentThread
mov rcx,rax
call CloseHandle
ret
; __________
label_900:
exit_label_for_PrimeNumber_fn:
mov rdi,Return_Data_Array ; Final return to the calling process
mov rax,rdi
ret
;__________
;Free the memory
FreeMem_fn:
sub rsp,40
call free
add rsp,40
ret
; __________
; Main Entry
Main_Entry_fn:
push rdi
push rbp
push rbx
push r15
xor r15,r15
push r14
xor r14,r14
push r13
xor r13,r13
push r12
xor r12,r12
push r11
xor r11,r11
push r10
xor r10,r10
push r9
xor r9,r9
movsd [XMM_Stack+0],xmm15
movsd [XMM_Stack+8],xmm14
movsd [XMM_Stack+16],xmm13
push r8
xor r8,r8
mov [numbers_ptr],rcx
mov [data_master_ptr],rdx
; Now assign lengths
lea rdi,[data_master_ptr]
mov rbp,[rdi]
xor rcx,rcx
movsd xmm0,qword[rbp+rcx]
cvttsd2si rax,xmm0
mov [numbers_length],rax
add rcx,8
; __________
; malloc for dynamic arrays
lea rdi,[data_master_ptr]
mov rbp,[rdi]
movsd xmm0,qword[rbp]
cvttsd2si rax,xmm0
mov r8,rax
mov rdx,10
mul rdx
mov rdx,10000000
cmp rax,rdx
jl malloc_next
mov rax,r8
malloc_next:
mov rax,50000000
mov [initial_dynamic_length],rax
;__________
; Output buffer #1
mov rcx,qword[initial_dynamic_length] ; Initial size
xor rax,rax
sub rsp,40
call malloc
mov rdi,output_buffer_pointers
mov [rdi],rax
add rsp,40
mov rax,qword[initial_dynamic_length]
mov [list_of_results_length],rax
;__________
; Output buffer #2
mov rcx,qword[initial_dynamic_length] ; Initial size
xor rax,rax
sub rsp,40
call malloc
mov rdi,output_buffer_pointers
mov [rdi+8],rax
add rsp,40
mov rax,qword[initial_dynamic_length]
mov [list_of_results_length],rax
;__________
; Output buffer #3
mov rcx,qword[initial_dynamic_length] ; Initial size
xor rax,rax
sub rsp,40
call malloc
mov rdi,output_buffer_pointers
mov [rdi+16],rax
add rsp,40
mov rax,qword[initial_dynamic_length]
mov [list_of_results_length],rax
;__________
; Output buffer #4
mov rcx,qword[initial_dynamic_length] ; Initial size
xor rax,rax
sub rsp,40
call malloc
mov rdi,output_buffer_pointers
mov [rdi+24],rax
add rsp,40
mov rax,qword[initial_dynamic_length]
mov [list_of_results_length],rax
; __________
; Write variables to assigned registers
lea rdi,[rel list_of_results_ptr]
mov r15,qword[rdi]
mov r14,[list_of_results_ctr]
mov r13,[list_of_results_length]
lea rdi,[rel numbers_ptr]
mov r12,qword[rdi]
mov r11,[numbers_ctr]
mov r10,[numbers_length]
mov r9,[num]
movsd xmm15,[num_float]
movsd xmm14,[result]
movsd xmm13,[i]
mov r8,[loop_counter_401]
; __________
call Init_Cores_fn
exit_label_for_Main_Entry_fn:
pop r8
movsd xmm13,[XMM_Stack+0]
movsd xmm14,[XMM_Stack+8]
movsd xmm15,[XMM_Stack+16]
pop r9
pop r10
pop r11
pop r12
pop r13
pop r14
pop r15
pop rbx
pop rbp
pop rdi
ret
;_____________
DupThreadInfo:
mov rdi,ThreadInfo2
; StartByte and EndByte
mov rax,[Division_Start]
mov [rdi+8],rax
add rax,[Division_Size]
mov [Division_Start],rax
sub rax,8
mov [rdi+56],rax
mov rax,[stride]
mov [rdi+16],rax ; 8 x number of cores (32 in this example)
; Vars (these registers are populated on main entry)
mov rbp,output_buffer_pointers
mov rax,[rbp+8]
mov [rdi+24],rax
mov rax,8
mov [rdi+32],rax
mov [rdi+40],r13
mov [rdi+48],r12
; 56 is above
mov rax,8
mov [rdi+64],rax
mov rax,8 ; Thread number based on startbyte, for Return_Data_Array
mov [rdi+72],rax
; _____
mov rdi,ThreadInfo3
; StartByte and EndByte
mov rax,[Division_Start]
mov [rdi+8],rax
mov rbx,[Division_Size]
add rax,rbx
mov [Division_Start],rax
sub rax,8
mov [rdi+56],rax
mov rax,[stride]
mov [rdi+16],rax ; 8 x number of cores (32 in this example)
; Vars (these registers are populated on main entry)
mov rbp,output_buffer_pointers
mov rax,[rbp+16]
mov [rdi+24],rax
mov rax,16
mov [rdi+32],rax
mov [rdi+40],r13
mov [rdi+48],r12
; [rdi+56] is above
mov rax,16
mov [rdi+64],rax
mov rax,16 ; Thread number based on startbyte, for Return_Data_Array
mov [rdi+72],rax
mov rdi,ThreadInfo4
; StartByte and EndByte
mov rax,[Division_Start]
mov [rdi+8],rax
mov rax,[numbers_length] ; final segment goes to the end
mov [rdi+56],rax
mov rax,[stride]
mov [rdi+16],rax ; 8 x number of cores (32 in this example)
; Vars (these registers are populated on main entry)
mov rbp,output_buffer_pointers
mov rax,[rbp+24]
mov [rdi+24],rax
mov rax,24
mov [rdi+32],rax
mov [rdi+40],r13
mov [rdi+48],r12
; [rdi+56] is above
mov rax,24
mov [rdi+64],rax
mov rax,24 ; Thread number based on startbyte, for Return_Data_Array
mov [rdi+72],rax
ret