Using mutex to synchronize two kernel threads causes a system panic - linux

Question

I used the mutex to synchronize the two kernel threads. After running, panic occurred in the system and abnormal memory was found in mutex by kdump.
Here is a simplified code example, You can run it directly to reproduce the problem.
I changed the memory allocation method to use kmalloc instead of vmalloc, and then it worked, Who knows why？
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/delay.h>
#include <linux/spinlock.h>
#include <linux/mutex.h>
#include <linux/kthread.h>
#include <linux/slab.h>
struct product {
struct list_head list;
struct mutex lock;
bool finish;
};
struct task_struct *task1;
struct task_struct *task2;
spinlock_t spin;
struct list_head products;
struct product *create_product(void)
{
struct product *p_prod;
p_prod = vmalloc(sizeof(struct product));
// p_prod = kmalloc(sizeof(struct product), GFP_KERNEL);
if(!p_prod)
return NULL;
INIT_LIST_HEAD(&p_prod->list);
mutex_init(&p_prod->lock);
p_prod->finish = false;
return p_prod;
}
void remove_product(struct product **pp_prod)
{
vfree(*pp_prod);
// kfree(*pp_prod);
*pp_prod = NULL;
}
int producer(void *data)
{
while(!kthread_should_stop())
{
struct product *p_prod = create_product();
if(!p_prod)
continue;
spin_lock(&spin);
list_add_tail(&p_prod->list, &products);
spin_unlock(&spin);
while (true)
{
mutex_lock(&p_prod->lock);
if(p_prod->finish)
{
mutex_unlock(&p_prod->lock);
schedule();
break;
}
mutex_unlock(&p_prod->lock);
}
remove_product(&p_prod);
}
do_exit(0);
}
int consumer(void *data)
{
while(!kthread_should_stop())
{
struct product *p_prod;
spin_lock(&spin);
if(list_empty(&products))
{
spin_unlock(&spin);
schedule();
continue;
}
p_prod = list_first_entry(&products, struct product, list);
list_del(&p_prod->list);
spin_unlock(&spin);
mutex_lock(&p_prod->lock);
p_prod->finish = true;
mutex_unlock(&p_prod->lock);
}
do_exit(0);
}
static int __init kdemo_init(void) {
printk(">>> demo driver begin!\n");
spin_lock_init(&spin);
INIT_LIST_HEAD(&products);
task1 = kthread_run(producer, NULL, "hdz-producer");
task2 = kthread_run(consumer, NULL, "hdz-consumer");
return 0;
}
static void __exit kdemo_exit(void) {
kthread_stop(task1);
kthread_stop(task2);
printk(">>> demo driver exit!\n");
}
module_init(kdemo_init);
module_exit(kdemo_exit);
MODULE_LICENSE("GPL");
MODULE_AUTHOR("xxxx#xxx.com");
MODULE_VERSION("1.0");
dmesg log and consumer stack
[ 176.599116] >>> demo driver begin!
[ 177.167659] BUG: unable to handle kernel NULL pointer dereference at 0000000000000fb0
[ 177.167695] IP: [<ffffffff9e0caa47>] wake_q_add+0x17/0x50
[ 177.167719] PGD 0
[ 177.167729] Oops: 0002 [#1] SMP
[ 177.167743] Modules linked in: kdemo(OE) mpt3sas mptctl mptbase nvmet_rdma nvmet nvme_rdma nvme_fabrics nvme nvme_core drbd(OE) dell_rbu kvdo(OE) uds(OE) bonding sha512_ssse3 sha512_generic qat_api(OE) usdm_drv(OE) intel_qat(OE) authenc uio ib_isert iscsi_target_mod ib_srpt target_core_mod ib_srp scsi_transport_srp scsi_tgt ib_ucm rpcrdma sunrpc rdma_ucm ib_umad ib_uverbs ib_iser rdma_cm ib_ipoib iw_cm libiscsi scsi_transport_iscsi ib_cm mlx5_ib ib_core intelcas(OE) inteldisk(OE) iTCO_wdt iTCO_vendor_support dell_smbios sparse_keymap dcdbas skx_edac intel_powerclamp coretemp intel_rapl iosf_mbi kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd sg joydev pcspkr ipmi_si i2c_i801 lpc_ich shpchp ipmi_devintf ipmi_msghandler mei_me acpi_power_meter
[ 177.168071] mei acpi_pad wmi nfit libnvdimm dm_multipath binfmt_misc ip_tables xfs libcrc32c mgag200 drm_kms_helper crc32c_intel syscopyarea sysfillrect sysimgblt mlx5_core fb_sys_fops ttm ixgbe drm igb mlxfw devlink mdio ptp i2c_algo_bit pps_core i2c_core dca sr_mod cdrom sd_mod crc_t10dif crct10dif_generic crct10dif_pclmul crct10dif_common ahci libahci libata mpt2sas raid_class scsi_transport_sas megaraid_sas dm_mirror dm_region_hash dm_log dm_mod
[ 177.168263] CPU: 24 PID: 5412 Comm: hdz-consumer Kdump: loaded Tainted: G OE ------------ 3.10.0-862.el7.x86_64 #1
[ 177.168297] Hardware name: Dell Inc. PowerEdge R740/08D89F, BIOS 2.10.2 02/24/2021
[ 177.168320] task: ffff93db22af3f40 ti: ffff93dc89354000 task.ti: ffff93dc89354000
[ 177.168344] RIP: 0010:[<ffffffff9e0caa47>] [<ffffffff9e0caa47>] wake_q_add+0x17/0x50
[ 177.168372] RSP: 0018:ffff93dc89357e48 EFLAGS: 00010246
[ 177.168389] RAX: 0000000000000000 RBX: ffffbe2ce6533018 RCX: 0000000000000fb0
[ 177.168410] RDX: 0000000000000001 RSI: 0000000000000000 RDI: ffff93dc89357e58
[ 177.168432] RBP: ffff93dc89357e48 R08: ffffbe2ce6533000 R09: 0000000000000000
[ 177.168453] R10: 0000000000000001 R11: 0000000000000001 R12: ffffbe2ce6533014
[ 177.168475] R13: ffff93dc89357e58 R14: 0000000000000000 R15: 0000000000000000
[ 177.168497] FS: 0000000000000000(0000) GS:ffff93dca9400000(0000) knlGS:0000000000000000
[ 177.168540] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 177.168560] CR2: 0000000000000fb0 CR3: 00000002b7a0e000 CR4: 00000000007607e0
[ 177.168583] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000
[ 177.168606] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400
[ 177.168629] PKRU: 00000000
[ 177.168640] Call Trace:
[ 177.168656] [<ffffffff9e711b2e>] __mutex_unlock_slowpath+0x5e/0x90
[ 177.168679] [<ffffffff9e710fab>] mutex_unlock+0x1b/0x20
[ 177.168699] [<ffffffffc0637064>] consumer+0x64/0x90 [kdemo]
[ 177.168723] [<ffffffffc0637000>] ? 0xffffffffc0636fff
[ 177.168746] [<ffffffff9e0bae31>] kthread+0xd1/0xe0
[ 177.168765] [<ffffffff9e0bad60>] ? insert_kthread_work+0x40/0x40
[ 177.168788] [<ffffffff9e71f61d>] ret_from_fork_nospec_begin+0x7/0x21
[ 177.168811] [<ffffffff9e0bad60>] ? insert_kthread_work+0x40/0x40
[ 177.168831] Code: 09 00 00 31 c9 31 d2 e8 18 41 ff ff eb e4 66 0f 1f 44 00 00 0f 1f 44 00 00 55 48 8d 8e b0 0f 00 00 31 c0 ba 01 00 00 00 48 89 e5 <f0> 48 0f b1 96 b0 0f 00 00 48 85 c0 74 0b 5d c3 66 0f 1f 84 00
[ 177.168996] RIP [<ffffffff9e0caa47>] wake_q_add+0x17/0x50
[ 177.169017] RSP <ffff93dc89357e48>
[ 177.169959] CR2: 0000000000000fb0
producer stask
crash> bt 5411
PID: 5411 TASK: ffff93db22af4f10 CPU: 1 COMMAND: "hdz-producer"
bt: page excluded: kernel virtual address: ffffffffffffffff type: "cpu_online_map"
#0 [ffff93dca8e48e48] crash_nmi_callback at ffffffff9e0533b7
#1 [ffff93dca8e48e58] nmi_handle at ffffffff9e71790c
#2 [ffff93dca8e48eb0] do_nmi at ffffffff9e717b2d
#3 [ffff93dca8e48ef0] end_repeat_nmi at ffffffff9e716d79
#4 [ffff93dca8e48f28] __vmalloc_node_range at ffffffff9e1d7518
[exception RIP: mutex_unlock+20]
RIP: ffffffff9e710fa4 RSP: ffff93ddafd73e98 RFLAGS: 00000202
RAX: 0000000000000010 RBX: 0000000000000010 RCX: 0000000000000202
RDX: ffff93ddafd73e98 RSI: 0000000000000018 RDI: 0000000000000001
RBP: ffffffff9e710fa4 R8: ffffffff9e710fa4 R9: 0000000000000018
R10: ffff93ddafd73e98 R11: 0000000000000202 R12: ffffffffffffffff
R13: ffffbe2ce6535010 R14: ffffffffc0639240 R15: 0000000000000000
ORIG_RAX: ffffffffc0639240 CS: 0010 SS: 0018
--- <(unknown) exception stack> ---
#5 [ffff93ddafd73e98] mutex_unlock at ffffffff9e710fa4
#6 [ffff93ddafd73ea0] producer at ffffffffc0637145 [kdemo]
#7 [ffff93ddafd73ec8] kthread at ffffffff9e0bae31
#8 [ffff93ddafd73f50] ret_from_fork_nospec_begin at ffffffff9e71f61d