From 2572a03f1e747fa4defcd94bb357236930362bb1 Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Thu, 28 Apr 2022 18:53:03 +0900 Subject: [PATCH 01/25] Introduce CC-lock infrastructure CC-lock is based on flat-lock combining algorithm. In this lock, only one thread, called combiner thread the request of critical section. So, combiner thread can exploit locality and aviod high contetetion between lock varible. When each cpu use only one node, let's assume lock hold node A. In this case, node A's (wait, completed) status should be (false, false). Lock | A When A,B cpu race occured, Let's assume that B is win. Then, B will try to spin on A's wait Status. A -> B w:F w:T At the same time, A was enqueued. So, A's wait status was set to True like below. A -> B -> A w:T w:T w:T This lead to deadlock. To avoid above node-reusing problem, each cpu has two cc_node. Those node are used alternately. A_0 -> B_0 -> A_1 w:f w:T w:T Signed-off-by: Wonhyuk Yang --- lock_module/Kbuild | 3 + lock_module/Makefile | 13 ++ lock_module/modul.c | 458 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 474 insertions(+) create mode 100644 lock_module/Kbuild create mode 100644 lock_module/Makefile create mode 100644 lock_module/modul.c diff --git a/lock_module/Kbuild b/lock_module/Kbuild new file mode 100644 index 000000000..aa07e2251 --- /dev/null +++ b/lock_module/Kbuild @@ -0,0 +1,3 @@ +EXTRA_CFLAGS = -Wall -g + +obj-m = modul.o diff --git a/lock_module/Makefile b/lock_module/Makefile new file mode 100644 index 000000000..0585d56fd --- /dev/null +++ b/lock_module/Makefile @@ -0,0 +1,13 @@ +KDIR = /lib/modules/`uname -r`/build +MODULE_NAME=modul +kbuild: + make -C $(KDIR) M=`pwd` + +clean: + make -C $(KDIR) M=`pwd` clean + +install: kbuild + sudo insmod $(MODULE_NAME).ko + +uninstall: + sudo rmmod $(MODULE_NAME) diff --git a/lock_module/modul.c b/lock_module/modul.c new file mode 100644 index 000000000..67370ac0a --- /dev/null +++ b/lock_module/modul.c @@ -0,0 +1,458 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +MODULE_DESCRIPTION("Simple Lock benchmark module"); +MODULE_AUTHOR("Wonhyuk Yang"); +MODULE_LICENSE("GPL"); + +#define MAX_COMBINER_OPERATIONS 5 + +/* CPU number and index are encoded in cc_node.next + * Now, each cpu has two node and these nodes are + * used alternatley. In this way, we can avoid + * node overwrite problem. + */ +#define INDEX_SHIFT (1) +#define INDEX_MASK ((1 << INDEX_SHIFT) - 1) +#define ENCODE_NEXT(x, y) ((x << INDEX_SHIFT) | (y & INDEX_MASK)) + +#define DECODE_IDX(x) (x & INDEX_MASK) +#define DECODE_CPU(x) (x >> INDEX_SHIFT) + +#define GET_NEXT_NODE(x, y) (per_cpu(x, DECODE_CPU(y)) + DECODE_IDX(y)) + +typedef void* (*request_t)(void *); +int prepare_tests(void); + +struct cc_node { + request_t req; + void* params; + void* ret; + bool wait; + bool completed; + int next; +}; + +DEFINE_PER_CPU(struct cc_node, node_array[2]) = { + { + .req = NULL, + .params = NULL, + .ret = NULL, + .wait = false, + .completed = false, + .next = ENCODE_NEXT(NR_CPUS, 1), + }, + { + .req = NULL, + .params = NULL, + .ret = NULL, + .wait = false, + .completed = false, + .next = ENCODE_NEXT(NR_CPUS, 0), + }, +}; + +struct lb_info { + request_t req; + void *params; + atomic_t *lock; + int counter; + bool monitor; + bool quit; +}; + +DEFINE_PER_CPU(struct lb_info, lb_info_array); +DEFINE_PER_CPU(struct task_struct *, task_array); + +/* At first, lock is NULL value. This mean pointing the + * CPU 0, idx 0. Thus, To make it consistent node_array_idx + * should be set 1 + */ +DEFINE_PER_CPU(int, node_array_idx) = 1; + +void* execute_cs(request_t req, void *params, atomic_t *lock) +{ + struct cc_node *prev, *pending; + struct cc_node *next; + int counter = 0; + unsigned int this_cpu = get_cpu(); + unsigned int pending_cpu; + unsigned int prev_cpu; + + /* get/update node_arra_idx */ + int this_cpu_idx = per_cpu(node_array_idx, this_cpu); + per_cpu(node_array_idx, this_cpu) = this_cpu_idx? 0 : 1; + + this_cpu = ENCODE_NEXT(this_cpu, this_cpu_idx); + next = GET_NEXT_NODE(node_array, this_cpu); + next->req = NULL; + next->params = NULL; + next->ret = NULL; + next->wait = true; + next->completed = false; + next->next = ENCODE_NEXT(NR_CPUS, 0); + + prev_cpu = atomic_xchg(lock, this_cpu); + WARN(prev_cpu == this_cpu, "lockbench: prev_cpu == this_cpu, Can't be happend!!!"); + + prev = GET_NEXT_NODE(node_array, prev_cpu); + WRITE_ONCE(prev->req, req); + WRITE_ONCE(prev->params, params); + smp_mb(); + WRITE_ONCE(prev->next, this_cpu); + + /* Failed to get lock */ + pr_debug("lockbench: prev{CPU: (%d, %d), wait:%d, completed:%d}\n" + "lockbench: next{CPU: (%d, %d), wait:%d, completed:%d}\n", + DECODE_CPU(prev_cpu), DECODE_IDX(prev_cpu), + prev->wait, prev->completed, + DECODE_CPU(this_cpu), DECODE_IDX(this_cpu), + next->wait, next->completed); + + pr_debug("lockbench: Spinning start!\n"); + while (READ_ONCE(prev->wait)) + cpu_relax(); + + if (READ_ONCE(prev->completed)) { + put_cpu(); + pr_debug("lockbench: CPU: (%d, %d) end of critical section!\n", + DECODE_CPU(this_cpu), DECODE_IDX(this_cpu)); + return prev->ret; + } + + /* Success to get lock */ + pending_cpu = prev_cpu; + + while (DECODE_CPU(pending_cpu) != NR_CPUS && counter++ < MAX_COMBINER_OPERATIONS) { + pending = GET_NEXT_NODE(node_array, pending_cpu); + pr_debug("lockbench: CPU: (%d, %d), next_cpu: (%d, %d), request: %pF\n", + DECODE_CPU(pending_cpu), DECODE_IDX(pending_cpu), + DECODE_CPU(pending->next), DECODE_IDX(pending->next), + pending->req); + + /* Preserve store order completed -> wait -> next */ + if (pending->req != NULL) { + pending->ret = pending->req(pending->params); + if (DECODE_CPU(READ_ONCE(pending->next)) != NR_CPUS) { + /* In case of this_cpu == prev_cpu, Don't set */ + WRITE_ONCE(pending->completed, true); + } + } + WRITE_ONCE(pending->wait, false); + pending_cpu = READ_ONCE(pending->next); + } + /* Pass tho combiner thread role */ + if (DECODE_CPU(pending_cpu) != NR_CPUS) { + pr_debug("lockbench: pass the combiner role to CPU: (%d, %d)\n", + DECODE_CPU(pending_cpu), DECODE_IDX(pending_cpu)); + + pending = GET_NEXT_NODE(node_array, pending_cpu); + pending->wait = false; + } + + put_cpu(); + pr_debug("lockbench: end of critical section!\n"); + return prev->ret; +} + +/* Dummy workload */ +atomic_t dummy_lock = ATOMIC_INIT(0); +int dummy_counter = 0; +void* dummy_increment(void* params) +{ + int *counter = (int*)params; + if (unlikely(counter == NULL)) { + printk("!!!! counter: %p", (counter)); + } else { + (*counter)++; + } + return params; +} + +/* Debugfs */ +static int +lb_open(struct inode *inode, struct file *filep) +{ + return 0; +} + +static int +lb_release(struct inode *inode, struct file *filep) +{ + return 0; +} + +static ssize_t lb_write(struct file *filp, const char __user *ubuf, + size_t cnt, loff_t *ppos) +{ + unsigned long val; + int ret; + ret = kstrtoul_from_user(ubuf, cnt, 10, &val); + + if (ret) + return ret; + + if (val == 1) { + prepare_tests(); + } + (*ppos)++; + return cnt; +} + +static ssize_t lb_quit(struct file *filp, const char __user *ubuf, + size_t cnt, loff_t *ppos) +{ + unsigned long val; + int ret, cpu; + struct lb_info *ld; + struct cc_node *node; + ret = kstrtoul_from_user(ubuf, cnt, 10, &val); + + if (ret) + return ret; + + if (val == 1) { + for_each_online_cpu(cpu) { + ld = &per_cpu(lb_info_array, cpu); + ld->quit = true; + + smp_mb(); + node = per_cpu(node_array, cpu); + node[0].wait = false; + node[0].completed = true; + node[1].wait = false; + node[1].completed = true; + } + } + (*ppos)++; + return cnt; +} + +static void *t_start(struct seq_file *m, loff_t *pos) +{ + return *pos < 1 ? (void *)1 : NULL; +} + +static void *t_next(struct seq_file *m, void *v, loff_t *pos) +{ + ++*pos; + return NULL; +} + +static void t_stop(struct seq_file *m, void *v) +{ +} + +static int t_show(struct seq_file *m, void *v) +{ + int cpu; + struct cc_node *node; + + seq_printf(m, "\n"); + seq_printf(m, "dummy_lock: (%d, %d)\n", + DECODE_CPU(dummy_lock.counter), DECODE_IDX(dummy_lock.counter)); + for_each_online_cpu(cpu) { + node = per_cpu(node_array, cpu); + seq_printf(m, "Node(%d, %d) {\n" + "\treq = %pF,\n" + "\tparams = %p,\n" + "\twait = %d, completed = %d,\n" + "\tNext (%d, %d)\n}\n", + cpu, 0, + node[0].req, node[0].params, + node[0].wait, node[0].completed, + DECODE_CPU(node[0].next), + DECODE_IDX(node[0].next)); + seq_printf(m, "Node(%d, %d) {\n" + "\treq = %pF,\n" + "\tparams = %p,\n" + "\twait = %d, completed = %d,\n" + "\tNext (%d, %d)\n}\n", + cpu, 1, + node[1].req, node[1].params, + node[1].wait, node[1].completed, + DECODE_CPU(node[1].next), + DECODE_IDX(node[1].next)); + } + + return 0; +} + +static const struct seq_operations show_status_seq_ops= { + .start = t_start, + .next = t_next, + .stop = t_stop, + .show = t_show, +}; + +static int lb_status_open(struct inode *inode, struct file *file) +{ + struct seq_file *m; + int ret; + + ret = seq_open(file, &show_status_seq_ops); + if (ret) { + return ret; + } + + m = file->private_data; + return 0; +} + +static int lb_status_release(struct inode *inode, struct file *file) +{ + return seq_release(inode, file); +} + +static const struct file_operations lb_trigger_fops = { + .open = lb_open, + .read = NULL, + .write = lb_write, + .release = lb_release, + .llseek = NULL, +}; + +static const struct file_operations lb_quit_fops = { + .open = lb_open, + .read = NULL, + .write = lb_quit, + .release = lb_release, + .llseek = NULL, +}; + +static const struct file_operations lb_status_fops= { + .open = lb_status_open, + .read = seq_read, + .llseek = seq_lseek, + .release = lb_status_release, +}; + +static struct dentry *lb_debugfs_root; + +static int lb_debugfs_init(void) +{ + lb_debugfs_root = debugfs_create_dir("lock_benchmark", NULL); + + debugfs_create_file("trigger", 0200, + lb_debugfs_root, NULL, &lb_trigger_fops); + debugfs_create_file("quit", 0200, + lb_debugfs_root, NULL, &lb_quit_fops); + debugfs_create_file("status", 0400, + lb_debugfs_root, NULL, &lb_status_fops); + + return 0; +} + +static int lb_debugfs_exit(void) +{ + debugfs_remove_recursive(lb_debugfs_root); + return 0; +} + +int test_thread(void *data) +{ + int i; + struct lb_info * lb_data = (struct lb_info *)data; + int cpu = get_cpu(); + u64 prev, cur; + + prev = get_jiffies_64(); + for (i=0; icounter && !READ_ONCE(lb_data->quit); i++) { + if (unlikely(lb_data->monitor && i && ((i % 1000)==0))) { + cur = get_jiffies_64(); + printk("lockbench: monitor thread %dth [%lld]\n", i, cur-prev); + prev = cur; + } + execute_cs(lb_data->req, lb_data->params, lb_data->lock); + } + + per_cpu(task_array, cpu) = NULL; + put_cpu(); + return 0; +} + +int prepare_tests(void) +{ + struct task_struct *thread; + struct lb_info *li; + bool monitor_thread = true; + int cpu; + int max_cpus = 6; + int nr_cpus = 0; + + for_each_online_cpu(cpu) { + thread = per_cpu(task_array, cpu); + if (thread != NULL) { + pr_debug("lockbench: test is progressing!\n"); + return 1; + } + } + for_each_online_cpu(cpu) { + if (nr_cpus++ >= max_cpus) + break; + + li = &per_cpu(lb_info_array, cpu); + li->req = dummy_increment; + li->params = &dummy_counter; + li->lock = &dummy_lock; + li->counter = 1000 * 10; + li->monitor = monitor_thread; + + thread = kthread_create(test_thread, (void *)li, "lockbench/%u", cpu); + if (IS_ERR(thread)) { + pr_err("Failed to create kthread on CPU %u\n", cpu); + continue; + } + kthread_bind(thread, cpu); + + wake_up_process(thread); + per_cpu(task_array, cpu) = thread; + + monitor_thread = false; + } + return 0; +} + +/* module init/exit */ +static int lock_benchmark_init(void) +{ + lb_debugfs_init(); + return 0; +} + +static void lock_benchmark_exit(void) +{ + int cpu; + struct lb_info *ld; + struct cc_node *node; + + for_each_online_cpu(cpu) { + ld = &per_cpu(lb_info_array, cpu); + ld->quit = true; + + smp_mb(); + node = per_cpu(node_array, cpu); + node[0].wait = false; + node[0].completed = true; + node[1].wait = false; + node[1].completed = true; + } + lb_debugfs_exit(); +} + +module_init(lock_benchmark_init); +module_exit(lock_benchmark_exit); From a002e541da97bed04fe561a92e61c942a3a3217c Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Fri, 29 Apr 2022 11:32:00 +0900 Subject: [PATCH 02/25] Fix node's memory ordering issue Test reported that there is a deadlock. Situation are below. Node(0, 1) { req = 00000000d0495726, params = 000000002f36f5ac, wait = 0, completed = 1, refcount = 0, Next (2, 0) Prev (0, 0) } Node(2, 0) { req = 00000000d0495726, params = 000000002f36f5ac, wait = 1, completed = 0, refcount = 0, Next (2, 1) Prev (0, 1) } Node (0, 1)'s request are handled. So, it wait, completed status are (0, 1). But, it's next node Node(2, 0)'s wait are still 1. The combiner thread should set Node(2, 0) wait = 0. Previous logic set wait = 0, when DECODE_CPU(pending_cpu) != NR_CPUS. But there can be race between combiner thread and normal thread. In the combiner thread it check node->req first, then it check node->next. So there could be a situation below Node(0, 1) Node(2, 0) prev->req = req if(pending->req) ... DECODE_CPU(pending->next) prev->next = this_cpu To fix this, combiner thread check node->next first. Signed-off-by: Wonhyuk Yang --- lock_module/modul.c | 74 ++++++++++++++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 24 deletions(-) diff --git a/lock_module/modul.c b/lock_module/modul.c index 67370ac0a..c6360ce16 100644 --- a/lock_module/modul.c +++ b/lock_module/modul.c @@ -44,7 +44,9 @@ struct cc_node { void* ret; bool wait; bool completed; + int prev; int next; + atomic_t refcount; }; DEFINE_PER_CPU(struct cc_node, node_array[2]) = { @@ -54,7 +56,9 @@ DEFINE_PER_CPU(struct cc_node, node_array[2]) = { .ret = NULL, .wait = false, .completed = false, + .prev = ENCODE_NEXT(NR_CPUS, 0), .next = ENCODE_NEXT(NR_CPUS, 1), + .refcount = ATOMIC_INIT(0), }, { .req = NULL, @@ -62,7 +66,9 @@ DEFINE_PER_CPU(struct cc_node, node_array[2]) = { .ret = NULL, .wait = false, .completed = false, + .prev = ENCODE_NEXT(NR_CPUS, 1), .next = ENCODE_NEXT(NR_CPUS, 0), + .refcount = ATOMIC_INIT(0), }, }; @@ -105,14 +111,18 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) next->wait = true; next->completed = false; next->next = ENCODE_NEXT(NR_CPUS, 0); + smp_wmb(); + atomic_inc(&next->refcount); prev_cpu = atomic_xchg(lock, this_cpu); WARN(prev_cpu == this_cpu, "lockbench: prev_cpu == this_cpu, Can't be happend!!!"); + next->prev = prev_cpu; prev = GET_NEXT_NODE(node_array, prev_cpu); + /* request and parameter should be set. Then, next should be set */ WRITE_ONCE(prev->req, req); WRITE_ONCE(prev->params, params); - smp_mb(); + smp_wmb(); WRITE_ONCE(prev->next, this_cpu); /* Failed to get lock */ @@ -128,6 +138,7 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) cpu_relax(); if (READ_ONCE(prev->completed)) { + atomic_dec(&next->refcount); put_cpu(); pr_debug("lockbench: CPU: (%d, %d) end of critical section!\n", DECODE_CPU(this_cpu), DECODE_IDX(this_cpu)); @@ -137,33 +148,36 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) /* Success to get lock */ pending_cpu = prev_cpu; - while (DECODE_CPU(pending_cpu) != NR_CPUS && counter++ < MAX_COMBINER_OPERATIONS) { + while (counter++ < MAX_COMBINER_OPERATIONS) { pending = GET_NEXT_NODE(node_array, pending_cpu); + pending_cpu = READ_ONCE(pending->next); + /* Keep ordering next -> (req, params)*/ + smp_rmb(); + + /* Branch prediction: which case is more profitable? */ + if (DECODE_CPU(pending_cpu) == NR_CPUS) + goto out; + pr_debug("lockbench: CPU: (%d, %d), next_cpu: (%d, %d), request: %pF\n", DECODE_CPU(pending_cpu), DECODE_IDX(pending_cpu), DECODE_CPU(pending->next), DECODE_IDX(pending->next), pending->req); /* Preserve store order completed -> wait -> next */ - if (pending->req != NULL) { - pending->ret = pending->req(pending->params); - if (DECODE_CPU(READ_ONCE(pending->next)) != NR_CPUS) { - /* In case of this_cpu == prev_cpu, Don't set */ - WRITE_ONCE(pending->completed, true); - } - } + WARN(pending->req == NULL, "lockbench: pending->req == NULL..."); + pending->ret = pending->req(pending->params); + WRITE_ONCE(pending->completed, true); WRITE_ONCE(pending->wait, false); - pending_cpu = READ_ONCE(pending->next); } /* Pass tho combiner thread role */ - if (DECODE_CPU(pending_cpu) != NR_CPUS) { - pr_debug("lockbench: pass the combiner role to CPU: (%d, %d)\n", - DECODE_CPU(pending_cpu), DECODE_IDX(pending_cpu)); - - pending = GET_NEXT_NODE(node_array, pending_cpu); - pending->wait = false; - } - + WARN(DECODE_CPU(pending_cpu) == NR_CPUS, "lockbench: DECODE_CPU(pedning_cpu) == NR_CPUS..."); + pr_debug("lockbench: pass the combiner role to CPU: (%d, %d)\n", + DECODE_CPU(pending_cpu), DECODE_IDX(pending_cpu)); + + pending = GET_NEXT_NODE(node_array, pending_cpu); +out: + pending->wait = false; + atomic_dec(&next->refcount); put_cpu(); pr_debug("lockbench: end of critical section!\n"); return prev->ret; @@ -259,7 +273,7 @@ static void t_stop(struct seq_file *m, void *v) static int t_show(struct seq_file *m, void *v) { - int cpu; + int cpu, idx; struct cc_node *node; seq_printf(m, "\n"); @@ -267,26 +281,38 @@ static int t_show(struct seq_file *m, void *v) DECODE_CPU(dummy_lock.counter), DECODE_IDX(dummy_lock.counter)); for_each_online_cpu(cpu) { node = per_cpu(node_array, cpu); + idx = per_cpu(node_array_idx, cpu); + seq_printf(m, "Node idx: %d\n", idx); seq_printf(m, "Node(%d, %d) {\n" "\treq = %pF,\n" "\tparams = %p,\n" "\twait = %d, completed = %d,\n" - "\tNext (%d, %d)\n}\n", + "\trefcount = %lld,\n" + "\tNext (%d, %d)\n" + "\tPrev (%d, %d)\n}\n", cpu, 0, node[0].req, node[0].params, node[0].wait, node[0].completed, + node[0].refcount, DECODE_CPU(node[0].next), - DECODE_IDX(node[0].next)); + DECODE_IDX(node[0].next), + DECODE_CPU(node[0].prev), + DECODE_IDX(node[0].prev)); seq_printf(m, "Node(%d, %d) {\n" "\treq = %pF,\n" "\tparams = %p,\n" "\twait = %d, completed = %d,\n" - "\tNext (%d, %d)\n}\n", + "\trefcount = %lld,\n" + "\tNext (%d, %d)\n" + "\tPrev (%d, %d)\n}\n", cpu, 1, node[1].req, node[1].params, node[1].wait, node[1].completed, + node[1].refcount, DECODE_CPU(node[1].next), - DECODE_IDX(node[1].next)); + DECODE_IDX(node[1].next), + DECODE_CPU(node[1].prev), + DECODE_IDX(node[1].prev)); } return 0; @@ -391,7 +417,7 @@ int prepare_tests(void) struct lb_info *li; bool monitor_thread = true; int cpu; - int max_cpus = 6; + int max_cpus = 3; int nr_cpus = 0; for_each_online_cpu(cpu) { From 7d9bd9226dfe5a9b3d3bb7038fe72568eebdd6d6 Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Fri, 29 Apr 2022 12:31:00 +0900 Subject: [PATCH 03/25] Use sched_clock to measure time with high-resolution Previous, test thread used jiffes to measure the spent time. But, it's resolution is low. So all the results are zero or one. So use sched_clock. Signed-off-by: Wonhyuk Yang --- lock_module/modul.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lock_module/modul.c b/lock_module/modul.c index c6360ce16..9962df325 100644 --- a/lock_module/modul.c +++ b/lock_module/modul.c @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include #include #include @@ -394,18 +394,21 @@ int test_thread(void *data) int i; struct lb_info * lb_data = (struct lb_info *)data; int cpu = get_cpu(); - u64 prev, cur; + unsigned long prev, cur; - prev = get_jiffies_64(); + prev = sched_clock(); for (i=0; icounter && !READ_ONCE(lb_data->quit); i++) { if (unlikely(lb_data->monitor && i && ((i % 1000)==0))) { - cur = get_jiffies_64(); - printk("lockbench: monitor thread %dth [%lld]\n", i, cur-prev); + cur = sched_clock(); + printk("lockbench: monitor thread %dth [%lu]\n", i, cur-prev); prev = cur; } execute_cs(lb_data->req, lb_data->params, lb_data->lock); } + cur = sched_clock(); + printk("lockbench: monitor thread %dth [%lu]\n", i, cur-prev); + per_cpu(task_array, cpu) = NULL; put_cpu(); return 0; From be597542fa61900206acd26d7eb1440230a8e0af Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Fri, 29 Apr 2022 17:08:37 +0900 Subject: [PATCH 04/25] Change smp_rmb to smp_mb To keep order of reading node->next and writing of node->wait, node->completed, smp_mb should be used instead of smp_mb(). So fix it Signed-off-by: Wonhyuk Yang --- lock_module/modul.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lock_module/modul.c b/lock_module/modul.c index 9962df325..d740882a1 100644 --- a/lock_module/modul.c +++ b/lock_module/modul.c @@ -152,7 +152,7 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) pending = GET_NEXT_NODE(node_array, pending_cpu); pending_cpu = READ_ONCE(pending->next); /* Keep ordering next -> (req, params)*/ - smp_rmb(); + smp_mb(); /* Branch prediction: which case is more profitable? */ if (DECODE_CPU(pending_cpu) == NR_CPUS) From 6568b4910532751f9af83dfa8bdbfe2f6c8bea31 Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Fri, 29 Apr 2022 17:21:46 +0900 Subject: [PATCH 05/25] Make quit file to trigger reinit cc-lock Signed-off-by: Wonhyuk Yang --- lock_module/modul.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lock_module/modul.c b/lock_module/modul.c index d740882a1..f75512d10 100644 --- a/lock_module/modul.c +++ b/lock_module/modul.c @@ -240,6 +240,7 @@ static ssize_t lb_quit(struct file *filp, const char __user *ubuf, return ret; if (val == 1) { + dummy_lock.counter = ENCODE_NEXT(0, 0); for_each_online_cpu(cpu) { ld = &per_cpu(lb_info_array, cpu); ld->quit = true; @@ -251,6 +252,7 @@ static ssize_t lb_quit(struct file *filp, const char __user *ubuf, node[1].wait = false; node[1].completed = true; } + per_cpu(node_array, 0)[0].completed = false; } (*ppos)++; return cnt; @@ -430,6 +432,7 @@ int prepare_tests(void) return 1; } } + for_each_online_cpu(cpu) { if (nr_cpus++ >= max_cpus) break; From aac3ab2fa20a65d117517c28f676b82003207905 Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Mon, 2 May 2022 11:12:05 +0900 Subject: [PATCH 06/25] Add spinlock benchmark Using the "echo 2 > trigger", spinlock based benchmark can be run. Signed-off-by: Wonhyuk Yang --- lock_module/modul.c | 105 +++++++++++++++++++++++++++++++++----------- 1 file changed, 80 insertions(+), 25 deletions(-) diff --git a/lock_module/modul.c b/lock_module/modul.c index f75512d10..770af7db8 100644 --- a/lock_module/modul.c +++ b/lock_module/modul.c @@ -19,6 +19,8 @@ MODULE_DESCRIPTION("Simple Lock benchmark module"); MODULE_AUTHOR("Wonhyuk Yang"); MODULE_LICENSE("GPL"); +#define NR_BENCH (500000) + #define MAX_COMBINER_OPERATIONS 5 /* CPU number and index are encoded in cc_node.next @@ -36,7 +38,11 @@ MODULE_LICENSE("GPL"); #define GET_NEXT_NODE(x, y) (per_cpu(x, DECODE_CPU(y)) + DECODE_IDX(y)) typedef void* (*request_t)(void *); -int prepare_tests(void); +typedef int (*test_thread_t)(void *); + +int prepare_tests(test_thread_t, void *, const char *); +int test_thread(void *data); +int test_thread2(void *data); struct cc_node { request_t req; @@ -75,7 +81,7 @@ DEFINE_PER_CPU(struct cc_node, node_array[2]) = { struct lb_info { request_t req; void *params; - atomic_t *lock; + void *lock; int counter; bool monitor; bool quit; @@ -184,6 +190,7 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) } /* Dummy workload */ +DEFINE_SPINLOCK(dummy_spinlock); atomic_t dummy_lock = ATOMIC_INIT(0); int dummy_counter = 0; void* dummy_increment(void* params) @@ -215,14 +222,39 @@ static ssize_t lb_write(struct file *filp, const char __user *ubuf, { unsigned long val; int ret; + int cpu; + struct lb_info *li; + bool monitor_thread = true; + ret = kstrtoul_from_user(ubuf, cnt, 10, &val); if (ret) return ret; if (val == 1) { - prepare_tests(); + for_each_online_cpu(cpu) { + li = &per_cpu(lb_info_array, cpu); + li->req = dummy_increment; + li->params = &dummy_counter; + li->lock = (void *)&dummy_lock; + li->counter = NR_BENCH; + li->monitor = monitor_thread; + monitor_thread = false; + } + prepare_tests(test_thread, (void *)&lb_info_array, "cc-lockbench"); + } else if (val == 2) { + for_each_online_cpu(cpu) { + li = &per_cpu(lb_info_array, cpu); + li->req = dummy_increment; + li->params = &dummy_counter; + li->lock = (void *)&dummy_spinlock; + li->counter = NR_BENCH; + li->monitor = monitor_thread; + monitor_thread = false; + } + prepare_tests(test_thread2, (void *)&lb_info_array, "spinlockbench"); } + (*ppos)++; return cnt; } @@ -289,13 +321,13 @@ static int t_show(struct seq_file *m, void *v) "\treq = %pF,\n" "\tparams = %p,\n" "\twait = %d, completed = %d,\n" - "\trefcount = %lld,\n" + "\trefcount = %d,\n" "\tNext (%d, %d)\n" "\tPrev (%d, %d)\n}\n", cpu, 0, node[0].req, node[0].params, node[0].wait, node[0].completed, - node[0].refcount, + node[0].refcount.counter, DECODE_CPU(node[0].next), DECODE_IDX(node[0].next), DECODE_CPU(node[0].prev), @@ -304,13 +336,13 @@ static int t_show(struct seq_file *m, void *v) "\treq = %pF,\n" "\tparams = %p,\n" "\twait = %d, completed = %d,\n" - "\trefcount = %lld,\n" + "\trefcount = %d,\n" "\tNext (%d, %d)\n" "\tPrev (%d, %d)\n}\n", cpu, 1, node[1].req, node[1].params, node[1].wait, node[1].completed, - node[1].refcount, + node[1].refcount.counter, DECODE_CPU(node[1].next), DECODE_IDX(node[1].next), DECODE_CPU(node[1].prev), @@ -394,33 +426,65 @@ static int lb_debugfs_exit(void) int test_thread(void *data) { int i; - struct lb_info * lb_data = (struct lb_info *)data; int cpu = get_cpu(); + struct lb_info * lb_data = &per_cpu(*((struct lb_info *)data), cpu); unsigned long prev, cur; - prev = sched_clock(); + if (unlikely(lb_data->monitor)) + prev = sched_clock(); for (i=0; icounter && !READ_ONCE(lb_data->quit); i++) { if (unlikely(lb_data->monitor && i && ((i % 1000)==0))) { cur = sched_clock(); - printk("lockbench: monitor thread %dth [%lu]\n", i, cur-prev); + printk("lockbench: monitor thread %dth [%lu]\n", i, cur-prev); prev = cur; } execute_cs(lb_data->req, lb_data->params, lb_data->lock); } - cur = sched_clock(); - printk("lockbench: monitor thread %dth [%lu]\n", i, cur-prev); + if (unlikely(lb_data->monitor)) { + cur = sched_clock(); + printk("lockbench: monitor thread %dth [%lu]\n", i, cur-prev); + } + + per_cpu(task_array, cpu) = NULL; + put_cpu(); + return 0; +} + +int test_thread2(void *data) +{ + int i; + int cpu = get_cpu(); + struct lb_info * lb_data = &per_cpu(*((struct lb_info *)data), cpu); + unsigned long prev = 0, cur = 0; + spinlock_t *lock = (spinlock_t *)lb_data->lock; + + if (unlikely(lb_data->monitor)) + prev = sched_clock(); + for (i=0; icounter && !READ_ONCE(lb_data->quit); i++) { + if (unlikely(lb_data->monitor && i && ((i % 1000)==0))) { + cur = sched_clock(); + printk("lockbench: monitor thread %dth [%lu]\n", i, cur-prev); + prev = cur; + } + spin_lock(lock); + lb_data->req(lb_data->params); + spin_unlock(lock); + } + + if (unlikely(lb_data->monitor)) { + cur = sched_clock(); + printk("lockbench: monitor thread %dth [%lu]\n", i, cur-prev); + } per_cpu(task_array, cpu) = NULL; put_cpu(); return 0; } -int prepare_tests(void) +int prepare_tests(test_thread_t test, void *arg, const char *name) { struct task_struct *thread; - struct lb_info *li; - bool monitor_thread = true; int cpu; int max_cpus = 3; int nr_cpus = 0; @@ -437,14 +501,7 @@ int prepare_tests(void) if (nr_cpus++ >= max_cpus) break; - li = &per_cpu(lb_info_array, cpu); - li->req = dummy_increment; - li->params = &dummy_counter; - li->lock = &dummy_lock; - li->counter = 1000 * 10; - li->monitor = monitor_thread; - - thread = kthread_create(test_thread, (void *)li, "lockbench/%u", cpu); + thread = kthread_create(test, arg, "%s/%u", name, cpu); if (IS_ERR(thread)) { pr_err("Failed to create kthread on CPU %u\n", cpu); continue; @@ -453,8 +510,6 @@ int prepare_tests(void) wake_up_process(thread); per_cpu(task_array, cpu) = thread; - - monitor_thread = false; } return 0; } From b5d3e215601ebbfcd537ef03444f6e351ae8b5bd Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Mon, 2 May 2022 11:12:55 +0900 Subject: [PATCH 07/25] Add dmesg result parser Signed-off-by: Wonhyuk Yang --- lock_module/parse_dmesg.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 lock_module/parse_dmesg.py diff --git a/lock_module/parse_dmesg.py b/lock_module/parse_dmesg.py new file mode 100644 index 000000000..508221b88 --- /dev/null +++ b/lock_module/parse_dmesg.py @@ -0,0 +1,28 @@ +import subprocess + +cc_lock = {"name":"cc_lock", "nr":0, "time":list()} +spin_lock = {"name":"spin_lock", "nr":0, "time":list()} + +def print_stat(target): + print("[{0}] average time: {1}, nr: {2}".format(target["name"], + sum(target["time"]) / target["nr"], target["nr"])) + +if __name__ == "__main__": + lines = subprocess.check_output(["sudo", "dmesg"]).decode() + lines = lines.split("\n") + for line in lines: + if "cc-lock" in line: + target = cc_lock + elif "spinlock" in line: + target = spin_lock + else: + target = None + if target is None: + continue + + target["nr"] += 1 + target["time"].append(int(line[line.rfind("[")+1:-1])) + + print_stat(cc_lock) + print_stat(spin_lock) + From 8d7c31f46629cc655ce3de9e36da205eb82a8c7e Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Thu, 12 May 2022 03:32:27 +0000 Subject: [PATCH 08/25] Seperate debug code with ifdef macrow Signed-off-by: Wonhyuk Yang --- lock_module/modul.c | 58 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/lock_module/modul.c b/lock_module/modul.c index 770af7db8..2306e6d1e 100644 --- a/lock_module/modul.c +++ b/lock_module/modul.c @@ -19,6 +19,7 @@ MODULE_DESCRIPTION("Simple Lock benchmark module"); MODULE_AUTHOR("Wonhyuk Yang"); MODULE_LICENSE("GPL"); +//#define DEBUG #define NR_BENCH (500000) #define MAX_COMBINER_OPERATIONS 5 @@ -50,9 +51,11 @@ struct cc_node { void* ret; bool wait; bool completed; - int prev; int next; +#ifdef DEBUG + int prev; atomic_t refcount; +#endif }; DEFINE_PER_CPU(struct cc_node, node_array[2]) = { @@ -62,9 +65,11 @@ DEFINE_PER_CPU(struct cc_node, node_array[2]) = { .ret = NULL, .wait = false, .completed = false, - .prev = ENCODE_NEXT(NR_CPUS, 0), .next = ENCODE_NEXT(NR_CPUS, 1), +#ifdef DEBUG + .prev = ENCODE_NEXT(NR_CPUS, 0), .refcount = ATOMIC_INIT(0), +#endif }, { .req = NULL, @@ -72,9 +77,11 @@ DEFINE_PER_CPU(struct cc_node, node_array[2]) = { .ret = NULL, .wait = false, .completed = false, - .prev = ENCODE_NEXT(NR_CPUS, 1), .next = ENCODE_NEXT(NR_CPUS, 0), +#ifdef DEBUG + .prev = ENCODE_NEXT(NR_CPUS, 1), .refcount = ATOMIC_INIT(0), +#endif }, }; @@ -104,6 +111,7 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) unsigned int this_cpu = get_cpu(); unsigned int pending_cpu; unsigned int prev_cpu; + request_t pending_req; /* get/update node_arra_idx */ int this_cpu_idx = per_cpu(node_array_idx, this_cpu); @@ -119,10 +127,14 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) next->next = ENCODE_NEXT(NR_CPUS, 0); smp_wmb(); +#ifdef DEBUG atomic_inc(&next->refcount); +#endif prev_cpu = atomic_xchg(lock, this_cpu); WARN(prev_cpu == this_cpu, "lockbench: prev_cpu == this_cpu, Can't be happend!!!"); +#ifdef DEBUG next->prev = prev_cpu; +#endif prev = GET_NEXT_NODE(node_array, prev_cpu); /* request and parameter should be set. Then, next should be set */ @@ -144,7 +156,9 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) cpu_relax(); if (READ_ONCE(prev->completed)) { +#ifdef DEBUG atomic_dec(&next->refcount); +#endif put_cpu(); pr_debug("lockbench: CPU: (%d, %d) end of critical section!\n", DECODE_CPU(this_cpu), DECODE_IDX(this_cpu)); @@ -158,7 +172,7 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) pending = GET_NEXT_NODE(node_array, pending_cpu); pending_cpu = READ_ONCE(pending->next); /* Keep ordering next -> (req, params)*/ - smp_mb(); + smp_rmb(); /* Branch prediction: which case is more profitable? */ if (DECODE_CPU(pending_cpu) == NR_CPUS) @@ -169,10 +183,12 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) DECODE_CPU(pending->next), DECODE_IDX(pending->next), pending->req); + pending_req = READ_ONCE(pending->req); /* Preserve store order completed -> wait -> next */ - WARN(pending->req == NULL, "lockbench: pending->req == NULL..."); - pending->ret = pending->req(pending->params); + WARN(pending_req == NULL, "lockbench: pending->req == NULL..."); + WRITE_ONCE(pending->ret, pending_req(pending->params)); WRITE_ONCE(pending->completed, true); + smp_wmb(); WRITE_ONCE(pending->wait, false); } /* Pass tho combiner thread role */ @@ -183,7 +199,9 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) pending = GET_NEXT_NODE(node_array, pending_cpu); out: pending->wait = false; +#ifdef DEBUG atomic_dec(&next->refcount); +#endif put_cpu(); pr_debug("lockbench: end of critical section!\n"); return prev->ret; @@ -321,32 +339,52 @@ static int t_show(struct seq_file *m, void *v) "\treq = %pF,\n" "\tparams = %p,\n" "\twait = %d, completed = %d,\n" +#ifdef DEBUG "\trefcount = %d,\n" "\tNext (%d, %d)\n" "\tPrev (%d, %d)\n}\n", +#else + "\tNext (%d, %d)\n", +#endif cpu, 0, node[0].req, node[0].params, node[0].wait, node[0].completed, +#ifdef DEBUG node[0].refcount.counter, DECODE_CPU(node[0].next), DECODE_IDX(node[0].next), DECODE_CPU(node[0].prev), - DECODE_IDX(node[0].prev)); + DECODE_IDX(node[0].prev) +#else + DECODE_CPU(node[0].next), + DECODE_IDX(node[0].next) +#endif + ); seq_printf(m, "Node(%d, %d) {\n" "\treq = %pF,\n" "\tparams = %p,\n" "\twait = %d, completed = %d,\n" +#ifdef DEBUG "\trefcount = %d,\n" "\tNext (%d, %d)\n" "\tPrev (%d, %d)\n}\n", +#else + "\tNext (%d, %d)\n", +#endif cpu, 1, node[1].req, node[1].params, node[1].wait, node[1].completed, +#ifdef DEBUG node[1].refcount.counter, DECODE_CPU(node[1].next), DECODE_IDX(node[1].next), DECODE_CPU(node[1].prev), - DECODE_IDX(node[1].prev)); + DECODE_IDX(node[1].prev) +#else + DECODE_CPU(node[1].next), + DECODE_IDX(node[1].next) +#endif + ); } return 0; @@ -428,7 +466,7 @@ int test_thread(void *data) int i; int cpu = get_cpu(); struct lb_info * lb_data = &per_cpu(*((struct lb_info *)data), cpu); - unsigned long prev, cur; + unsigned long prev = 0, cur; if (unlikely(lb_data->monitor)) prev = sched_clock(); @@ -486,7 +524,7 @@ int prepare_tests(test_thread_t test, void *arg, const char *name) { struct task_struct *thread; int cpu; - int max_cpus = 3; + int max_cpus = 12; int nr_cpus = 0; for_each_online_cpu(cpu) { From 798d56eb4f29c6d1358a4e67cd881e68035e80a6 Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Thu, 12 May 2022 03:33:21 +0000 Subject: [PATCH 09/25] Fix print format of dmesg Signed-off-by: Wonhyuk Yang --- lock_module/parse_dmesg.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lock_module/parse_dmesg.py b/lock_module/parse_dmesg.py index 508221b88..2a3976f39 100644 --- a/lock_module/parse_dmesg.py +++ b/lock_module/parse_dmesg.py @@ -1,11 +1,13 @@ import subprocess -cc_lock = {"name":"cc_lock", "nr":0, "time":list()} +cc_lock = {"name":"cc_lock ", "nr":0, "time":list()} spin_lock = {"name":"spin_lock", "nr":0, "time":list()} def print_stat(target): + avg = sum(target["time"]) / target["nr"] print("[{0}] average time: {1}, nr: {2}".format(target["name"], - sum(target["time"]) / target["nr"], target["nr"])) + avg, target["nr"])) + return avg if __name__ == "__main__": lines = subprocess.check_output(["sudo", "dmesg"]).decode() @@ -23,6 +25,8 @@ def print_stat(target): target["nr"] += 1 target["time"].append(int(line[line.rfind("[")+1:-1])) - print_stat(cc_lock) - print_stat(spin_lock) + cc_avg = print_stat(cc_lock) + spin_avg = print_stat(spin_lock) + + print("improvment: {0:.3f}%".format((spin_avg-cc_avg)/spin_avg*100)) From 58a637cce96951d1b690d1429f279a72a76aac2a Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Thu, 12 May 2022 05:24:08 +0000 Subject: [PATCH 10/25] Introduce control file --- lock_module/modul.c | 100 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 95 insertions(+), 5 deletions(-) diff --git a/lock_module/modul.c b/lock_module/modul.c index 2306e6d1e..1c00f15cb 100644 --- a/lock_module/modul.c +++ b/lock_module/modul.c @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +25,13 @@ MODULE_LICENSE("GPL"); #define MAX_COMBINER_OPERATIONS 5 +#define MAX_CPU 16 +#define MAX_DELAY 100 + +static int max_cpus = 11; +static int delay_time = 0; +static int nr_bench = 100000; + /* CPU number and index are encoded in cc_node.next * Now, each cpu has two node and these nodes are * used alternatley. In this way, we can avoid @@ -211,14 +219,23 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) DEFINE_SPINLOCK(dummy_spinlock); atomic_t dummy_lock = ATOMIC_INIT(0); int dummy_counter = 0; +int cache_table[128]; void* dummy_increment(void* params) { + int i; int *counter = (int*)params; if (unlikely(counter == NULL)) { printk("!!!! counter: %p", (counter)); } else { (*counter)++; } + + for (i = 0; i < 128; i++) + cache_table[i]++; + + if (delay_time) + udelay(delay_time); + return params; } @@ -255,7 +272,7 @@ static ssize_t lb_write(struct file *filp, const char __user *ubuf, li->req = dummy_increment; li->params = &dummy_counter; li->lock = (void *)&dummy_lock; - li->counter = NR_BENCH; + li->counter = nr_bench; li->monitor = monitor_thread; monitor_thread = false; } @@ -266,7 +283,7 @@ static ssize_t lb_write(struct file *filp, const char __user *ubuf, li->req = dummy_increment; li->params = &dummy_counter; li->lock = (void *)&dummy_spinlock; - li->counter = NR_BENCH; + li->counter = nr_bench; li->monitor = monitor_thread; monitor_thread = false; } @@ -308,6 +325,57 @@ static ssize_t lb_quit(struct file *filp, const char __user *ubuf, return cnt; } +static ssize_t lb_cpu(struct file *filp, const char __user *ubuf, + size_t cnt, loff_t *ppos) +{ + int ret; + unsigned long val; + + ret = kstrtoul_from_user(ubuf, cnt, 10, &val); + + if (ret) + return ret; + + if (val > 0 && val < 15) + max_cpus = val; + + (*ppos)++; + return cnt; +} + +static ssize_t lb_bench(struct file *filp, const char __user *ubuf, + size_t cnt, loff_t *ppos) +{ + int ret; + unsigned long val; + + ret = kstrtoul_from_user(ubuf, cnt, 10, &val); + + if (ret) + return ret; + + if (val > 0 && val < NR_BENCH) + nr_bench = val; + + (*ppos)++; + return cnt; +} + +static ssize_t lb_delay(struct file *filp, const char __user *ubuf, + size_t cnt, loff_t *ppos) +{ + int ret; + unsigned long val; + ret = kstrtoul_from_user(ubuf, cnt, 10, &val); + + if (ret) + return ret; + if (val >= 0 && val <= MAX_DELAY) + delay_time = val; + (*ppos)++; + return cnt; +} + static void *t_start(struct seq_file *m, loff_t *pos) { return *pos < 1 ? (void *)1 : NULL; @@ -426,10 +494,26 @@ static const struct file_operations lb_trigger_fops = { static const struct file_operations lb_quit_fops = { .open = lb_open, - .read = NULL, .write = lb_quit, .release = lb_release, - .llseek = NULL, +}; + +static const struct file_operations lb_cpu_fops = { + .open = lb_open, + .write = lb_cpu, + .release = lb_release, +}; + +static const struct file_operations lb_bench_fops = { + .open = lb_open, + .write = lb_bench, + .release = lb_release, +}; + +static const struct file_operations lb_delay_fops = { + .open = lb_open, + .write = lb_delay, + .release = lb_release, }; static const struct file_operations lb_status_fops= { @@ -449,6 +533,12 @@ static int lb_debugfs_init(void) lb_debugfs_root, NULL, &lb_trigger_fops); debugfs_create_file("quit", 0200, lb_debugfs_root, NULL, &lb_quit_fops); + debugfs_create_file("cpu", 0200, + lb_debugfs_root, NULL, &lb_cpu_fops); + debugfs_create_file("nr_bench", 0200, + lb_debugfs_root, NULL, &lb_bench_fops); + debugfs_create_file("delay", 0200, + lb_debugfs_root, NULL, &lb_delay_fops); debugfs_create_file("status", 0400, lb_debugfs_root, NULL, &lb_status_fops); @@ -524,7 +614,6 @@ int prepare_tests(test_thread_t test, void *arg, const char *name) { struct task_struct *thread; int cpu; - int max_cpus = 12; int nr_cpus = 0; for_each_online_cpu(cpu) { @@ -581,3 +670,4 @@ static void lock_benchmark_exit(void) module_init(lock_benchmark_init); module_exit(lock_benchmark_exit); + From 3adfe6d6dbf2007a62c9646a66e365bc9eedcf04 Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Mon, 16 May 2022 03:11:40 +0000 Subject: [PATCH 11/25] WIP --- lock_module/modul.c | 158 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 141 insertions(+), 17 deletions(-) diff --git a/lock_module/modul.c b/lock_module/modul.c index 1c00f15cb..b8d6fed98 100644 --- a/lock_module/modul.c +++ b/lock_module/modul.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -15,12 +16,13 @@ #include #include #include +#include MODULE_DESCRIPTION("Simple Lock benchmark module"); MODULE_AUTHOR("Wonhyuk Yang"); MODULE_LICENSE("GPL"); -//#define DEBUG +#define DEBUG #define NR_BENCH (500000) #define MAX_COMBINER_OPERATIONS 5 @@ -28,9 +30,9 @@ MODULE_LICENSE("GPL"); #define MAX_CPU 16 #define MAX_DELAY 100 -static int max_cpus = 11; +static int max_cpus = 2; static int delay_time = 0; -static int nr_bench = 100000; +static int nr_bench = 500000; /* CPU number and index are encoded in cc_node.next * Now, each cpu has two node and these nodes are @@ -57,13 +59,13 @@ struct cc_node { request_t req; void* params; void* ret; - bool wait; - bool completed; int next; #ifdef DEBUG - int prev; + int prev __attribute__((aligned(L1_CACHE_BYTES))); atomic_t refcount; #endif + bool wait __attribute__((aligned(L1_CACHE_BYTES))); + bool completed; }; DEFINE_PER_CPU(struct cc_node, node_array[2]) = { @@ -111,6 +113,12 @@ DEFINE_PER_CPU(struct task_struct *, task_array); */ DEFINE_PER_CPU(int, node_array_idx) = 1; +#ifdef DEBUG +static int node_trace[MAX_CPU*2]; +static int node_trace_idx = 0; +#endif +static int dump_cclock(void); + void* execute_cs(request_t req, void *params, atomic_t *lock) { struct cc_node *prev, *pending; @@ -127,28 +135,37 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) this_cpu = ENCODE_NEXT(this_cpu, this_cpu_idx); next = GET_NEXT_NODE(node_array, this_cpu); - next->req = NULL; - next->params = NULL; - next->ret = NULL; - next->wait = true; - next->completed = false; - next->next = ENCODE_NEXT(NR_CPUS, 0); + WRITE_ONCE(next->req, NULL); + WRITE_ONCE(next->params,NULL); + WRITE_ONCE(next->ret, NULL); + WRITE_ONCE(next->wait, true); + WRITE_ONCE(next->completed, false); + WRITE_ONCE(next->next, ENCODE_NEXT(NR_CPUS, 0)); smp_wmb(); #ifdef DEBUG atomic_inc(&next->refcount); #endif - prev_cpu = atomic_xchg(lock, this_cpu); + do { + prev_cpu = READ_ONCE(lock->counter); + } while(atomic_cmpxchg(lock, prev_cpu, this_cpu) != prev_cpu); WARN(prev_cpu == this_cpu, "lockbench: prev_cpu == this_cpu, Can't be happend!!!"); #ifdef DEBUG next->prev = prev_cpu; #endif - prev = GET_NEXT_NODE(node_array, prev_cpu); /* request and parameter should be set. Then, next should be set */ WRITE_ONCE(prev->req, req); WRITE_ONCE(prev->params, params); smp_wmb(); + if (DECODE_CPU(prev->next)!=NR_CPUS) { + pr_err("prev->next's value is not NR_CPUS!!!" + "prev_cpu (%d,%d) prev->next (%d,%d) this_cpu (%d,%d)", + DECODE_CPU(prev_cpu), DECODE_IDX(prev_cpu), + DECODE_CPU(prev->next), DECODE_IDX(prev->next), + DECODE_CPU(this_cpu), DECODE_IDX(this_cpu)); + dump_cclock(); + } WRITE_ONCE(prev->next, this_cpu); /* Failed to get lock */ @@ -163,6 +180,7 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) while (READ_ONCE(prev->wait)) cpu_relax(); + smp_rmb(); if (READ_ONCE(prev->completed)) { #ifdef DEBUG atomic_dec(&next->refcount); @@ -178,9 +196,23 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) while (counter++ < MAX_COMBINER_OPERATIONS) { pending = GET_NEXT_NODE(node_array, pending_cpu); +#ifdef DEBUG + node_trace[node_trace_idx] = pending_cpu; + node_trace_idx = (node_trace_idx + 1) % (MAX_CPU*2); +#endif pending_cpu = READ_ONCE(pending->next); + /* Keep ordering next -> (req, params)*/ smp_rmb(); + if(!(READ_ONCE(pending->wait)) && (READ_ONCE(pending->completed))) { + pr_err("Target node already done..." + "prev_cpu: (%d,%d), wait:%d, complete: %d " + "this_cpu: (%d,%d)", + DECODE_CPU(pending_cpu), DECODE_IDX(pending_cpu), + READ_ONCE(pending->wait), READ_ONCE(pending->completed), + DECODE_CPU(this_cpu), DECODE_IDX(this_cpu)); + dump_cclock(); + } /* Branch prediction: which case is more profitable? */ if (DECODE_CPU(pending_cpu) == NR_CPUS) @@ -206,7 +238,8 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) pending = GET_NEXT_NODE(node_array, pending_cpu); out: - pending->wait = false; + smp_wmb(); + WRITE_ONCE(pending->wait, false); #ifdef DEBUG atomic_dec(&next->refcount); #endif @@ -555,6 +588,7 @@ int test_thread(void *data) { int i; int cpu = get_cpu(); + unsigned long flags; struct lb_info * lb_data = &per_cpu(*((struct lb_info *)data), cpu); unsigned long prev = 0, cur; @@ -566,7 +600,11 @@ int test_thread(void *data) printk("lockbench: monitor thread %dth [%lu]\n", i, cur-prev); prev = cur; } + local_irq_save(flags); + preempt_disable(); execute_cs(lb_data->req, lb_data->params, lb_data->lock); + local_irq_restore(flags); + preempt_enable(); } if (unlikely(lb_data->monitor)) { @@ -585,6 +623,7 @@ int test_thread2(void *data) int cpu = get_cpu(); struct lb_info * lb_data = &per_cpu(*((struct lb_info *)data), cpu); unsigned long prev = 0, cur = 0; + unsigned long flags; spinlock_t *lock = (spinlock_t *)lb_data->lock; if (unlikely(lb_data->monitor)) @@ -595,9 +634,9 @@ int test_thread2(void *data) printk("lockbench: monitor thread %dth [%lu]\n", i, cur-prev); prev = cur; } - spin_lock(lock); + spin_lock_irqsave(lock, flags); lb_data->req(lb_data->params); - spin_unlock(lock); + spin_unlock_irqrestore(lock, flags); } if (unlikely(lb_data->monitor)) { @@ -641,6 +680,91 @@ int prepare_tests(test_thread_t test, void *arg, const char *name) return 0; } +static int dump_cclock(void) +{ + int cpu, idx; + struct cc_node *node; + struct lb_info *ld; + int tmp; + + pr_err("\n"); + pr_err("dummy_lock: (%d, %d)\n", + DECODE_CPU(dummy_lock.counter), DECODE_IDX(dummy_lock.counter)); + pr_err("last used node:\n"); + for (idx=6; idx>0; idx--){ + tmp = node_trace[(node_trace_idx + (MAX_CPU*2) - (idx)) % (MAX_CPU*2)]; + pr_err("(%d,%d)->", DECODE_CPU(tmp), DECODE_IDX(tmp)); + } + for_each_online_cpu(cpu) { + node = per_cpu(node_array, cpu); + idx = per_cpu(node_array_idx, cpu); + pr_err("Node idx: %d\n", idx); + pr_err("Node(%d, %d) {\n" + "\treq = %pF,\n" + "\tparams = %p,\n" + "\twait = %d, completed = %d,\n" +#ifdef DEBUG + "\trefcount = %d,\n" + "\tNext (%d, %d)\n" + "\tPrev (%d, %d)\n}\n", +#else + "\tNext (%d, %d)\n", +#endif + cpu, 0, + node[0].req, node[0].params, + node[0].wait, node[0].completed, +#ifdef DEBUG + node[0].refcount.counter, + DECODE_CPU(node[0].next), + DECODE_IDX(node[0].next), + DECODE_CPU(node[0].prev), + DECODE_IDX(node[0].prev) +#else + DECODE_CPU(node[0].next), + DECODE_IDX(node[0].next) +#endif + ); + pr_err("Node(%d, %d) {\n" + "\treq = %pF,\n" + "\tparams = %p,\n" + "\twait = %d, completed = %d,\n" +#ifdef DEBUG + "\trefcount = %d,\n" + "\tNext (%d, %d)\n" + "\tPrev (%d, %d)\n}\n", +#else + "\tNext (%d, %d)\n", +#endif + cpu, 1, + node[1].req, node[1].params, + node[1].wait, node[1].completed, +#ifdef DEBUG + node[1].refcount.counter, + DECODE_CPU(node[1].next), + DECODE_IDX(node[1].next), + DECODE_CPU(node[1].prev), + DECODE_IDX(node[1].prev) +#else + DECODE_CPU(node[1].next), + DECODE_IDX(node[1].next) +#endif + ); + ld = &per_cpu(lb_info_array, cpu); + ld->quit = true; + + smp_mb(); + node[0].wait = false; + node[0].completed = true; + node[1].wait = false; + node[1].completed = true; + } + dummy_lock.counter = ENCODE_NEXT(0, 0); + per_cpu(node_array, 0)[0].completed = false; + + BUG(); + return 0; +} + /* module init/exit */ static int lock_benchmark_init(void) { From 515fafae7886561161c21936916f828601d9566f Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Mon, 16 May 2022 07:13:20 +0000 Subject: [PATCH 12/25] Fix? --- lock_module/modul.c | 70 +++++++++++++++++++-------------------------- 1 file changed, 30 insertions(+), 40 deletions(-) diff --git a/lock_module/modul.c b/lock_module/modul.c index b8d6fed98..44cf685a3 100644 --- a/lock_module/modul.c +++ b/lock_module/modul.c @@ -62,35 +62,25 @@ struct cc_node { int next; #ifdef DEBUG int prev __attribute__((aligned(L1_CACHE_BYTES))); - atomic_t refcount; #endif bool wait __attribute__((aligned(L1_CACHE_BYTES))); bool completed; + atomic_t refcount; }; DEFINE_PER_CPU(struct cc_node, node_array[2]) = { { - .req = NULL, - .params = NULL, - .ret = NULL, - .wait = false, - .completed = false, .next = ENCODE_NEXT(NR_CPUS, 1), + .refcount = ATOMIC_INIT(0), #ifdef DEBUG .prev = ENCODE_NEXT(NR_CPUS, 0), - .refcount = ATOMIC_INIT(0), #endif }, { - .req = NULL, - .params = NULL, - .ret = NULL, - .wait = false, - .completed = false, .next = ENCODE_NEXT(NR_CPUS, 0), + .refcount = ATOMIC_INIT(0), #ifdef DEBUG .prev = ENCODE_NEXT(NR_CPUS, 1), - .refcount = ATOMIC_INIT(0), #endif }, }; @@ -127,6 +117,7 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) unsigned int this_cpu = get_cpu(); unsigned int pending_cpu; unsigned int prev_cpu; + unsigned int cur_cpu; request_t pending_req; /* get/update node_arra_idx */ @@ -135,6 +126,11 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) this_cpu = ENCODE_NEXT(this_cpu, this_cpu_idx); next = GET_NEXT_NODE(node_array, this_cpu); + + /* Wait for spinning thread */ + while (READ_ONCE(next->refcount.counter)) + cpu_relax(); + WRITE_ONCE(next->req, NULL); WRITE_ONCE(next->params,NULL); WRITE_ONCE(next->ret, NULL); @@ -143,9 +139,6 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) WRITE_ONCE(next->next, ENCODE_NEXT(NR_CPUS, 0)); smp_wmb(); -#ifdef DEBUG - atomic_inc(&next->refcount); -#endif do { prev_cpu = READ_ONCE(lock->counter); } while(atomic_cmpxchg(lock, prev_cpu, this_cpu) != prev_cpu); @@ -166,6 +159,7 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) DECODE_CPU(this_cpu), DECODE_IDX(this_cpu)); dump_cclock(); } + atomic_inc(&prev->refcount); WRITE_ONCE(prev->next, this_cpu); /* Failed to get lock */ @@ -182,9 +176,7 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) smp_rmb(); if (READ_ONCE(prev->completed)) { -#ifdef DEBUG - atomic_dec(&next->refcount); -#endif + atomic_dec(&prev->refcount); put_cpu(); pr_debug("lockbench: CPU: (%d, %d) end of critical section!\n", DECODE_CPU(this_cpu), DECODE_IDX(this_cpu)); @@ -196,20 +188,19 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) while (counter++ < MAX_COMBINER_OPERATIONS) { pending = GET_NEXT_NODE(node_array, pending_cpu); -#ifdef DEBUG - node_trace[node_trace_idx] = pending_cpu; - node_trace_idx = (node_trace_idx + 1) % (MAX_CPU*2); -#endif + cur_cpu = pending_cpu; pending_cpu = READ_ONCE(pending->next); /* Keep ordering next -> (req, params)*/ smp_rmb(); if(!(READ_ONCE(pending->wait)) && (READ_ONCE(pending->completed))) { pr_err("Target node already done..." - "prev_cpu: (%d,%d), wait:%d, complete: %d " + "cur_cpu: (%d,%d), wait:%d, complete: %d " + "next_cpu: (%d,%d), " "this_cpu: (%d,%d)", - DECODE_CPU(pending_cpu), DECODE_IDX(pending_cpu), + DECODE_CPU(cur_cpu), DECODE_IDX(cur_cpu), READ_ONCE(pending->wait), READ_ONCE(pending->completed), + DECODE_CPU(pending_cpu), DECODE_IDX(pending_cpu), DECODE_CPU(this_cpu), DECODE_IDX(this_cpu)); dump_cclock(); } @@ -218,6 +209,10 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) if (DECODE_CPU(pending_cpu) == NR_CPUS) goto out; +#ifdef DEBUG + node_trace[node_trace_idx] = pending_cpu; + node_trace_idx = (node_trace_idx + 1) % (MAX_CPU*2); +#endif pr_debug("lockbench: CPU: (%d, %d), next_cpu: (%d, %d), request: %pF\n", DECODE_CPU(pending_cpu), DECODE_IDX(pending_cpu), DECODE_CPU(pending->next), DECODE_IDX(pending->next), @@ -230,6 +225,10 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) WRITE_ONCE(pending->completed, true); smp_wmb(); WRITE_ONCE(pending->wait, false); + + /* request is done */ + if (prev == pending) + atomic_dec(&prev->refcount); } /* Pass tho combiner thread role */ WARN(DECODE_CPU(pending_cpu) == NR_CPUS, "lockbench: DECODE_CPU(pedning_cpu) == NR_CPUS..."); @@ -240,9 +239,6 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) out: smp_wmb(); WRITE_ONCE(pending->wait, false); -#ifdef DEBUG - atomic_dec(&next->refcount); -#endif put_cpu(); pr_debug("lockbench: end of critical section!\n"); return prev->ret; @@ -440,50 +436,44 @@ static int t_show(struct seq_file *m, void *v) "\treq = %pF,\n" "\tparams = %p,\n" "\twait = %d, completed = %d,\n" -#ifdef DEBUG "\trefcount = %d,\n" "\tNext (%d, %d)\n" +#ifdef DEBUG "\tPrev (%d, %d)\n}\n", #else - "\tNext (%d, %d)\n", + , #endif cpu, 0, node[0].req, node[0].params, node[0].wait, node[0].completed, -#ifdef DEBUG node[0].refcount.counter, DECODE_CPU(node[0].next), DECODE_IDX(node[0].next), +#ifdef DEBUG DECODE_CPU(node[0].prev), DECODE_IDX(node[0].prev) -#else - DECODE_CPU(node[0].next), - DECODE_IDX(node[0].next) #endif ); seq_printf(m, "Node(%d, %d) {\n" "\treq = %pF,\n" "\tparams = %p,\n" "\twait = %d, completed = %d,\n" -#ifdef DEBUG "\trefcount = %d,\n" "\tNext (%d, %d)\n" +#ifdef DEBUG "\tPrev (%d, %d)\n}\n", #else - "\tNext (%d, %d)\n", + , #endif cpu, 1, node[1].req, node[1].params, node[1].wait, node[1].completed, -#ifdef DEBUG node[1].refcount.counter, DECODE_CPU(node[1].next), DECODE_IDX(node[1].next), +#ifdef DEBUG DECODE_CPU(node[1].prev), DECODE_IDX(node[1].prev) -#else - DECODE_CPU(node[1].next), - DECODE_IDX(node[1].next) #endif ); } From 3b4491f293ef61149ba9f96e4b17aba80f05dc32 Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Mon, 16 May 2022 07:32:56 +0000 Subject: [PATCH 13/25] Minor --- lock_module/modul.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lock_module/modul.c b/lock_module/modul.c index 44cf685a3..7df390846 100644 --- a/lock_module/modul.c +++ b/lock_module/modul.c @@ -22,7 +22,7 @@ MODULE_DESCRIPTION("Simple Lock benchmark module"); MODULE_AUTHOR("Wonhyuk Yang"); MODULE_LICENSE("GPL"); -#define DEBUG +//#define DEBUG #define NR_BENCH (500000) #define MAX_COMBINER_OPERATIONS 5 @@ -448,8 +448,9 @@ static int t_show(struct seq_file *m, void *v) node[0].wait, node[0].completed, node[0].refcount.counter, DECODE_CPU(node[0].next), - DECODE_IDX(node[0].next), + DECODE_IDX(node[0].next) #ifdef DEBUG + , DECODE_CPU(node[0].prev), DECODE_IDX(node[0].prev) #endif @@ -470,8 +471,9 @@ static int t_show(struct seq_file *m, void *v) node[1].wait, node[1].completed, node[1].refcount.counter, DECODE_CPU(node[1].next), - DECODE_IDX(node[1].next), + DECODE_IDX(node[1].next) #ifdef DEBUG + , DECODE_CPU(node[1].prev), DECODE_IDX(node[1].prev) #endif @@ -675,16 +677,20 @@ static int dump_cclock(void) int cpu, idx; struct cc_node *node; struct lb_info *ld; +#ifdef DEBUG int tmp; +#endif pr_err("\n"); pr_err("dummy_lock: (%d, %d)\n", DECODE_CPU(dummy_lock.counter), DECODE_IDX(dummy_lock.counter)); +#ifdef DEBUG pr_err("last used node:\n"); for (idx=6; idx>0; idx--){ tmp = node_trace[(node_trace_idx + (MAX_CPU*2) - (idx)) % (MAX_CPU*2)]; pr_err("(%d,%d)->", DECODE_CPU(tmp), DECODE_IDX(tmp)); } +#endif for_each_online_cpu(cpu) { node = per_cpu(node_array, cpu); idx = per_cpu(node_array_idx, cpu); From d2c42b619453a7f1a96195ff114311551f0a0eee Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Thu, 19 May 2022 04:11:03 +0000 Subject: [PATCH 14/25] Fix2? --- lock_module/modul.c | 156 +++++++++++++++++++++++++------------------- 1 file changed, 88 insertions(+), 68 deletions(-) diff --git a/lock_module/modul.c b/lock_module/modul.c index 7df390846..5fff83e81 100644 --- a/lock_module/modul.c +++ b/lock_module/modul.c @@ -23,14 +23,15 @@ MODULE_AUTHOR("Wonhyuk Yang"); MODULE_LICENSE("GPL"); //#define DEBUG -#define NR_BENCH (500000) +//#define BLOCK_IRQ +#define NR_BENCH (5000000) -#define MAX_COMBINER_OPERATIONS 5 +#define MAX_COMBINER_OPERATIONS 3 #define MAX_CPU 16 #define MAX_DELAY 100 -static int max_cpus = 2; +static int max_cpus = 6; static int delay_time = 0; static int nr_bench = 500000; @@ -39,8 +40,9 @@ static int nr_bench = 500000; * used alternatley. In this way, we can avoid * node overwrite problem. */ -#define INDEX_SHIFT (1) -#define INDEX_MASK ((1 << INDEX_SHIFT) - 1) +#define INDEX_SHIFT (2) +#define INDEX_SIZE (1<refcount.counter)) - cpu_relax(); + while (READ_ONCE(next->refcount.counter)); WRITE_ONCE(next->req, NULL); WRITE_ONCE(next->params,NULL); @@ -142,8 +162,8 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) do { prev_cpu = READ_ONCE(lock->counter); } while(atomic_cmpxchg(lock, prev_cpu, this_cpu) != prev_cpu); - WARN(prev_cpu == this_cpu, "lockbench: prev_cpu == this_cpu, Can't be happend!!!"); #ifdef DEBUG + WARN(prev_cpu == this_cpu, "lockbench: prev_cpu == this_cpu, Can't be happend!!!"); next->prev = prev_cpu; #endif prev = GET_NEXT_NODE(node_array, prev_cpu); @@ -151,6 +171,7 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) WRITE_ONCE(prev->req, req); WRITE_ONCE(prev->params, params); smp_wmb(); +#ifdef DEBUG if (DECODE_CPU(prev->next)!=NR_CPUS) { pr_err("prev->next's value is not NR_CPUS!!!" "prev_cpu (%d,%d) prev->next (%d,%d) this_cpu (%d,%d)", @@ -159,6 +180,7 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) DECODE_CPU(this_cpu), DECODE_IDX(this_cpu)); dump_cclock(); } +#endif atomic_inc(&prev->refcount); WRITE_ONCE(prev->next, this_cpu); @@ -176,7 +198,7 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) smp_rmb(); if (READ_ONCE(prev->completed)) { - atomic_dec(&prev->refcount); + WRITE_ONCE(prev->refcount.counter, prev->refcount.counter-1); put_cpu(); pr_debug("lockbench: CPU: (%d, %d) end of critical section!\n", DECODE_CPU(this_cpu), DECODE_IDX(this_cpu)); @@ -186,13 +208,14 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) /* Success to get lock */ pending_cpu = prev_cpu; - while (counter++ < MAX_COMBINER_OPERATIONS) { + while (counter++ < MAX_COMBINER_OPERATIONS*max_cpus) { pending = GET_NEXT_NODE(node_array, pending_cpu); cur_cpu = pending_cpu; pending_cpu = READ_ONCE(pending->next); /* Keep ordering next -> (req, params)*/ smp_rmb(); +#ifdef DEBUG if(!(READ_ONCE(pending->wait)) && (READ_ONCE(pending->completed))) { pr_err("Target node already done..." "cur_cpu: (%d,%d), wait:%d, complete: %d " @@ -204,6 +227,7 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) DECODE_CPU(this_cpu), DECODE_IDX(this_cpu)); dump_cclock(); } +#endif /* Branch prediction: which case is more profitable? */ if (DECODE_CPU(pending_cpu) == NR_CPUS) @@ -220,18 +244,18 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) pending_req = READ_ONCE(pending->req); /* Preserve store order completed -> wait -> next */ +#ifdef DEBUG WARN(pending_req == NULL, "lockbench: pending->req == NULL..."); +#endif WRITE_ONCE(pending->ret, pending_req(pending->params)); WRITE_ONCE(pending->completed, true); smp_wmb(); WRITE_ONCE(pending->wait, false); - - /* request is done */ - if (prev == pending) - atomic_dec(&prev->refcount); } /* Pass tho combiner thread role */ +#ifdef DEBUG WARN(DECODE_CPU(pending_cpu) == NR_CPUS, "lockbench: DECODE_CPU(pedning_cpu) == NR_CPUS..."); +#endif pr_debug("lockbench: pass the combiner role to CPU: (%d, %d)\n", DECODE_CPU(pending_cpu), DECODE_IDX(pending_cpu)); @@ -239,6 +263,7 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) out: smp_wmb(); WRITE_ONCE(pending->wait, false); + WRITE_ONCE(prev->refcount.counter, prev->refcount.counter-1); put_cpu(); pr_debug("lockbench: end of critical section!\n"); return prev->ret; @@ -248,23 +273,18 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) DEFINE_SPINLOCK(dummy_spinlock); atomic_t dummy_lock = ATOMIC_INIT(0); int dummy_counter = 0; -int cache_table[128]; +int cache_table[1024*4+1]; void* dummy_increment(void* params) { int i; int *counter = (int*)params; - if (unlikely(counter == NULL)) { - printk("!!!! counter: %p", (counter)); - } else { - (*counter)++; - } + (*counter)++; - for (i = 0; i < 128; i++) - cache_table[i]++; + for (i = 0; i < 4; i++) + cache_table[i*1024]++; if (delay_time) udelay(delay_time); - return params; } @@ -328,6 +348,7 @@ static ssize_t lb_quit(struct file *filp, const char __user *ubuf, { unsigned long val; int ret, cpu; + int j; struct lb_info *ld; struct cc_node *node; ret = kstrtoul_from_user(ubuf, cnt, 10, &val); @@ -343,10 +364,10 @@ static ssize_t lb_quit(struct file *filp, const char __user *ubuf, smp_mb(); node = per_cpu(node_array, cpu); - node[0].wait = false; - node[0].completed = true; - node[1].wait = false; - node[1].completed = true; + for (j=0; j monitor thread %dth [%lu]\n", i, cur-prev); prev = cur; } +#ifdef BLOCK_IRQ local_irq_save(flags); - preempt_disable(); +#endif execute_cs(lb_data->req, lb_data->params, lb_data->lock); +#ifdef BLOCK_IRQ local_irq_restore(flags); - preempt_enable(); +#endif } if (unlikely(lb_data->monitor)) { @@ -615,7 +640,9 @@ int test_thread2(void *data) int cpu = get_cpu(); struct lb_info * lb_data = &per_cpu(*((struct lb_info *)data), cpu); unsigned long prev = 0, cur = 0; +#ifdef BLOCK_IRQ unsigned long flags; +#endif spinlock_t *lock = (spinlock_t *)lb_data->lock; if (unlikely(lb_data->monitor)) @@ -626,9 +653,17 @@ int test_thread2(void *data) printk("lockbench: monitor thread %dth [%lu]\n", i, cur-prev); prev = cur; } +#ifdef BLOCK_IRQ spin_lock_irqsave(lock, flags); +#else + spin_lock(lock); +#endif lb_data->req(lb_data->params); +#ifdef BLOCK_IRQ spin_unlock_irqrestore(lock, flags); +#else + spin_unlock(lock); +#endif } if (unlikely(lb_data->monitor)) { @@ -671,26 +706,23 @@ int prepare_tests(test_thread_t test, void *arg, const char *name) } return 0; } - +#ifdef DEBUG static int dump_cclock(void) { int cpu, idx; + int j; struct cc_node *node; struct lb_info *ld; -#ifdef DEBUG int tmp; -#endif pr_err("\n"); pr_err("dummy_lock: (%d, %d)\n", DECODE_CPU(dummy_lock.counter), DECODE_IDX(dummy_lock.counter)); -#ifdef DEBUG pr_err("last used node:\n"); for (idx=6; idx>0; idx--){ tmp = node_trace[(node_trace_idx + (MAX_CPU*2) - (idx)) % (MAX_CPU*2)]; pr_err("(%d,%d)->", DECODE_CPU(tmp), DECODE_IDX(tmp)); } -#endif for_each_online_cpu(cpu) { node = per_cpu(node_array, cpu); idx = per_cpu(node_array_idx, cpu); @@ -699,60 +731,42 @@ static int dump_cclock(void) "\treq = %pF,\n" "\tparams = %p,\n" "\twait = %d, completed = %d,\n" -#ifdef DEBUG "\trefcount = %d,\n" "\tNext (%d, %d)\n" "\tPrev (%d, %d)\n}\n", -#else - "\tNext (%d, %d)\n", -#endif cpu, 0, node[0].req, node[0].params, node[0].wait, node[0].completed, -#ifdef DEBUG node[0].refcount.counter, DECODE_CPU(node[0].next), DECODE_IDX(node[0].next), DECODE_CPU(node[0].prev), DECODE_IDX(node[0].prev) -#else - DECODE_CPU(node[0].next), - DECODE_IDX(node[0].next) -#endif ); pr_err("Node(%d, %d) {\n" "\treq = %pF,\n" "\tparams = %p,\n" "\twait = %d, completed = %d,\n" -#ifdef DEBUG "\trefcount = %d,\n" "\tNext (%d, %d)\n" "\tPrev (%d, %d)\n}\n", -#else - "\tNext (%d, %d)\n", -#endif cpu, 1, node[1].req, node[1].params, node[1].wait, node[1].completed, -#ifdef DEBUG node[1].refcount.counter, DECODE_CPU(node[1].next), DECODE_IDX(node[1].next), DECODE_CPU(node[1].prev), DECODE_IDX(node[1].prev) -#else - DECODE_CPU(node[1].next), - DECODE_IDX(node[1].next) -#endif ); ld = &per_cpu(lb_info_array, cpu); ld->quit = true; smp_mb(); - node[0].wait = false; - node[0].completed = true; - node[1].wait = false; - node[1].completed = true; + for (j=0; j Date: Thu, 19 May 2022 04:11:28 +0000 Subject: [PATCH 15/25] Fix dmesg format --- lock_module/parse_dmesg.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lock_module/parse_dmesg.py b/lock_module/parse_dmesg.py index 2a3976f39..c541f3ef9 100644 --- a/lock_module/parse_dmesg.py +++ b/lock_module/parse_dmesg.py @@ -5,7 +5,7 @@ def print_stat(target): avg = sum(target["time"]) / target["nr"] - print("[{0}] average time: {1}, nr: {2}".format(target["name"], + print("[{0}] average time: {1:15.5f}, nr: {2}".format(target["name"], avg, target["nr"])) return avg @@ -28,5 +28,5 @@ def print_stat(target): cc_avg = print_stat(cc_lock) spin_avg = print_stat(spin_lock) - print("improvment: {0:.3f}%".format((spin_avg-cc_avg)/spin_avg*100)) + print("improvment: {0:10.5f}%".format((spin_avg-cc_avg)/spin_avg*100)) From b2f92a797803f35771ee38445b6e30ea9f72375b Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Thu, 19 May 2022 05:20:53 +0000 Subject: [PATCH 16/25] =?UTF-8?q?=EC=8B=A4=ED=97=98=201?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lock_module/modul.c | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/lock_module/modul.c b/lock_module/modul.c index 5fff83e81..7fe2ce5b6 100644 --- a/lock_module/modul.c +++ b/lock_module/modul.c @@ -26,12 +26,12 @@ MODULE_LICENSE("GPL"); //#define BLOCK_IRQ #define NR_BENCH (5000000) -#define MAX_COMBINER_OPERATIONS 3 +#define MAX_COMBINER_OPERATIONS 1 #define MAX_CPU 16 #define MAX_DELAY 100 -static int max_cpus = 6; +static int max_cpus = 14; static int delay_time = 0; static int nr_bench = 500000; @@ -51,6 +51,12 @@ static int nr_bench = 500000; #define GET_NEXT_NODE(x, y) (per_cpu(x, DECODE_CPU(y)) + DECODE_IDX(y)) #define GVM_CACHE_BYTES (1<<12) +#define arch_lock_xchg(ptr, v) __xchg_op((ptr), (v), xchg, "lock; ") + +static inline int atomic_lock_xchg(atomic_t *v, int new) +{ + return arch_lock_xchg(&v->counter, new); +} typedef void* (*request_t)(void *); typedef int (*test_thread_t)(void *); @@ -63,11 +69,11 @@ struct cc_node { void* params; void* ret; int next; - atomic_t refcount; #ifdef DEBUG int prev __attribute__((aligned(L1_CACHE_BYTES))); #endif - bool wait __attribute__((aligned(L1_CACHE_BYTES))); + atomic_t refcount __attribute__((aligned(L1_CACHE_BYTES))); + bool wait; bool completed; }; @@ -151,17 +157,15 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) /* Wait for spinning thread */ while (READ_ONCE(next->refcount.counter)); - WRITE_ONCE(next->req, NULL); - WRITE_ONCE(next->params,NULL); - WRITE_ONCE(next->ret, NULL); - WRITE_ONCE(next->wait, true); - WRITE_ONCE(next->completed, false); - WRITE_ONCE(next->next, ENCODE_NEXT(NR_CPUS, 0)); + next->req = NULL; + next->params = NULL; + next->ret = NULL; + next->wait = true; + next->completed = false; + next->next = ENCODE_NEXT(NR_CPUS, 0); smp_wmb(); - do { - prev_cpu = READ_ONCE(lock->counter); - } while(atomic_cmpxchg(lock, prev_cpu, this_cpu) != prev_cpu); + prev_cpu = atomic_lock_xchg(lock, this_cpu); #ifdef DEBUG WARN(prev_cpu == this_cpu, "lockbench: prev_cpu == this_cpu, Can't be happend!!!"); next->prev = prev_cpu; @@ -193,7 +197,7 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) next->wait, next->completed); pr_debug("lockbench: Spinning start!\n"); - while (READ_ONCE(prev->wait)) + while (likely(READ_ONCE(prev->wait))) cpu_relax(); smp_rmb(); @@ -262,8 +266,8 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) pending = GET_NEXT_NODE(node_array, pending_cpu); out: smp_wmb(); - WRITE_ONCE(pending->wait, false); WRITE_ONCE(prev->refcount.counter, prev->refcount.counter-1); + WRITE_ONCE(pending->wait, false); put_cpu(); pr_debug("lockbench: end of critical section!\n"); return prev->ret; @@ -271,8 +275,8 @@ void* execute_cs(request_t req, void *params, atomic_t *lock) /* Dummy workload */ DEFINE_SPINLOCK(dummy_spinlock); -atomic_t dummy_lock = ATOMIC_INIT(0); -int dummy_counter = 0; +atomic_t dummy_lock __attribute__((aligned(L1_CACHE_BYTES))) = ATOMIC_INIT(0); +int dummy_counter __attribute__((aligned(L1_CACHE_BYTES))) = 0; int cache_table[1024*4+1]; void* dummy_increment(void* params) { From 9eeb60950722bb44c4d3c7c79deb559d73fc6a7d Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Fri, 20 May 2022 11:58:33 +0900 Subject: [PATCH 17/25] Change cycle --- lock_module/parse_dmesg.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lock_module/parse_dmesg.py b/lock_module/parse_dmesg.py index c541f3ef9..5ff6b4728 100644 --- a/lock_module/parse_dmesg.py +++ b/lock_module/parse_dmesg.py @@ -1,12 +1,14 @@ import subprocess +import psutil +clk = 1 / psutil.cpu_freq().current cc_lock = {"name":"cc_lock ", "nr":0, "time":list()} spin_lock = {"name":"spin_lock", "nr":0, "time":list()} def print_stat(target): avg = sum(target["time"]) / target["nr"] - print("[{0}] average time: {1:15.5f}, nr: {2}".format(target["name"], - avg, target["nr"])) + print("[{0}] average clock: {1:15.5f}, nr: {2}".format(target["name"], + avg/clk, target["nr"])) return avg if __name__ == "__main__": @@ -22,7 +24,7 @@ def print_stat(target): if target is None: continue - target["nr"] += 1 + target["nr"] += 1000 target["time"].append(int(line[line.rfind("[")+1:-1])) cc_avg = print_stat(cc_lock) From f9e1a8f3b9e3e097a3d46e8c50e85873693c4dd6 Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Mon, 23 May 2022 05:16:58 +0000 Subject: [PATCH 18/25] Call printk after measurement done --- lock_module/modul.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/lock_module/modul.c b/lock_module/modul.c index 7fe2ce5b6..3f3e06eae 100644 --- a/lock_module/modul.c +++ b/lock_module/modul.c @@ -24,7 +24,8 @@ MODULE_LICENSE("GPL"); //#define DEBUG //#define BLOCK_IRQ -#define NR_BENCH (5000000) +#define NR_BENCH (5000000) +#define NR_SAMPLE 1000 #define MAX_COMBINER_OPERATIONS 1 @@ -35,6 +36,7 @@ static int max_cpus = 14; static int delay_time = 0; static int nr_bench = 500000; +static unsigned long perf_result[NR_BENCH/NR_SAMPLE]; /* CPU number and index are encoded in cc_node.next * Now, each cpu has two node and these nodes are * used alternatley. In this way, we can avoid @@ -283,8 +285,8 @@ void* dummy_increment(void* params) int i; int *counter = (int*)params; (*counter)++; - - for (i = 0; i < 4; i++) + return NULL; + for (i = 0; i < 0; i++) cache_table[i*1024]++; if (delay_time) @@ -342,7 +344,6 @@ static ssize_t lb_write(struct file *filp, const char __user *ubuf, } prepare_tests(test_thread2, (void *)&lb_info_array, "spinlockbench"); } - (*ppos)++; return cnt; } @@ -613,10 +614,10 @@ int test_thread(void *data) if (unlikely(lb_data->monitor)) prev = sched_clock(); - for (i=0; icounter && !READ_ONCE(lb_data->quit); i++) { - if (unlikely(lb_data->monitor && i && ((i % 1000)==0))) { + for (i=0; icounter && !READ_ONCE(lb_data->quit);i++) { + if (unlikely(lb_data->monitor && i && (i%1000==0))) { cur = sched_clock(); - printk("lockbench: monitor thread %dth [%lu]\n", i, cur-prev); + perf_result[(i/NR_SAMPLE)-1] = cur-prev; prev = cur; } #ifdef BLOCK_IRQ @@ -630,7 +631,9 @@ int test_thread(void *data) if (unlikely(lb_data->monitor)) { cur = sched_clock(); - printk("lockbench: monitor thread %dth [%lu]\n", i, cur-prev); + perf_result[(i+NR_SAMPLE-1)/NR_SAMPLE-1] = cur-prev; + for (i=0;icounter;i+=NR_SAMPLE) + printk("lockbench: monitor thread %dth [%lu]\n", i, perf_result[i/NR_SAMPLE]); } per_cpu(task_array, cpu) = NULL; @@ -652,9 +655,9 @@ int test_thread2(void *data) if (unlikely(lb_data->monitor)) prev = sched_clock(); for (i=0; icounter && !READ_ONCE(lb_data->quit); i++) { - if (unlikely(lb_data->monitor && i && ((i % 1000)==0))) { + if (unlikely(lb_data->monitor && i && (i%1000==0))) { cur = sched_clock(); - printk("lockbench: monitor thread %dth [%lu]\n", i, cur-prev); + perf_result[(i/NR_SAMPLE)-1] = cur-prev; prev = cur; } #ifdef BLOCK_IRQ @@ -672,7 +675,9 @@ int test_thread2(void *data) if (unlikely(lb_data->monitor)) { cur = sched_clock(); - printk("lockbench: monitor thread %dth [%lu]\n", i, cur-prev); + perf_result[(i+NR_SAMPLE-1)/NR_SAMPLE-1] = cur-prev; + for (i=0;icounter;i+=NR_SAMPLE) + printk("lockbench: monitor thread %dth [%lu]\n", i, perf_result[i/NR_SAMPLE]); } per_cpu(task_array, cpu) = NULL; From 803f725769ecdab96c0e363123850bb1974f7bce Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Mon, 23 May 2022 05:17:38 +0000 Subject: [PATCH 19/25] Add utility script --- lock_module/bench.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100755 lock_module/bench.sh diff --git a/lock_module/bench.sh b/lock_module/bench.sh new file mode 100755 index 000000000..f460cd955 --- /dev/null +++ b/lock_module/bench.sh @@ -0,0 +1,12 @@ +LOCK_PATH=/sys/kernel/debug/lock_benchmark +CPU=$1 + +echo $CPU > $LOCK_PATH/cpu +dmesg -c > /dev/null +echo 1 > $LOCK_PATH/trigger +sleep 8 +echo 2 > $LOCK_PATH/trigger +sleep 8 +echo "CPU:$CPU" >> result +python3 parse_dmesg.py >> result + From 11d832fa7ebaf96c9f3d2d0cd27f4fa4967177b5 Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Fri, 27 May 2022 02:21:30 +0000 Subject: [PATCH 20/25] Add variance --- lock_module/parse_dmesg.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lock_module/parse_dmesg.py b/lock_module/parse_dmesg.py index 5ff6b4728..889cce7ee 100644 --- a/lock_module/parse_dmesg.py +++ b/lock_module/parse_dmesg.py @@ -1,14 +1,17 @@ import subprocess import psutil +import math -clk = 1 / psutil.cpu_freq().current +clk = 1 / (psutil.cpu_freq().current / 1000) cc_lock = {"name":"cc_lock ", "nr":0, "time":list()} spin_lock = {"name":"spin_lock", "nr":0, "time":list()} def print_stat(target): - avg = sum(target["time"]) / target["nr"] - print("[{0}] average clock: {1:15.5f}, nr: {2}".format(target["name"], - avg/clk, target["nr"])) + avg = sum(target["time"])*1000 / target["nr"] + print(target["time"]) + var = sum([(avg-i)**2 for i in target["time"]]) / len(target["time"]) + print("[{0}] average clock: {1:15.5f} co: {2:15.5f}, nr: {3}".format(target["name"], + avg, math.sqrt(var), target["nr"])) return avg if __name__ == "__main__": @@ -25,7 +28,7 @@ def print_stat(target): continue target["nr"] += 1000 - target["time"].append(int(line[line.rfind("[")+1:-1])) + target["time"].append(int(line[line.rfind("[")+1:-1])/(clk*1000)) cc_avg = print_stat(cc_lock) spin_avg = print_stat(spin_lock) From 33ad9ee4bcd2152bee705472e3721f96cfdee717 Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Fri, 27 May 2022 02:29:05 +0000 Subject: [PATCH 21/25] Fix --- lock_module/parse_dmesg.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lock_module/parse_dmesg.py b/lock_module/parse_dmesg.py index 889cce7ee..70311a36f 100644 --- a/lock_module/parse_dmesg.py +++ b/lock_module/parse_dmesg.py @@ -8,9 +8,8 @@ def print_stat(target): avg = sum(target["time"])*1000 / target["nr"] - print(target["time"]) var = sum([(avg-i)**2 for i in target["time"]]) / len(target["time"]) - print("[{0}] average clock: {1:15.5f} co: {2:15.5f}, nr: {3}".format(target["name"], + print("[{0}] average clock: {1:15.5f} std: {2:15.5f}, nr: {3}".format(target["name"], avg, math.sqrt(var), target["nr"])) return avg @@ -28,7 +27,7 @@ def print_stat(target): continue target["nr"] += 1000 - target["time"].append(int(line[line.rfind("[")+1:-1])/(clk*1000)) + target["time"].append(int(line[line.rfind("[")+1:-1])/(clk)) cc_avg = print_stat(cc_lock) spin_avg = print_stat(spin_lock) From 4d1f6a0774216e6879235d4269ddca92a5fa11c8 Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Tue, 7 Jun 2022 02:07:14 +0000 Subject: [PATCH 22/25] Add histogram feature --- lock_module/parse_dmesg.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/lock_module/parse_dmesg.py b/lock_module/parse_dmesg.py index 70311a36f..68a04765c 100644 --- a/lock_module/parse_dmesg.py +++ b/lock_module/parse_dmesg.py @@ -1,14 +1,21 @@ import subprocess import psutil +import sys +import os import math +import matplotlib.pyplot as plt +import numpy as np -clk = 1 / (psutil.cpu_freq().current / 1000) -cc_lock = {"name":"cc_lock ", "nr":0, "time":list()} -spin_lock = {"name":"spin_lock", "nr":0, "time":list()} +clk = 1 / (psutil.cpu_freq().min / (1000*1000)) +print(clk) +cc_lock = {"name":"cc_lock ", "nr":0, "time":list(), "color":"r"} +spin_lock = {"name":"spin_lock", "nr":0, "time":list(), "color":"b"} def print_stat(target): avg = sum(target["time"])*1000 / target["nr"] var = sum([(avg-i)**2 for i in target["time"]]) / len(target["time"]) + d = np.array(target["time"]) + plt.hist(d, bins=200, alpha=0.7, histtype='step', color=target["color"], label=target["name"]) print("[{0}] average clock: {1:15.5f} std: {2:15.5f}, nr: {3}".format(target["name"], avg, math.sqrt(var), target["nr"])) return avg @@ -31,6 +38,14 @@ def print_stat(target): cc_avg = print_stat(cc_lock) spin_avg = print_stat(spin_lock) + plt.xlim(0,1000*10) + plt.xticks(np.arange(0, 1000*10, 1000)) + plt.minorticks_on() + plt.ylabel("instuction") + plt.xlabel("clock") + plt.legend(loc='upper right') + plt.title("CPU: {0}".format(sys.argv[1])) + plt.savefig("{0}.png".format(sys.argv[1])) print("improvment: {0:10.5f}%".format((spin_avg-cc_avg)/spin_avg*100)) From 27c9db06ddecaa00ee5d488a75353bc2772fca59 Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Tue, 7 Jun 2022 02:08:43 +0000 Subject: [PATCH 23/25] Add argument --- lock_module/bench.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lock_module/bench.sh b/lock_module/bench.sh index f460cd955..91978e06b 100755 --- a/lock_module/bench.sh +++ b/lock_module/bench.sh @@ -8,5 +8,5 @@ sleep 8 echo 2 > $LOCK_PATH/trigger sleep 8 echo "CPU:$CPU" >> result -python3 parse_dmesg.py >> result +python3 parse_dmesg.py $1 >> result From 6903c8ce873539f8c28e96e859c5a4631fb59ba2 Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Thu, 9 Jun 2022 00:50:33 +0000 Subject: [PATCH 24/25] Introuduce more fair start --- lock_module/modul.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/lock_module/modul.c b/lock_module/modul.c index 3f3e06eae..83dba017b 100644 --- a/lock_module/modul.c +++ b/lock_module/modul.c @@ -34,7 +34,8 @@ MODULE_LICENSE("GPL"); static int max_cpus = 14; static int delay_time = 0; -static int nr_bench = 500000; +static int nr_bench = 500000<<1; +static int thread_switch = 0; static unsigned long perf_result[NR_BENCH/NR_SAMPLE]; /* CPU number and index are encoded in cc_node.next @@ -317,10 +318,10 @@ static ssize_t lb_write(struct file *filp, const char __user *ubuf, bool monitor_thread = true; ret = kstrtoul_from_user(ubuf, cnt, 10, &val); - if (ret) return ret; + WRITE_ONCE(thread_switch, 0); if (val == 1) { for_each_online_cpu(cpu) { li = &per_cpu(lb_info_array, cpu); @@ -345,6 +346,8 @@ static ssize_t lb_write(struct file *filp, const char __user *ubuf, prepare_tests(test_thread2, (void *)&lb_info_array, "spinlockbench"); } (*ppos)++; + udelay(1000); + WRITE_ONCE(thread_switch, 1); return cnt; } @@ -609,8 +612,9 @@ int test_thread(void *data) #ifdef BLOCK_IRQ unsigned long flags; #endif - struct lb_info * lb_data = &per_cpu(*((struct lb_info *)data), cpu); + struct lb_info *lb_data = &per_cpu(*((struct lb_info *)data), cpu); unsigned long prev = 0, cur; + while(!READ_ONCE(thread_switch)); if (unlikely(lb_data->monitor)) prev = sched_clock(); @@ -651,6 +655,7 @@ int test_thread2(void *data) unsigned long flags; #endif spinlock_t *lock = (spinlock_t *)lb_data->lock; + while(!READ_ONCE(thread_switch)); if (unlikely(lb_data->monitor)) prev = sched_clock(); From cde0b90acb232b46dc04f08a7984edae72d51d6c Mon Sep 17 00:00:00 2001 From: Wonhyuk Yang Date: Thu, 23 Jun 2022 09:45:23 +0000 Subject: [PATCH 25/25] Add benchmark ready debubfs for polling Signed-off-by: Wonhyuk Yang --- lock_module/bench.sh | 16 ++++++++++++-- lock_module/modul.c | 52 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/lock_module/bench.sh b/lock_module/bench.sh index 91978e06b..036abc444 100755 --- a/lock_module/bench.sh +++ b/lock_module/bench.sh @@ -4,9 +4,21 @@ CPU=$1 echo $CPU > $LOCK_PATH/cpu dmesg -c > /dev/null echo 1 > $LOCK_PATH/trigger -sleep 8 +READY=$(cat $LOCK_PATH/ready) +while [ "$READY" -eq 0 ] +do + sleep 1 + READY=$(cat $LOCK_PATH/ready) +done + echo 2 > $LOCK_PATH/trigger -sleep 8 +READY=$(cat $LOCK_PATH/ready) +while [ "$READY" -eq 0 ] +do + sleep 1 + READY=$(cat $LOCK_PATH/ready) +done + echo "CPU:$CPU" >> result python3 parse_dmesg.py $1 >> result diff --git a/lock_module/modul.c b/lock_module/modul.c index 83dba017b..c8b29bb23 100644 --- a/lock_module/modul.c +++ b/lock_module/modul.c @@ -538,6 +538,48 @@ static int lb_status_release(struct inode *inode, struct file *file) return seq_release(inode, file); } +static int r_show(struct seq_file *m, void *v) +{ + int cpu; + struct task_struct *thread; + + for_each_online_cpu(cpu) { + thread = per_cpu(task_array, cpu); + if (thread) { + seq_printf(m, "0"); + return 0; + } + } + seq_printf(m, "1"); + return 0; +} + +static const struct seq_operations show_ready_seq_ops= { + .start = t_start, + .next = t_next, + .stop = t_stop, + .show = r_show, +}; + +static int lb_ready_open(struct inode *inode, struct file *file) +{ + struct seq_file *m; + int ret; + + ret = seq_open(file, &show_ready_seq_ops); + if (ret) { + return ret; + } + + m = file->private_data; + return 0; +} + +static int lb_ready_release(struct inode *inode, struct file *file) +{ + return seq_release(inode, file); +} + static const struct file_operations lb_trigger_fops = { .open = lb_open, .read = NULL, @@ -577,6 +619,13 @@ static const struct file_operations lb_status_fops= { .release = lb_status_release, }; +static const struct file_operations lb_ready_fops= { + .open = lb_ready_open, + .read = seq_read, + .llseek = seq_lseek, + .release = lb_ready_release, +}; + static struct dentry *lb_debugfs_root; static int lb_debugfs_init(void) @@ -595,6 +644,9 @@ static int lb_debugfs_init(void) lb_debugfs_root, NULL, &lb_delay_fops); debugfs_create_file("status", 0400, lb_debugfs_root, NULL, &lb_status_fops); + debugfs_create_file("ready", 0400, + lb_debugfs_root, NULL, &lb_ready_fops); + return 0; }