5 #include <linux/module.h>
6 #include <linux/proc_fs.h>
7 #include <linux/sched.h>
8 #include <linux/timex.h>
9 #include <linux/wbias-rwlock.h>
10 #include <linux/kthread.h>
11 #include <linux/delay.h>
12 #include <linux/hardirq.h>
13 #include <linux/module.h>
14 #include <linux/percpu.h>
15 #include <linux/spinlock.h>
16 #include <asm/ptrace.h>
18 #if (NR_CPUS > 64 && (BITS_PER_LONG == 32 || NR_CPUS > 32768))
19 #error "writer-biased rwlock needs more bits per long to deal with so many CPUs"
22 /* Test with no contention duration, in seconds */
23 #define SINGLE_WRITER_TEST_DURATION 10
24 #define SINGLE_READER_TEST_DURATION 10
25 #define MULTIPLE_READERS_TEST_DURATION 10
27 /* Test duration, in seconds */
28 #define TEST_DURATION 60
31 //#define NR_WRITERS 2
33 //#define NR_TRYLOCK_WRITERS 2
34 #define NR_TRYLOCK_WRITERS 0
36 //#define NR_TRYLOCK_READERS 2
37 #define NR_TRYLOCK_READERS 0
40 * 1 : test standard rwlock
41 * 0 : test wbiasrwlock
43 #define TEST_STD_RWLOCK 0
46 * 1 : test with thread and interrupt readers.
47 * 0 : test only with thread readers.
49 #define TEST_INTERRUPTS 1
52 #define NR_INTERRUPT_READERS 1
53 #define NR_TRYLOCK_INTERRUPT_READERS 1
55 #define NR_INTERRUPT_READERS 0
56 #define NR_TRYLOCK_INTERRUPT_READERS 0
60 * Writer iteration delay, in us. 0 for busy loop. Caution : writers can
63 #define WRITER_DELAY 100
64 #define TRYLOCK_WRITER_DELAY 1000
67 * Number of iterations after which a trylock writer fails.
68 * -1 for infinite loop.
70 #define TRYLOCK_WRITERS_FAIL_ITER 100
72 /* Thread and interrupt reader delay, in ms */
73 #define THREAD_READER_DELAY 0 /* busy loop */
74 #define INTERRUPT_READER_DELAY 100
76 static int var
[NR_VARS
];
77 static struct task_struct
*reader_threads
[NR_READERS
];
78 static struct task_struct
*trylock_reader_threads
[NR_TRYLOCK_READERS
];
79 static struct task_struct
*writer_threads
[NR_WRITERS
];
80 static struct task_struct
*trylock_writer_threads
[NR_TRYLOCK_WRITERS
];
81 static struct task_struct
*interrupt_reader
[NR_INTERRUPT_READERS
];
82 static struct task_struct
*trylock_interrupt_reader
[NR_TRYLOCK_INTERRUPT_READERS
];
86 static DEFINE_RWLOCK(std_rw_lock
);
88 #define wrap_read_lock() read_lock(&std_rw_lock)
89 #define wrap_read_trylock() read_trylock(&std_rw_lock)
90 #define wrap_read_unlock() read_unlock(&std_rw_lock)
92 #define wrap_read_lock_irq() read_lock(&std_rw_lock)
93 #define wrap_read_trylock_irq() read_trylock(&std_rw_lock)
94 #define wrap_read_unlock_irq() read_unlock(&std_rw_lock)
97 #define wrap_write_lock() write_lock_irq(&std_rw_lock)
98 #define wrap_write_unlock() write_unlock_irq(&std_rw_lock)
100 #define wrap_write_lock() write_lock(&std_rw_lock)
101 #define wrap_write_unlock() write_unlock(&std_rw_lock)
106 static DEFINE_WBIAS_RWLOCK(wbiasrwlock
);
108 #define wrap_read_lock() wbias_read_lock(&wbiasrwlock)
109 #define wrap_read_trylock() wbias_read_trylock(&wbiasrwlock)
110 #define wrap_read_unlock() wbias_read_unlock(&wbiasrwlock)
112 #define wrap_read_lock_irq() wbias_read_lock_irq(&wbiasrwlock)
113 #define wrap_read_trylock_irq() wbias_read_trylock_irq(&wbiasrwlock)
114 #define wrap_read_unlock_irq() wbias_read_unlock_irq(&wbiasrwlock)
116 #if (TEST_INTERRUPTS)
117 #define wrap_write_lock() wbias_write_lock_irq(&wbiasrwlock)
118 #define wrap_write_unlock() wbias_write_unlock_irq(&wbiasrwlock)
120 #define wrap_write_lock() wbias_write_lock(&wbiasrwlock)
121 #define wrap_write_unlock() wbias_write_unlock(&wbiasrwlock)
126 static cycles_t cycles_calibration_min
,
127 cycles_calibration_avg
,
128 cycles_calibration_max
;
130 static inline cycles_t
calibrate_cycles(cycles_t cycles
)
132 return cycles
- cycles_calibration_avg
;
135 struct proc_dir_entry
*pentry
= NULL
;
137 static int reader_thread(void *data
)
141 unsigned long iter
= 0;
142 cycles_t time1
, time2
, delay
, delaymax
= 0, delaymin
= ULLONG_MAX
,
145 printk("reader_thread/%lu runnning\n", (unsigned long)data
);
148 preempt_disable(); /* for get_cycles accuracy */
150 time1
= get_cycles();
156 time2
= get_cycles();
158 delay
= time2
- time1
;
159 delaymax
= max(delaymax
, delay
);
160 delaymin
= min(delaymin
, delay
);
163 for (i
= 1; i
< NR_VARS
; i
++) {
167 "Unequal cur %d/prev %d at i %d, iter %lu "
168 "in thread\n", cur
, prev
, i
, iter
);
173 preempt_enable(); /* for get_cycles accuracy */
174 if (THREAD_READER_DELAY
)
175 msleep(THREAD_READER_DELAY
);
176 } while (!kthread_should_stop());
178 printk("reader_thread/%lu iterations : %lu",
179 (unsigned long)data
, iter
);
182 printk("reader_thread/%lu iterations : %lu, "
183 "lock delay [min,avg,max] %llu,%llu,%llu cycles\n",
184 (unsigned long)data
, iter
,
185 calibrate_cycles(delaymin
),
186 calibrate_cycles(delayavg
),
187 calibrate_cycles(delaymax
));
192 static int trylock_reader_thread(void *data
)
196 unsigned long iter
= 0, success_iter
= 0;
198 printk("trylock_reader_thread/%lu runnning\n", (unsigned long)data
);
200 while (!wrap_read_trylock())
204 for (i
= 1; i
< NR_VARS
; i
++) {
208 "Unequal cur %d/prev %d at i %d, iter %lu "
209 "in thread\n", cur
, prev
, i
, iter
);
212 if (THREAD_READER_DELAY
)
213 msleep(THREAD_READER_DELAY
);
214 } while (!kthread_should_stop());
215 printk("trylock_reader_thread/%lu iterations : %lu, "
216 "successful iterations : %lu\n",
217 (unsigned long)data
, iter
, success_iter
);
221 DEFINE_PER_CPU(cycles_t
, int_delaymin
);
222 DEFINE_PER_CPU(cycles_t
, int_delayavg
);
223 DEFINE_PER_CPU(cycles_t
, int_delaymax
);
224 DEFINE_PER_CPU(cycles_t
, int_ipi_nr
);
226 static void interrupt_reader_ipi(void *data
)
230 cycles_t time1
, time2
;
231 cycles_t
*delaymax
, *delaymin
, *delayavg
, *ipi_nr
, delay
;
234 * Skip the ipi caller, not in irq context.
239 delaymax
= &per_cpu(int_delaymax
, smp_processor_id());
240 delaymin
= &per_cpu(int_delaymin
, smp_processor_id());
241 delayavg
= &per_cpu(int_delayavg
, smp_processor_id());
242 ipi_nr
= &per_cpu(int_ipi_nr
, smp_processor_id());
245 time1
= get_cycles();
248 wrap_read_lock_irq();
251 time2
= get_cycles();
253 delay
= time2
- time1
;
254 *delaymax
= max(*delaymax
, delay
);
255 *delaymin
= min(*delaymin
, delay
);
259 for (i
= 1; i
< NR_VARS
; i
++) {
263 "Unequal cur %d/prev %d at i %d in interrupt\n",
266 wrap_read_unlock_irq();
269 DEFINE_PER_CPU(unsigned long, trylock_int_iter
);
270 DEFINE_PER_CPU(unsigned long, trylock_int_success
);
272 static void trylock_interrupt_reader_ipi(void *data
)
278 * Skip the ipi caller, not in irq context.
283 per_cpu(trylock_int_iter
, smp_processor_id())++;
284 while (!wrap_read_trylock_irq())
285 per_cpu(trylock_int_iter
, smp_processor_id())++;
286 per_cpu(trylock_int_success
, smp_processor_id())++;
288 for (i
= 1; i
< NR_VARS
; i
++) {
292 "Unequal cur %d/prev %d at i %d in interrupt\n",
295 wrap_read_unlock_irq();
299 static int interrupt_reader_thread(void *data
)
301 unsigned long iter
= 0;
304 for_each_online_cpu(i
) {
305 per_cpu(int_delaymax
, i
) = 0;
306 per_cpu(int_delaymin
, i
) = ULLONG_MAX
;
307 per_cpu(int_delayavg
, i
) = 0;
308 per_cpu(int_ipi_nr
, i
) = 0;
312 on_each_cpu(interrupt_reader_ipi
, NULL
, 0);
313 if (INTERRUPT_READER_DELAY
)
314 msleep(INTERRUPT_READER_DELAY
);
315 } while (!kthread_should_stop());
316 printk("interrupt_reader_thread/%lu iterations : %lu\n",
317 (unsigned long)data
, iter
);
318 for_each_online_cpu(i
) {
319 if (!per_cpu(int_ipi_nr
, i
))
321 per_cpu(int_delayavg
, i
) /= per_cpu(int_ipi_nr
, i
);
322 printk("interrupt readers on CPU %i, "
323 "lock delay [min,avg,max] %llu,%llu,%llu cycles\n",
325 calibrate_cycles(per_cpu(int_delaymin
, i
)),
326 calibrate_cycles(per_cpu(int_delayavg
, i
)),
327 calibrate_cycles(per_cpu(int_delaymax
, i
)));
332 static int trylock_interrupt_reader_thread(void *data
)
334 unsigned long iter
= 0;
339 on_each_cpu(trylock_interrupt_reader_ipi
, NULL
, 0);
340 if (INTERRUPT_READER_DELAY
)
341 msleep(INTERRUPT_READER_DELAY
);
342 } while (!kthread_should_stop());
343 printk("trylock_interrupt_reader_thread/%lu iterations : %lu\n",
344 (unsigned long)data
, iter
);
345 for_each_online_cpu(i
) {
346 printk("trylock interrupt readers on CPU %i, "
348 "successful iterations : %lu\n",
349 i
, per_cpu(trylock_int_iter
, i
),
350 per_cpu(trylock_int_success
, i
));
351 per_cpu(trylock_int_iter
, i
) = 0;
352 per_cpu(trylock_int_success
, i
) = 0;
357 static int writer_thread(void *data
)
361 unsigned long iter
= 0;
362 cycles_t time1
, time2
, delay
, delaymax
= 0, delaymin
= ULLONG_MAX
,
365 printk("writer_thread/%lu runnning\n", (unsigned long)data
);
368 preempt_disable(); /* for get_cycles accuracy */
370 time1
= get_cycles();
376 time2
= get_cycles();
378 delay
= time2
- time1
;
379 delaymax
= max(delaymax
, delay
);
380 delaymin
= min(delaymin
, delay
);
382 new = (int)get_cycles();
383 for (i
= 0; i
< NR_VARS
; i
++) {
389 preempt_enable(); /* for get_cycles accuracy */
390 if (WRITER_DELAY
> 0)
391 udelay(WRITER_DELAY
);
392 } while (!kthread_should_stop());
394 printk("writer_thread/%lu iterations : %lu, "
395 "lock delay [min,avg,max] %llu,%llu,%llu cycles\n",
396 (unsigned long)data
, iter
,
397 calibrate_cycles(delaymin
),
398 calibrate_cycles(delayavg
),
399 calibrate_cycles(delaymax
));
403 #if (TEST_STD_RWLOCK)
404 static int trylock_writer_thread(void *data
)
408 unsigned long iter
= 0, success
= 0, fail
= 0;
410 printk("trylock_writer_thread/%lu runnning\n", (unsigned long)data
);
412 #if (TEST_INTERRUPTS)
413 /* std write trylock cannot disable interrupts. */
417 #if (TRYLOCK_WRITERS_FAIL_ITER == -1)
420 if (write_trylock(&std_rw_lock
))
424 for (i
= 0; i
< TRYLOCK_WRITERS_FAIL_ITER
; i
++) {
426 if (write_trylock(&std_rw_lock
))
431 #if (TEST_INTERRUPTS)
437 new = (int)get_cycles();
438 for (i
= 0; i
< NR_VARS
; i
++) {
441 #if (TEST_INTERRUPTS)
442 write_unlock_irq(&std_rw_lock
);
444 write_unlock(&std_rw_lock
);
447 if (TRYLOCK_WRITER_DELAY
> 0)
448 udelay(TRYLOCK_WRITER_DELAY
);
449 } while (!kthread_should_stop());
450 printk("trylock_writer_thread/%lu iterations : "
451 "[try,success,fail after %d try], "
453 (unsigned long)data
, TRYLOCK_WRITERS_FAIL_ITER
,
454 iter
, success
, fail
);
458 #else /* !TEST_STD_RWLOCK */
460 static int trylock_writer_thread(void *data
)
464 unsigned long iter
= 0, success
= 0, fail
= 0;
466 printk("trylock_writer_thread/%lu runnning\n", (unsigned long)data
);
469 #if (TEST_INTERRUPTS)
470 if (wbias_write_trylock_irq_else_subscribe(&wbiasrwlock
))
472 if (wbias_write_trylock_else_subscribe(&wbiasrwlock
))
476 #if (TRYLOCK_WRITERS_FAIL_ITER == -1)
479 #if (TEST_INTERRUPTS)
480 if (wbias_write_trylock_irq_subscribed(&wbiasrwlock
))
482 if (wbias_write_trylock_subscribed(&wbiasrwlock
))
487 for (i
= 0; i
< TRYLOCK_WRITERS_FAIL_ITER
- 1; i
++) {
489 #if (TEST_INTERRUPTS)
490 if (wbias_write_trylock_irq_subscribed(&wbiasrwlock
))
492 if (wbias_write_trylock_subscribed(&wbiasrwlock
))
498 wbias_write_unsubscribe(&wbiasrwlock
);
502 new = (int)get_cycles();
503 for (i
= 0; i
< NR_VARS
; i
++) {
506 #if (TEST_INTERRUPTS)
507 wbias_write_unlock_irq(&wbiasrwlock
);
509 wbias_write_unlock(&wbiasrwlock
);
512 if (TRYLOCK_WRITER_DELAY
> 0)
513 udelay(TRYLOCK_WRITER_DELAY
);
514 } while (!kthread_should_stop());
515 printk("trylock_writer_thread/%lu iterations : "
516 "[try,success,fail after %d try], "
518 (unsigned long)data
, TRYLOCK_WRITERS_FAIL_ITER
,
519 iter
, success
, fail
);
523 #endif /* TEST_STD_RWLOCK */
525 static void wbias_rwlock_create(void)
529 for (i
= 0; i
< NR_READERS
; i
++) {
530 printk("starting reader thread %lu\n", i
);
531 reader_threads
[i
] = kthread_run(reader_thread
, (void *)i
,
532 "wbiasrwlock_reader");
533 BUG_ON(!reader_threads
[i
]);
536 for (i
= 0; i
< NR_TRYLOCK_READERS
; i
++) {
537 printk("starting trylock reader thread %lu\n", i
);
538 trylock_reader_threads
[i
] = kthread_run(trylock_reader_thread
,
539 (void *)i
, "wbiasrwlock_trylock_reader");
540 BUG_ON(!trylock_reader_threads
[i
]);
542 for (i
= 0; i
< NR_INTERRUPT_READERS
; i
++) {
543 printk("starting interrupt reader %lu\n", i
);
544 interrupt_reader
[i
] = kthread_run(interrupt_reader_thread
,
546 "wbiasrwlock_interrupt_reader");
548 for (i
= 0; i
< NR_TRYLOCK_INTERRUPT_READERS
; i
++) {
549 printk("starting trylock interrupt reader %lu\n", i
);
550 trylock_interrupt_reader
[i
] =
551 kthread_run(trylock_interrupt_reader_thread
,
552 (void *)i
, "wbiasrwlock_trylock_interrupt_reader");
554 for (i
= 0; i
< NR_WRITERS
; i
++) {
555 printk("starting writer thread %lu\n", i
);
556 writer_threads
[i
] = kthread_run(writer_thread
, (void *)i
,
557 "wbiasrwlock_writer");
558 BUG_ON(!writer_threads
[i
]);
560 for (i
= 0; i
< NR_TRYLOCK_WRITERS
; i
++) {
561 printk("starting trylock writer thread %lu\n", i
);
562 trylock_writer_threads
[i
] = kthread_run(trylock_writer_thread
,
563 (void *)i
, "wbiasrwlock_trylock_writer");
564 BUG_ON(!trylock_writer_threads
[i
]);
568 static void wbias_rwlock_stop(void)
572 for (i
= 0; i
< NR_WRITERS
; i
++)
573 kthread_stop(writer_threads
[i
]);
574 for (i
= 0; i
< NR_TRYLOCK_WRITERS
; i
++)
575 kthread_stop(trylock_writer_threads
[i
]);
576 for (i
= 0; i
< NR_READERS
; i
++)
577 kthread_stop(reader_threads
[i
]);
578 for (i
= 0; i
< NR_TRYLOCK_READERS
; i
++)
579 kthread_stop(trylock_reader_threads
[i
]);
580 for (i
= 0; i
< NR_INTERRUPT_READERS
; i
++)
581 kthread_stop(interrupt_reader
[i
]);
582 for (i
= 0; i
< NR_TRYLOCK_INTERRUPT_READERS
; i
++)
583 kthread_stop(trylock_interrupt_reader
[i
]);
587 static void perform_test(const char *name
, void (*callback
)(void))
589 printk("%s\n", name
);
593 static int my_open(struct inode
*inode
, struct file
*file
)
596 cycles_t time1
, time2
, delay
;
598 printk("** get_cycles calibration **\n");
599 cycles_calibration_min
= ULLONG_MAX
;
600 cycles_calibration_avg
= 0;
601 cycles_calibration_max
= 0;
604 for (i
= 0; i
< 10; i
++) {
606 time1
= get_cycles();
609 time2
= get_cycles();
611 delay
= time2
- time1
;
612 cycles_calibration_min
= min(cycles_calibration_min
, delay
);
613 cycles_calibration_avg
+= delay
;
614 cycles_calibration_max
= max(cycles_calibration_max
, delay
);
616 cycles_calibration_avg
/= 10;
619 printk("get_cycles takes [min,avg,max] %llu,%llu,%llu cycles, "
620 "results calibrated on avg\n",
621 cycles_calibration_min
,
622 cycles_calibration_avg
,
623 cycles_calibration_max
);
625 printk("** Single writer test, no contention **\n");
626 writer_threads
[0] = kthread_run(writer_thread
, (void *)0,
627 "wbiasrwlock_writer");
628 BUG_ON(!writer_threads
[0]);
629 ssleep(SINGLE_WRITER_TEST_DURATION
);
630 kthread_stop(writer_threads
[0]);
632 printk("** Single trylock writer test, no contention **\n");
633 trylock_writer_threads
[0] = kthread_run(trylock_writer_thread
,
635 "trylock_wbiasrwlock_writer");
636 BUG_ON(!trylock_writer_threads
[0]);
637 ssleep(SINGLE_WRITER_TEST_DURATION
);
638 kthread_stop(trylock_writer_threads
[0]);
640 printk("** Single reader test, no contention **\n");
641 reader_threads
[0] = kthread_run(reader_thread
, (void *)0,
642 "wbiasrwlock_reader");
643 BUG_ON(!reader_threads
[0]);
644 ssleep(SINGLE_READER_TEST_DURATION
);
645 kthread_stop(reader_threads
[0]);
647 printk("** Multiple readers test, no contention **\n");
648 for (i
= 0; i
< NR_READERS
; i
++) {
649 printk("starting reader thread %lu\n", i
);
650 reader_threads
[i
] = kthread_run(reader_thread
, (void *)i
,
651 "wbiasrwlock_reader");
652 BUG_ON(!reader_threads
[i
]);
654 ssleep(SINGLE_READER_TEST_DURATION
);
655 for (i
= 0; i
< NR_READERS
; i
++)
656 kthread_stop(reader_threads
[i
]);
658 printk("** High contention test **\n");
659 perform_test("wbias-rwlock-create", wbias_rwlock_create
);
660 ssleep(TEST_DURATION
);
661 perform_test("wbias-rwlock-stop", wbias_rwlock_stop
);
667 static struct file_operations my_operations
= {
671 int init_module(void)
673 pentry
= create_proc_entry("testwbiasrwlock", 0444, NULL
);
675 pentry
->proc_fops
= &my_operations
;
677 printk("NR_CPUS : %d\n", NR_CPUS
);
678 printk("THREAD_ROFFSET : %lX\n", THREAD_ROFFSET
);
679 printk("THREAD_RMASK : %lX\n", THREAD_RMASK
);
680 printk("SOFTIRQ_ROFFSET : %lX\n", SOFTIRQ_ROFFSET
);
681 printk("SOFTIRQ_RMASK : %lX\n", SOFTIRQ_RMASK
);
682 printk("HARDIRQ_ROFFSET : %lX\n", HARDIRQ_ROFFSET
);
683 printk("HARDIRQ_RMASK : %lX\n", HARDIRQ_RMASK
);
684 printk("SUBSCRIBERS_WOFFSET : %lX\n", SUBSCRIBERS_WOFFSET
);
685 printk("SUBSCRIBERS_WMASK : %lX\n", SUBSCRIBERS_WMASK
);
686 printk("WRITER_MUTEX : %lX\n", WRITER_MUTEX
);
687 printk("SOFTIRQ_WMASK : %lX\n", SOFTIRQ_WMASK
);
688 printk("HARDIRQ_WMASK : %lX\n", HARDIRQ_WMASK
);
693 void cleanup_module(void)
695 remove_proc_entry("testwbiasrwlock", NULL
);
698 MODULE_LICENSE("GPL");
699 MODULE_AUTHOR("Mathieu Desnoyers");
700 MODULE_DESCRIPTION("wbias rwlock test");