5 #include <linux/module.h>
6 #include <linux/proc_fs.h>
7 #include <linux/sched.h>
8 #include <linux/timex.h>
9 #include <linux/wbias-rwlock.h>
10 #include <linux/kthread.h>
11 #include <linux/delay.h>
12 #include <linux/hardirq.h>
13 #include <linux/module.h>
14 #include <linux/percpu.h>
15 #include <linux/spinlock.h>
16 #include <asm/ptrace.h>
18 /* Test with no contention duration, in seconds */
19 #define SINGLE_WRITER_TEST_DURATION 10
20 #define SINGLE_READER_TEST_DURATION 10
21 #define MULTIPLE_READERS_TEST_DURATION 10
23 /* Test duration, in seconds */
24 #define TEST_DURATION 60
28 #define NR_TRYLOCK_WRITERS 1
30 #define NR_TRYLOCK_READERS 1
33 * 1 : test standard rwlock
34 * 0 : test wbiasrwlock
36 #define TEST_STD_RWLOCK 0
39 * 1 : test with thread and interrupt readers.
40 * 0 : test only with thread readers.
42 #define TEST_INTERRUPTS 1
45 #define NR_INTERRUPT_READERS 1
46 #define NR_TRYLOCK_INTERRUPT_READERS 1
48 #define NR_INTERRUPT_READERS 0
49 #define NR_TRYLOCK_INTERRUPT_READERS 0
53 * Writer iteration delay, in us. 0 for busy loop. Caution : writers can
56 #define WRITER_DELAY 100
57 #define TRYLOCK_WRITER_DELAY 1000
60 * Number of iterations after which a trylock writer fails.
61 * -1 for infinite loop.
63 #define TRYLOCK_WRITERS_FAIL_ITER 100
65 /* Thread and interrupt reader delay, in ms */
66 #define THREAD_READER_DELAY 0 /* busy loop */
67 #define INTERRUPT_READER_DELAY 100
69 static int var
[NR_VARS
];
70 static struct task_struct
*reader_threads
[NR_READERS
];
71 static struct task_struct
*trylock_reader_threads
[NR_TRYLOCK_READERS
];
72 static struct task_struct
*writer_threads
[NR_WRITERS
];
73 static struct task_struct
*trylock_writer_threads
[NR_TRYLOCK_WRITERS
];
74 static struct task_struct
*interrupt_reader
[NR_INTERRUPT_READERS
];
75 static struct task_struct
*trylock_interrupt_reader
[NR_TRYLOCK_INTERRUPT_READERS
];
79 static DEFINE_RWLOCK(std_rw_lock
);
81 #define wrap_read_lock() read_lock(&std_rw_lock)
82 #define wrap_read_trylock() read_trylock(&std_rw_lock)
83 #define wrap_read_unlock() read_unlock(&std_rw_lock)
85 #define wrap_read_lock_irq() read_lock(&std_rw_lock)
86 #define wrap_read_trylock_irq() read_trylock(&std_rw_lock)
87 #define wrap_read_unlock_irq() read_unlock(&std_rw_lock)
90 #define wrap_write_lock() write_lock_irq(&std_rw_lock)
91 #define wrap_write_unlock() write_unlock_irq(&std_rw_lock)
93 #define wrap_write_lock() write_lock(&std_rw_lock)
94 #define wrap_write_unlock() write_unlock(&std_rw_lock)
99 static DEFINE_WBIAS_RWLOCK(wbiasrwlock
);
101 #define wrap_read_lock() wbias_read_lock(&wbiasrwlock)
102 #define wrap_read_trylock() wbias_read_trylock(&wbiasrwlock)
103 #define wrap_read_unlock() wbias_read_unlock(&wbiasrwlock)
105 #define wrap_read_lock_irq() wbias_read_lock_irq(&wbiasrwlock)
106 #define wrap_read_trylock_irq() wbias_read_trylock_irq(&wbiasrwlock)
107 #define wrap_read_unlock_irq() wbias_read_unlock_irq(&wbiasrwlock)
109 #if (TEST_INTERRUPTS)
110 #define wrap_write_lock() wbias_write_lock_irq(&wbiasrwlock)
111 #define wrap_write_unlock() wbias_write_unlock_irq(&wbiasrwlock)
113 #define wrap_write_lock() wbias_write_lock(&wbiasrwlock)
114 #define wrap_write_unlock() wbias_write_unlock(&wbiasrwlock)
119 static cycles_t cycles_calibration_min
,
120 cycles_calibration_avg
,
121 cycles_calibration_max
;
123 static inline cycles_t
calibrate_cycles(cycles_t cycles
)
125 return cycles
- cycles_calibration_avg
;
128 struct proc_dir_entry
*pentry
= NULL
;
130 static int reader_thread(void *data
)
134 unsigned long iter
= 0;
135 cycles_t time1
, time2
, delay
, delaymax
= 0, delaymin
= ULLONG_MAX
,
138 printk("reader_thread/%lu runnning\n", (unsigned long)data
);
141 preempt_disable(); /* for get_cycles accuracy */
143 time1
= get_cycles();
149 time2
= get_cycles();
151 delay
= time2
- time1
;
152 delaymax
= max(delaymax
, delay
);
153 delaymin
= min(delaymin
, delay
);
156 for (i
= 1; i
< NR_VARS
; i
++) {
160 "Unequal cur %d/prev %d at i %d, iter %lu "
161 "in thread\n", cur
, prev
, i
, iter
);
166 preempt_enable(); /* for get_cycles accuracy */
167 if (THREAD_READER_DELAY
)
168 msleep(THREAD_READER_DELAY
);
169 } while (!kthread_should_stop());
171 printk("reader_thread/%lu iterations : %lu",
172 (unsigned long)data
, iter
);
175 printk("reader_thread/%lu iterations : %lu, "
176 "lock delay [min,avg,max] %llu,%llu,%llu cycles\n",
177 (unsigned long)data
, iter
,
178 calibrate_cycles(delaymin
),
179 calibrate_cycles(delayavg
),
180 calibrate_cycles(delaymax
));
185 static int trylock_reader_thread(void *data
)
189 unsigned long iter
= 0, success_iter
= 0;
191 printk("trylock_reader_thread/%lu runnning\n", (unsigned long)data
);
193 while (!wrap_read_trylock())
197 for (i
= 1; i
< NR_VARS
; i
++) {
201 "Unequal cur %d/prev %d at i %d, iter %lu "
202 "in thread\n", cur
, prev
, i
, iter
);
205 if (THREAD_READER_DELAY
)
206 msleep(THREAD_READER_DELAY
);
207 } while (!kthread_should_stop());
208 printk("trylock_reader_thread/%lu iterations : %lu, "
209 "successful iterations : %lu\n",
210 (unsigned long)data
, iter
, success_iter
);
214 DEFINE_PER_CPU(cycles_t
, int_delaymin
);
215 DEFINE_PER_CPU(cycles_t
, int_delayavg
);
216 DEFINE_PER_CPU(cycles_t
, int_delaymax
);
217 DEFINE_PER_CPU(cycles_t
, int_ipi_nr
);
219 static void interrupt_reader_ipi(void *data
)
223 cycles_t time1
, time2
;
224 cycles_t
*delaymax
, *delaymin
, *delayavg
, *ipi_nr
, delay
;
227 * Skip the ipi caller, not in irq context.
232 delaymax
= &per_cpu(int_delaymax
, smp_processor_id());
233 delaymin
= &per_cpu(int_delaymin
, smp_processor_id());
234 delayavg
= &per_cpu(int_delayavg
, smp_processor_id());
235 ipi_nr
= &per_cpu(int_ipi_nr
, smp_processor_id());
238 time1
= get_cycles();
241 wrap_read_lock_irq();
244 time2
= get_cycles();
246 delay
= time2
- time1
;
247 *delaymax
= max(*delaymax
, delay
);
248 *delaymin
= min(*delaymin
, delay
);
252 for (i
= 1; i
< NR_VARS
; i
++) {
256 "Unequal cur %d/prev %d at i %d in interrupt\n",
259 wrap_read_unlock_irq();
262 DEFINE_PER_CPU(unsigned long, trylock_int_iter
);
263 DEFINE_PER_CPU(unsigned long, trylock_int_success
);
265 static void trylock_interrupt_reader_ipi(void *data
)
271 * Skip the ipi caller, not in irq context.
276 per_cpu(trylock_int_iter
, smp_processor_id())++;
277 while (!wrap_read_trylock_irq())
278 per_cpu(trylock_int_iter
, smp_processor_id())++;
279 per_cpu(trylock_int_success
, smp_processor_id())++;
281 for (i
= 1; i
< NR_VARS
; i
++) {
285 "Unequal cur %d/prev %d at i %d in interrupt\n",
288 wrap_read_unlock_irq();
292 static int interrupt_reader_thread(void *data
)
294 unsigned long iter
= 0;
297 for_each_online_cpu(i
) {
298 per_cpu(int_delaymax
, i
) = 0;
299 per_cpu(int_delaymin
, i
) = ULLONG_MAX
;
300 per_cpu(int_delayavg
, i
) = 0;
301 per_cpu(int_ipi_nr
, i
) = 0;
305 on_each_cpu(interrupt_reader_ipi
, NULL
, 0);
306 if (INTERRUPT_READER_DELAY
)
307 msleep(INTERRUPT_READER_DELAY
);
308 } while (!kthread_should_stop());
309 printk("interrupt_reader_thread/%lu iterations : %lu\n",
310 (unsigned long)data
, iter
);
311 for_each_online_cpu(i
) {
312 if (!per_cpu(int_ipi_nr
, i
))
314 per_cpu(int_delayavg
, i
) /= per_cpu(int_ipi_nr
, i
);
315 printk("interrupt readers on CPU %i, "
316 "lock delay [min,avg,max] %llu,%llu,%llu cycles\n",
318 calibrate_cycles(per_cpu(int_delaymin
, i
)),
319 calibrate_cycles(per_cpu(int_delayavg
, i
)),
320 calibrate_cycles(per_cpu(int_delaymax
, i
)));
325 static int trylock_interrupt_reader_thread(void *data
)
327 unsigned long iter
= 0;
332 on_each_cpu(trylock_interrupt_reader_ipi
, NULL
, 0);
333 if (INTERRUPT_READER_DELAY
)
334 msleep(INTERRUPT_READER_DELAY
);
335 } while (!kthread_should_stop());
336 printk("trylock_interrupt_reader_thread/%lu iterations : %lu\n",
337 (unsigned long)data
, iter
);
338 for_each_online_cpu(i
) {
339 printk("trylock interrupt readers on CPU %i, "
341 "successful iterations : %lu\n",
342 i
, per_cpu(trylock_int_iter
, i
),
343 per_cpu(trylock_int_success
, i
));
344 per_cpu(trylock_int_iter
, i
) = 0;
345 per_cpu(trylock_int_success
, i
) = 0;
350 static int writer_thread(void *data
)
354 unsigned long iter
= 0;
355 cycles_t time1
, time2
, delay
, delaymax
= 0, delaymin
= ULLONG_MAX
,
358 printk("writer_thread/%lu runnning\n", (unsigned long)data
);
361 preempt_disable(); /* for get_cycles accuracy */
363 time1
= get_cycles();
369 time2
= get_cycles();
371 delay
= time2
- time1
;
372 delaymax
= max(delaymax
, delay
);
373 delaymin
= min(delaymin
, delay
);
375 new = (int)get_cycles();
376 for (i
= 0; i
< NR_VARS
; i
++) {
382 preempt_enable(); /* for get_cycles accuracy */
383 if (WRITER_DELAY
> 0)
384 udelay(WRITER_DELAY
);
385 } while (!kthread_should_stop());
387 printk("writer_thread/%lu iterations : %lu, "
388 "lock delay [min,avg,max] %llu,%llu,%llu cycles\n",
389 (unsigned long)data
, iter
,
390 calibrate_cycles(delaymin
),
391 calibrate_cycles(delayavg
),
392 calibrate_cycles(delaymax
));
396 #if (TEST_STD_RWLOCK)
397 static int trylock_writer_thread(void *data
)
401 unsigned long iter
= 0, success
= 0, fail
= 0;
403 printk("trylock_writer_thread/%lu runnning\n", (unsigned long)data
);
405 #if (TEST_INTERRUPTS)
406 /* std write trylock cannot disable interrupts. */
410 #if (TRYLOCK_WRITERS_FAIL_ITER == -1)
413 if (write_trylock(&std_rw_lock
))
417 for (i
= 0; i
< TRYLOCK_WRITERS_FAIL_ITER
; i
++) {
419 if (write_trylock(&std_rw_lock
))
424 #if (TEST_INTERRUPTS)
430 new = (int)get_cycles();
431 for (i
= 0; i
< NR_VARS
; i
++) {
434 #if (TEST_INTERRUPTS)
435 write_unlock_irq(&std_rw_lock
);
437 write_unlock(&std_rw_lock
);
440 if (TRYLOCK_WRITER_DELAY
> 0)
441 udelay(TRYLOCK_WRITER_DELAY
);
442 } while (!kthread_should_stop());
443 printk("trylock_writer_thread/%lu iterations : "
444 "[try,success,fail after %d try], "
446 (unsigned long)data
, TRYLOCK_WRITERS_FAIL_ITER
,
447 iter
, success
, fail
);
451 #else /* !TEST_STD_RWLOCK */
453 static int trylock_writer_thread(void *data
)
457 unsigned long iter
= 0, success
= 0, fail
= 0;
459 printk("trylock_writer_thread/%lu runnning\n", (unsigned long)data
);
462 #if (TEST_INTERRUPTS)
463 if (wbias_write_trylock_irq_else_subscribe(&wbiasrwlock
))
465 if (wbias_write_trylock_else_subscribe(&wbiasrwlock
))
469 #if (TRYLOCK_WRITERS_FAIL_ITER == -1)
472 #if (TEST_INTERRUPTS)
473 if (wbias_write_trylock_irq_subscribed(&wbiasrwlock
))
475 if (wbias_write_trylock_subscribed(&wbiasrwlock
))
480 for (i
= 0; i
< TRYLOCK_WRITERS_FAIL_ITER
- 1; i
++) {
482 #if (TEST_INTERRUPTS)
483 if (wbias_write_trylock_irq_subscribed(&wbiasrwlock
))
485 if (wbias_write_trylock_subscribed(&wbiasrwlock
))
491 wbias_write_unsubscribe(&wbiasrwlock
);
495 new = (int)get_cycles();
496 for (i
= 0; i
< NR_VARS
; i
++) {
499 #if (TEST_INTERRUPTS)
500 wbias_write_unlock_irq(&wbiasrwlock
);
502 wbias_write_unlock(&wbiasrwlock
);
505 if (TRYLOCK_WRITER_DELAY
> 0)
506 udelay(TRYLOCK_WRITER_DELAY
);
507 } while (!kthread_should_stop());
508 printk("trylock_writer_thread/%lu iterations : "
509 "[try,success,fail after %d try], "
511 (unsigned long)data
, TRYLOCK_WRITERS_FAIL_ITER
,
512 iter
, success
, fail
);
516 #endif /* TEST_STD_RWLOCK */
518 static void wbias_rwlock_create(void)
522 for (i
= 0; i
< NR_READERS
; i
++) {
523 printk("starting reader thread %lu\n", i
);
524 reader_threads
[i
] = kthread_run(reader_thread
, (void *)i
,
525 "wbiasrwlock_reader");
526 BUG_ON(!reader_threads
[i
]);
529 for (i
= 0; i
< NR_TRYLOCK_READERS
; i
++) {
530 printk("starting trylock reader thread %lu\n", i
);
531 trylock_reader_threads
[i
] = kthread_run(trylock_reader_thread
,
532 (void *)i
, "wbiasrwlock_trylock_reader");
533 BUG_ON(!trylock_reader_threads
[i
]);
535 for (i
= 0; i
< NR_INTERRUPT_READERS
; i
++) {
536 printk("starting interrupt reader %lu\n", i
);
537 interrupt_reader
[i
] = kthread_run(interrupt_reader_thread
,
539 "wbiasrwlock_interrupt_reader");
541 for (i
= 0; i
< NR_TRYLOCK_INTERRUPT_READERS
; i
++) {
542 printk("starting trylock interrupt reader %lu\n", i
);
543 trylock_interrupt_reader
[i
] =
544 kthread_run(trylock_interrupt_reader_thread
,
545 (void *)i
, "wbiasrwlock_trylock_interrupt_reader");
547 for (i
= 0; i
< NR_WRITERS
; i
++) {
548 printk("starting writer thread %lu\n", i
);
549 writer_threads
[i
] = kthread_run(writer_thread
, (void *)i
,
550 "wbiasrwlock_writer");
551 BUG_ON(!writer_threads
[i
]);
553 for (i
= 0; i
< NR_TRYLOCK_WRITERS
; i
++) {
554 printk("starting trylock writer thread %lu\n", i
);
555 trylock_writer_threads
[i
] = kthread_run(trylock_writer_thread
,
556 (void *)i
, "wbiasrwlock_trylock_writer");
557 BUG_ON(!trylock_writer_threads
[i
]);
561 static void wbias_rwlock_stop(void)
565 for (i
= 0; i
< NR_WRITERS
; i
++)
566 kthread_stop(writer_threads
[i
]);
567 for (i
= 0; i
< NR_TRYLOCK_WRITERS
; i
++)
568 kthread_stop(trylock_writer_threads
[i
]);
569 for (i
= 0; i
< NR_READERS
; i
++)
570 kthread_stop(reader_threads
[i
]);
571 for (i
= 0; i
< NR_TRYLOCK_READERS
; i
++)
572 kthread_stop(trylock_reader_threads
[i
]);
573 for (i
= 0; i
< NR_INTERRUPT_READERS
; i
++)
574 kthread_stop(interrupt_reader
[i
]);
575 for (i
= 0; i
< NR_TRYLOCK_INTERRUPT_READERS
; i
++)
576 kthread_stop(trylock_interrupt_reader
[i
]);
580 static void perform_test(const char *name
, void (*callback
)(void))
582 printk("%s\n", name
);
586 static int my_open(struct inode
*inode
, struct file
*file
)
589 cycles_t time1
, time2
, delay
;
591 printk("** get_cycles calibration **\n");
592 cycles_calibration_min
= ULLONG_MAX
;
593 cycles_calibration_avg
= 0;
594 cycles_calibration_max
= 0;
597 for (i
= 0; i
< 10; i
++) {
599 time1
= get_cycles();
602 time2
= get_cycles();
604 delay
= time2
- time1
;
605 cycles_calibration_min
= min(cycles_calibration_min
, delay
);
606 cycles_calibration_avg
+= delay
;
607 cycles_calibration_max
= max(cycles_calibration_max
, delay
);
609 cycles_calibration_avg
/= 10;
612 printk("get_cycles takes [min,avg,max] %llu,%llu,%llu cycles, "
613 "results calibrated on avg\n",
614 cycles_calibration_min
,
615 cycles_calibration_avg
,
616 cycles_calibration_max
);
619 printk("** Single writer test, no contention **\n");
620 wbias_rwlock_profile_latency_reset();
621 writer_threads
[0] = kthread_run(writer_thread
, (void *)0,
622 "wbiasrwlock_writer");
623 BUG_ON(!writer_threads
[0]);
624 ssleep(SINGLE_WRITER_TEST_DURATION
);
625 kthread_stop(writer_threads
[0]);
628 wbias_rwlock_profile_latency_print();
630 printk("** Single trylock writer test, no contention **\n");
631 wbias_rwlock_profile_latency_reset();
632 trylock_writer_threads
[0] = kthread_run(trylock_writer_thread
,
634 "trylock_wbiasrwlock_writer");
635 BUG_ON(!trylock_writer_threads
[0]);
636 ssleep(SINGLE_WRITER_TEST_DURATION
);
637 kthread_stop(trylock_writer_threads
[0]);
640 wbias_rwlock_profile_latency_print();
642 printk("** Single reader test, no contention **\n");
643 wbias_rwlock_profile_latency_reset();
644 reader_threads
[0] = kthread_run(reader_thread
, (void *)0,
645 "wbiasrwlock_reader");
646 BUG_ON(!reader_threads
[0]);
647 ssleep(SINGLE_READER_TEST_DURATION
);
648 kthread_stop(reader_threads
[0]);
651 wbias_rwlock_profile_latency_print();
653 printk("** Multiple readers test, no contention **\n");
654 wbias_rwlock_profile_latency_reset();
655 for (i
= 0; i
< NR_READERS
; i
++) {
656 printk("starting reader thread %lu\n", i
);
657 reader_threads
[i
] = kthread_run(reader_thread
, (void *)i
,
658 "wbiasrwlock_reader");
659 BUG_ON(!reader_threads
[i
]);
661 ssleep(SINGLE_READER_TEST_DURATION
);
662 for (i
= 0; i
< NR_READERS
; i
++)
663 kthread_stop(reader_threads
[i
]);
666 wbias_rwlock_profile_latency_print();
668 printk("** High contention test **\n");
669 wbias_rwlock_profile_latency_reset();
670 perform_test("wbias-rwlock-create", wbias_rwlock_create
);
671 ssleep(TEST_DURATION
);
672 perform_test("wbias-rwlock-stop", wbias_rwlock_stop
);
674 wbias_rwlock_profile_latency_print();
680 static struct file_operations my_operations
= {
684 int init_module(void)
686 pentry
= create_proc_entry("testwbiasrwlock", 0444, NULL
);
688 pentry
->proc_fops
= &my_operations
;
690 printk("pow2cpus : %lu\n", pow2cpus
);
691 printk("THREAD_ROFFSET : %lX\n", THREAD_ROFFSET
);
692 printk("THREAD_RMASK : %lX\n", THREAD_RMASK
);
693 printk("SOFTIRQ_ROFFSET : %lX\n", SOFTIRQ_ROFFSET
);
694 printk("SOFTIRQ_RMASK : %lX\n", SOFTIRQ_RMASK
);
695 printk("HARDIRQ_ROFFSET : %lX\n", HARDIRQ_ROFFSET
);
696 printk("HARDIRQ_RMASK : %lX\n", HARDIRQ_RMASK
);
697 printk("SUBSCRIBERS_WOFFSET : %lX\n", SUBSCRIBERS_WOFFSET
);
698 printk("SUBSCRIBERS_WMASK : %lX\n", SUBSCRIBERS_WMASK
);
699 printk("WRITER_MUTEX : %lX\n", WRITER_MUTEX
);
700 printk("SOFTIRQ_WMASK : %lX\n", SOFTIRQ_WMASK
);
701 printk("HARDIRQ_WMASK : %lX\n", HARDIRQ_WMASK
);
706 void cleanup_module(void)
708 remove_proc_entry("testwbiasrwlock", NULL
);
711 MODULE_LICENSE("GPL");
712 MODULE_AUTHOR("Mathieu Desnoyers");
713 MODULE_DESCRIPTION("wbias rwlock test");