5 #include <linux/module.h>
6 #include <linux/proc_fs.h>
7 #include <linux/sched.h>
8 #include <linux/timex.h>
9 #include <linux/fair-rwlock.h>
10 #include <linux/kthread.h>
11 #include <linux/delay.h>
12 #include <linux/hardirq.h>
13 #include <linux/module.h>
14 #include <linux/percpu.h>
15 #include <linux/spinlock.h>
16 #include <asm/ptrace.h>
18 #if (NR_CPUS > 64 && (BITS_PER_LONG == 32 || NR_CPUS > 32768))
19 #error "fair rwlock needs more bits per long to deal with that many CPUs"
22 /* Test with no contention duration, in seconds */
23 #define SINGLE_WRITER_TEST_DURATION 10
24 #define SINGLE_READER_TEST_DURATION 10
25 #define MULTIPLE_READERS_TEST_DURATION 10
27 /* Test duration, in seconds */
28 #define TEST_DURATION 60
31 //#define NR_WRITERS 2
33 //#define NR_TRYLOCK_WRITERS 2
34 #define NR_TRYLOCK_WRITERS 0
36 //#define NR_TRYLOCK_READERS 2
37 #define NR_TRYLOCK_READERS 0
40 * 1 : test standard rwlock
43 #define TEST_STD_RWLOCK 0
46 * 1 : test with thread and interrupt readers.
47 * 0 : test only with thread readers.
49 #define TEST_INTERRUPTS 1
52 #define NR_INTERRUPT_READERS 1
53 #define NR_TRYLOCK_INTERRUPT_READERS 1
55 #define NR_INTERRUPT_READERS 0
56 #define NR_TRYLOCK_INTERRUPT_READERS 0
60 * Writer iteration delay, in us. 0 for busy loop. Caution : writers can
63 #define WRITER_DELAY 100
64 #define TRYLOCK_WRITER_DELAY 1000
67 * Number of iterations after which a trylock writer fails.
68 * -1 for infinite loop.
70 #define TRYLOCK_WRITERS_FAIL_ITER 100
72 /* Thread and interrupt reader delay, in ms */
73 #define THREAD_READER_DELAY 0 /* busy loop */
74 #define INTERRUPT_READER_DELAY 100
76 static int var
[NR_VARS
];
77 static struct task_struct
*reader_threads
[NR_READERS
];
78 static struct task_struct
*trylock_reader_threads
[NR_TRYLOCK_READERS
];
79 static struct task_struct
*writer_threads
[NR_WRITERS
];
80 static struct task_struct
*trylock_writer_threads
[NR_TRYLOCK_WRITERS
];
81 static struct task_struct
*interrupt_reader
[NR_INTERRUPT_READERS
];
82 static struct task_struct
*trylock_interrupt_reader
[NR_TRYLOCK_INTERRUPT_READERS
];
86 static DEFINE_RWLOCK(std_rw_lock
);
88 #define wrap_read_lock() read_lock(&std_rw_lock)
89 #define wrap_read_trylock() read_trylock(&std_rw_lock)
90 #define wrap_read_unlock() read_unlock(&std_rw_lock)
92 #define wrap_read_lock_irq() read_lock(&std_rw_lock)
93 #define wrap_read_trylock_irq() read_trylock(&std_rw_lock)
94 #define wrap_read_unlock_irq() read_unlock(&std_rw_lock)
97 #define wrap_write_lock() write_lock_irq(&std_rw_lock)
98 #define wrap_write_unlock() write_unlock_irq(&std_rw_lock)
100 #define wrap_write_lock() write_lock(&std_rw_lock)
101 #define wrap_write_unlock() write_unlock(&std_rw_lock)
106 static struct fair_rwlock frwlock
= {
107 .value
= ATOMIC_LONG_INIT(0),
110 #define wrap_read_lock() fair_read_lock(&frwlock)
111 #define wrap_read_trylock() fair_read_trylock(&frwlock)
112 #define wrap_read_unlock() fair_read_unlock(&frwlock)
114 #define wrap_read_lock_irq() fair_read_lock_irq(&frwlock)
115 #define wrap_read_trylock_irq() fair_read_trylock_irq(&frwlock)
116 #define wrap_read_unlock_irq() fair_read_unlock_irq(&frwlock)
118 #if (TEST_INTERRUPTS)
119 #define wrap_write_lock() fair_write_lock_irq(&frwlock)
120 #define wrap_write_unlock() fair_write_unlock_irq(&frwlock)
122 #define wrap_write_lock() fair_write_lock(&frwlock)
123 #define wrap_write_unlock() fair_write_unlock(&frwlock)
128 static cycles_t cycles_calibration_min
,
129 cycles_calibration_avg
,
130 cycles_calibration_max
;
132 static inline cycles_t
calibrate_cycles(cycles_t cycles
)
134 return cycles
- cycles_calibration_avg
;
137 struct proc_dir_entry
*pentry
= NULL
;
139 static int reader_thread(void *data
)
143 unsigned long iter
= 0;
144 cycles_t time1
, time2
, delay
, delaymax
= 0, delaymin
= ULLONG_MAX
,
147 printk("reader_thread/%lu runnning\n", (unsigned long)data
);
150 preempt_disable(); /* for get_cycles accuracy */
152 time1
= get_cycles();
158 time2
= get_cycles();
160 delay
= time2
- time1
;
161 delaymax
= max(delaymax
, delay
);
162 delaymin
= min(delaymin
, delay
);
165 for (i
= 1; i
< NR_VARS
; i
++) {
169 "Unequal cur %d/prev %d at i %d, iter %lu "
170 "in thread\n", cur
, prev
, i
, iter
);
175 preempt_enable(); /* for get_cycles accuracy */
176 if (THREAD_READER_DELAY
)
177 msleep(THREAD_READER_DELAY
);
178 } while (!kthread_should_stop());
180 printk("reader_thread/%lu iterations : %lu",
181 (unsigned long)data
, iter
);
184 printk("reader_thread/%lu iterations : %lu, "
185 "lock delay [min,avg,max] %llu,%llu,%llu cycles\n",
186 (unsigned long)data
, iter
,
187 calibrate_cycles(delaymin
),
188 calibrate_cycles(delayavg
),
189 calibrate_cycles(delaymax
));
194 static int trylock_reader_thread(void *data
)
198 unsigned long iter
= 0, success_iter
= 0;
200 printk("trylock_reader_thread/%lu runnning\n", (unsigned long)data
);
202 while (!wrap_read_trylock())
206 for (i
= 1; i
< NR_VARS
; i
++) {
210 "Unequal cur %d/prev %d at i %d, iter %lu "
211 "in thread\n", cur
, prev
, i
, iter
);
214 if (THREAD_READER_DELAY
)
215 msleep(THREAD_READER_DELAY
);
216 } while (!kthread_should_stop());
217 printk("trylock_reader_thread/%lu iterations : %lu, "
218 "successful iterations : %lu\n",
219 (unsigned long)data
, iter
, success_iter
);
223 DEFINE_PER_CPU(cycles_t
, int_delaymin
);
224 DEFINE_PER_CPU(cycles_t
, int_delayavg
);
225 DEFINE_PER_CPU(cycles_t
, int_delaymax
);
226 DEFINE_PER_CPU(cycles_t
, int_ipi_nr
);
228 static void interrupt_reader_ipi(void *data
)
232 cycles_t time1
, time2
;
233 cycles_t
*delaymax
, *delaymin
, *delayavg
, *ipi_nr
, delay
;
236 * Skip the ipi caller, not in irq context.
241 delaymax
= &per_cpu(int_delaymax
, smp_processor_id());
242 delaymin
= &per_cpu(int_delaymin
, smp_processor_id());
243 delayavg
= &per_cpu(int_delayavg
, smp_processor_id());
244 ipi_nr
= &per_cpu(int_ipi_nr
, smp_processor_id());
247 time1
= get_cycles();
250 wrap_read_lock_irq();
253 time2
= get_cycles();
255 delay
= time2
- time1
;
256 *delaymax
= max(*delaymax
, delay
);
257 *delaymin
= min(*delaymin
, delay
);
261 for (i
= 1; i
< NR_VARS
; i
++) {
265 "Unequal cur %d/prev %d at i %d in interrupt\n",
268 wrap_read_unlock_irq();
271 DEFINE_PER_CPU(unsigned long, trylock_int_iter
);
272 DEFINE_PER_CPU(unsigned long, trylock_int_success
);
274 static void trylock_interrupt_reader_ipi(void *data
)
280 * Skip the ipi caller, not in irq context.
285 per_cpu(trylock_int_iter
, smp_processor_id())++;
286 while (!wrap_read_trylock_irq())
287 per_cpu(trylock_int_iter
, smp_processor_id())++;
288 per_cpu(trylock_int_success
, smp_processor_id())++;
290 for (i
= 1; i
< NR_VARS
; i
++) {
294 "Unequal cur %d/prev %d at i %d in interrupt\n",
297 wrap_read_unlock_irq();
301 static int interrupt_reader_thread(void *data
)
303 unsigned long iter
= 0;
306 for_each_online_cpu(i
) {
307 per_cpu(int_delaymax
, i
) = 0;
308 per_cpu(int_delaymin
, i
) = ULLONG_MAX
;
309 per_cpu(int_delayavg
, i
) = 0;
310 per_cpu(int_ipi_nr
, i
) = 0;
314 on_each_cpu(interrupt_reader_ipi
, NULL
, 0);
315 if (INTERRUPT_READER_DELAY
)
316 msleep(INTERRUPT_READER_DELAY
);
317 } while (!kthread_should_stop());
318 printk("interrupt_reader_thread/%lu iterations : %lu\n",
319 (unsigned long)data
, iter
);
320 for_each_online_cpu(i
) {
321 if (!per_cpu(int_ipi_nr
, i
))
323 per_cpu(int_delayavg
, i
) /= per_cpu(int_ipi_nr
, i
);
324 printk("interrupt readers on CPU %i, "
325 "lock delay [min,avg,max] %llu,%llu,%llu cycles\n",
327 calibrate_cycles(per_cpu(int_delaymin
, i
)),
328 calibrate_cycles(per_cpu(int_delayavg
, i
)),
329 calibrate_cycles(per_cpu(int_delaymax
, i
)));
334 static int trylock_interrupt_reader_thread(void *data
)
336 unsigned long iter
= 0;
341 on_each_cpu(trylock_interrupt_reader_ipi
, NULL
, 0);
342 if (INTERRUPT_READER_DELAY
)
343 msleep(INTERRUPT_READER_DELAY
);
344 } while (!kthread_should_stop());
345 printk("trylock_interrupt_reader_thread/%lu iterations : %lu\n",
346 (unsigned long)data
, iter
);
347 for_each_online_cpu(i
) {
348 printk("trylock interrupt readers on CPU %i, "
350 "successful iterations : %lu\n",
351 i
, per_cpu(trylock_int_iter
, i
),
352 per_cpu(trylock_int_success
, i
));
353 per_cpu(trylock_int_iter
, i
) = 0;
354 per_cpu(trylock_int_success
, i
) = 0;
359 static int writer_thread(void *data
)
363 unsigned long iter
= 0;
364 cycles_t time1
, time2
, delay
, delaymax
= 0, delaymin
= ULLONG_MAX
,
367 printk("writer_thread/%lu runnning\n", (unsigned long)data
);
370 preempt_disable(); /* for get_cycles accuracy */
372 time1
= get_cycles();
378 time2
= get_cycles();
380 delay
= time2
- time1
;
381 delaymax
= max(delaymax
, delay
);
382 delaymin
= min(delaymin
, delay
);
384 new = (int)get_cycles();
385 for (i
= 0; i
< NR_VARS
; i
++) {
391 preempt_enable(); /* for get_cycles accuracy */
392 if (WRITER_DELAY
> 0)
393 udelay(WRITER_DELAY
);
394 } while (!kthread_should_stop());
396 printk("writer_thread/%lu iterations : %lu, "
397 "lock delay [min,avg,max] %llu,%llu,%llu cycles\n",
398 (unsigned long)data
, iter
,
399 calibrate_cycles(delaymin
),
400 calibrate_cycles(delayavg
),
401 calibrate_cycles(delaymax
));
405 #if (TEST_STD_RWLOCK)
406 static int trylock_writer_thread(void *data
)
410 unsigned long iter
= 0, success
= 0, fail
= 0;
412 printk("trylock_writer_thread/%lu runnning\n", (unsigned long)data
);
414 #if (TEST_INTERRUPTS)
415 /* std write trylock cannot disable interrupts. */
419 #if (TRYLOCK_WRITERS_FAIL_ITER == -1)
422 if (write_trylock(&std_rw_lock
))
426 for (i
= 0; i
< TRYLOCK_WRITERS_FAIL_ITER
; i
++) {
428 if (write_trylock(&std_rw_lock
))
433 #if (TEST_INTERRUPTS)
439 new = (int)get_cycles();
440 for (i
= 0; i
< NR_VARS
; i
++) {
443 #if (TEST_INTERRUPTS)
444 write_unlock_irq(&std_rw_lock
);
446 write_unlock(&std_rw_lock
);
449 if (TRYLOCK_WRITER_DELAY
> 0)
450 udelay(TRYLOCK_WRITER_DELAY
);
451 } while (!kthread_should_stop());
452 printk("trylock_writer_thread/%lu iterations : "
453 "[try,success,fail after %d try], "
455 (unsigned long)data
, TRYLOCK_WRITERS_FAIL_ITER
,
456 iter
, success
, fail
);
460 #else /* !TEST_STD_RWLOCK */
462 static int trylock_writer_thread(void *data
)
466 unsigned long iter
= 0, success
= 0, fail
= 0;
468 printk("trylock_writer_thread/%lu runnning\n", (unsigned long)data
);
471 #if (TEST_INTERRUPTS)
472 if (fair_write_trylock_irq_else_subscribe(&frwlock
))
474 if (fair_write_trylock_else_subscribe(&frwlock
))
478 #if (TRYLOCK_WRITERS_FAIL_ITER == -1)
481 #if (TEST_INTERRUPTS)
482 if (fair_write_trylock_irq_subscribed(&frwlock
))
484 if (fair_write_trylock_subscribed(&frwlock
))
489 for (i
= 0; i
< TRYLOCK_WRITERS_FAIL_ITER
- 1; i
++) {
491 #if (TEST_INTERRUPTS)
492 if (fair_write_trylock_irq_subscribed(&frwlock
))
494 if (fair_write_trylock_subscribed(&frwlock
))
500 fair_write_unsubscribe(&frwlock
);
504 new = (int)get_cycles();
505 for (i
= 0; i
< NR_VARS
; i
++) {
508 #if (TEST_INTERRUPTS)
509 fair_write_unlock_irq(&frwlock
);
511 fair_write_unlock(&frwlock
);
514 if (TRYLOCK_WRITER_DELAY
> 0)
515 udelay(TRYLOCK_WRITER_DELAY
);
516 } while (!kthread_should_stop());
517 printk("trylock_writer_thread/%lu iterations : "
518 "[try,success,fail after %d try], "
520 (unsigned long)data
, TRYLOCK_WRITERS_FAIL_ITER
,
521 iter
, success
, fail
);
525 #endif /* TEST_STD_RWLOCK */
527 static void fair_rwlock_create(void)
531 for (i
= 0; i
< NR_READERS
; i
++) {
532 printk("starting reader thread %lu\n", i
);
533 reader_threads
[i
] = kthread_run(reader_thread
, (void *)i
,
535 BUG_ON(!reader_threads
[i
]);
538 for (i
= 0; i
< NR_TRYLOCK_READERS
; i
++) {
539 printk("starting trylock reader thread %lu\n", i
);
540 trylock_reader_threads
[i
] = kthread_run(trylock_reader_thread
,
541 (void *)i
, "frwlock_trylock_reader");
542 BUG_ON(!trylock_reader_threads
[i
]);
544 for (i
= 0; i
< NR_INTERRUPT_READERS
; i
++) {
545 printk("starting interrupt reader %lu\n", i
);
546 interrupt_reader
[i
] = kthread_run(interrupt_reader_thread
,
548 "frwlock_interrupt_reader");
550 for (i
= 0; i
< NR_TRYLOCK_INTERRUPT_READERS
; i
++) {
551 printk("starting trylock interrupt reader %lu\n", i
);
552 trylock_interrupt_reader
[i
] =
553 kthread_run(trylock_interrupt_reader_thread
,
554 (void *)i
, "frwlock_trylock_interrupt_reader");
556 for (i
= 0; i
< NR_WRITERS
; i
++) {
557 printk("starting writer thread %lu\n", i
);
558 writer_threads
[i
] = kthread_run(writer_thread
, (void *)i
,
560 BUG_ON(!writer_threads
[i
]);
562 for (i
= 0; i
< NR_TRYLOCK_WRITERS
; i
++) {
563 printk("starting trylock writer thread %lu\n", i
);
564 trylock_writer_threads
[i
] = kthread_run(trylock_writer_thread
,
565 (void *)i
, "frwlock_trylock_writer");
566 BUG_ON(!trylock_writer_threads
[i
]);
570 static void fair_rwlock_stop(void)
574 for (i
= 0; i
< NR_WRITERS
; i
++)
575 kthread_stop(writer_threads
[i
]);
576 for (i
= 0; i
< NR_TRYLOCK_WRITERS
; i
++)
577 kthread_stop(trylock_writer_threads
[i
]);
578 for (i
= 0; i
< NR_READERS
; i
++)
579 kthread_stop(reader_threads
[i
]);
580 for (i
= 0; i
< NR_TRYLOCK_READERS
; i
++)
581 kthread_stop(trylock_reader_threads
[i
]);
582 for (i
= 0; i
< NR_INTERRUPT_READERS
; i
++)
583 kthread_stop(interrupt_reader
[i
]);
584 for (i
= 0; i
< NR_TRYLOCK_INTERRUPT_READERS
; i
++)
585 kthread_stop(trylock_interrupt_reader
[i
]);
589 static void perform_test(const char *name
, void (*callback
)(void))
591 printk("%s\n", name
);
595 static int my_open(struct inode
*inode
, struct file
*file
)
598 cycles_t time1
, time2
, delay
;
600 printk("** get_cycles calibration **\n");
601 cycles_calibration_min
= ULLONG_MAX
;
602 cycles_calibration_avg
= 0;
603 cycles_calibration_max
= 0;
606 for (i
= 0; i
< 10; i
++) {
608 time1
= get_cycles();
611 time2
= get_cycles();
613 delay
= time2
- time1
;
614 cycles_calibration_min
= min(cycles_calibration_min
, delay
);
615 cycles_calibration_avg
+= delay
;
616 cycles_calibration_max
= max(cycles_calibration_max
, delay
);
618 cycles_calibration_avg
/= 10;
621 printk("get_cycles takes [min,avg,max] %llu,%llu,%llu cycles, "
622 "results calibrated on avg\n",
623 cycles_calibration_min
,
624 cycles_calibration_avg
,
625 cycles_calibration_max
);
627 printk("** Single writer test, no contention **\n");
628 writer_threads
[0] = kthread_run(writer_thread
, (void *)0,
630 BUG_ON(!writer_threads
[0]);
631 ssleep(SINGLE_WRITER_TEST_DURATION
);
632 kthread_stop(writer_threads
[0]);
634 printk("** Single trylock writer test, no contention **\n");
635 trylock_writer_threads
[0] = kthread_run(trylock_writer_thread
,
637 "trylock_frwlock_writer");
638 BUG_ON(!trylock_writer_threads
[0]);
639 ssleep(SINGLE_WRITER_TEST_DURATION
);
640 kthread_stop(trylock_writer_threads
[0]);
642 printk("** Single reader test, no contention **\n");
643 reader_threads
[0] = kthread_run(reader_thread
, (void *)0,
645 BUG_ON(!reader_threads
[0]);
646 ssleep(SINGLE_READER_TEST_DURATION
);
647 kthread_stop(reader_threads
[0]);
649 printk("** Multiple readers test, no contention **\n");
650 for (i
= 0; i
< NR_READERS
; i
++) {
651 printk("starting reader thread %lu\n", i
);
652 reader_threads
[i
] = kthread_run(reader_thread
, (void *)i
,
654 BUG_ON(!reader_threads
[i
]);
656 ssleep(SINGLE_READER_TEST_DURATION
);
657 for (i
= 0; i
< NR_READERS
; i
++)
658 kthread_stop(reader_threads
[i
]);
660 printk("** High contention test **\n");
661 perform_test("fair-rwlock-create", fair_rwlock_create
);
662 ssleep(TEST_DURATION
);
663 perform_test("fair-rwlock-stop", fair_rwlock_stop
);
669 static struct file_operations my_operations
= {
673 int init_module(void)
675 pentry
= create_proc_entry("testfrwlock", 0444, NULL
);
677 pentry
->proc_fops
= &my_operations
;
679 printk("NR_CPUS : %d\n", NR_CPUS
);
680 printk("THREAD_ROFFSET : %lX\n", THREAD_ROFFSET
);
681 printk("THREAD_RMASK : %lX\n", THREAD_RMASK
);
682 printk("SOFTIRQ_ROFFSET : %lX\n", SOFTIRQ_ROFFSET
);
683 printk("SOFTIRQ_RMASK : %lX\n", SOFTIRQ_RMASK
);
684 printk("HARDIRQ_ROFFSET : %lX\n", HARDIRQ_ROFFSET
);
685 printk("HARDIRQ_RMASK : %lX\n", HARDIRQ_RMASK
);
686 printk("SUBSCRIBERS_WOFFSET : %lX\n", SUBSCRIBERS_WOFFSET
);
687 printk("SUBSCRIBERS_WMASK : %lX\n", SUBSCRIBERS_WMASK
);
688 printk("WRITER_MUTEX : %lX\n", WRITER_MUTEX
);
689 printk("SOFTIRQ_WMASK : %lX\n", SOFTIRQ_WMASK
);
690 printk("HARDIRQ_WMASK : %lX\n", HARDIRQ_WMASK
);
695 void cleanup_module(void)
697 remove_proc_entry("testfrwlock", NULL
);
700 MODULE_LICENSE("GPL");
701 MODULE_AUTHOR("Mathieu Desnoyers");
702 MODULE_DESCRIPTION("Fair rwlock test");