5 #include <linux/module.h>
6 #include <linux/proc_fs.h>
7 #include <linux/sched.h>
8 #include <linux/timex.h>
9 #include <linux/fair-rwlock.h>
10 #include <linux/kthread.h>
11 #include <linux/delay.h>
12 #include <linux/hardirq.h>
13 #include <linux/module.h>
14 #include <linux/percpu.h>
15 #include <asm/ptrace.h>
17 #if (NR_CPUS > 64 && (BITS_PER_LONG == 32 || NR_CPUS > 32768))
18 #error "fair rwlock needs more bits per long to deal with that many CPUs"
21 /* Test with no contention duration, in seconds */
22 #define SINGLE_WRITER_TEST_DURATION 10
23 #define SINGLE_READER_TEST_DURATION 10
24 #define MULTIPLE_READERS_TEST_DURATION 10
26 /* Test duration, in seconds */
27 #define TEST_DURATION 60
29 #define THREAD_ROFFSET 1UL
30 #define THREAD_RMASK ((NR_CPUS - 1) * THREAD_ROFFSET)
31 #define SOFTIRQ_ROFFSET (THREAD_RMASK + 1)
32 #define SOFTIRQ_RMASK ((NR_CPUS - 1) * SOFTIRQ_ROFFSET)
33 #define HARDIRQ_ROFFSET ((SOFTIRQ_RMASK | THREAD_RMASK) + 1)
34 #define HARDIRQ_RMASK ((NR_CPUS - 1) * HARDIRQ_ROFFSET)
36 #define SUBSCRIBERS_WOFFSET \
37 ((HARDIRQ_RMASK | SOFTIRQ_RMASK | THREAD_RMASK) + 1)
38 #define SUBSCRIBERS_WMASK \
39 ((NR_CPUS - 1) * SUBSCRIBERS_WOFFSET)
40 #define WRITER_MUTEX \
41 ((SUBSCRIBERS_WMASK | HARDIRQ_RMASK | SOFTIRQ_RMASK | THREAD_RMASK) + 1)
42 #define SOFTIRQ_WMASK (WRITER_MUTEX << 1)
43 #define SOFTIRQ_WOFFSET SOFTIRQ_WMASK
44 #define HARDIRQ_WMASK (SOFTIRQ_WMASK << 1)
45 #define HARDIRQ_WOFFSET HARDIRQ_WMASK
49 #define NR_TRYLOCK_WRITERS 2
51 #define NR_TRYLOCK_READERS 2
54 * 1 : test with thread and interrupt readers.
55 * 0 : test only with thread readers.
57 #define TEST_INTERRUPTS 1
60 #define NR_INTERRUPT_READERS 1
61 #define NR_TRYLOCK_INTERRUPT_READERS 1
63 #define NR_INTERRUPT_READERS 0
64 #define NR_TRYLOCK_INTERRUPT_READERS 0
68 * Writer iteration delay, in us. 0 for busy loop. Caution : writers can
71 #define WRITER_DELAY 10
72 #define TRYLOCK_WRITER_DELAY 1000
75 * Number of iterations after which a trylock writer fails.
76 * -1 for infinite loop.
78 #define TRYLOCK_WRITERS_FAIL_ITER 100
80 /* Thread and interrupt reader delay, in ms */
81 #define THREAD_READER_DELAY 0 /* busy loop */
82 #define INTERRUPT_READER_DELAY 100
84 static int var
[NR_VARS
];
85 static struct task_struct
*reader_threads
[NR_READERS
];
86 static struct task_struct
*trylock_reader_threads
[NR_TRYLOCK_READERS
];
87 static struct task_struct
*writer_threads
[NR_WRITERS
];
88 static struct task_struct
*trylock_writer_threads
[NR_TRYLOCK_WRITERS
];
89 static struct task_struct
*interrupt_reader
[NR_INTERRUPT_READERS
];
90 static struct task_struct
*trylock_interrupt_reader
[NR_TRYLOCK_INTERRUPT_READERS
];
92 static struct fair_rwlock frwlock
= {
93 .value
= ATOMIC_LONG_INIT(0),
96 static cycles_t cycles_calibration_min
,
97 cycles_calibration_avg
,
98 cycles_calibration_max
;
100 static inline cycles_t
calibrate_cycles(cycles_t cycles
)
102 return cycles
- cycles_calibration_avg
;
105 struct proc_dir_entry
*pentry
= NULL
;
107 static int reader_thread(void *data
)
111 unsigned long iter
= 0;
112 cycles_t time1
, time2
, delay
, delaymax
= 0, delaymin
= ULLONG_MAX
,
115 printk("reader_thread/%lu runnning\n", (unsigned long)data
);
118 preempt_disable(); /* for get_cycles accuracy */
120 time1
= get_cycles();
123 fair_read_lock(&frwlock
);
126 time2
= get_cycles();
128 delay
= time2
- time1
;
129 delaymax
= max(delaymax
, delay
);
130 delaymin
= min(delaymin
, delay
);
133 for (i
= 1; i
< NR_VARS
; i
++) {
137 "Unequal cur %d/prev %d at i %d, iter %lu "
138 "in thread\n", cur
, prev
, i
, iter
);
140 fair_read_unlock(&frwlock
);
141 preempt_enable(); /* for get_cycles accuracy */
142 if (THREAD_READER_DELAY
)
143 msleep(THREAD_READER_DELAY
);
144 } while (!kthread_should_stop());
146 printk("reader_thread/%lu iterations : %lu",
147 (unsigned long)data
, iter
);
150 printk("reader_thread/%lu iterations : %lu, "
151 "lock delay [min,avg,max] %llu,%llu,%llu cycles\n",
152 (unsigned long)data
, iter
,
153 calibrate_cycles(delaymin
),
154 calibrate_cycles(delayavg
),
155 calibrate_cycles(delaymax
));
160 static int trylock_reader_thread(void *data
)
164 unsigned long iter
= 0, success_iter
= 0;
166 printk("trylock_reader_thread/%lu runnning\n", (unsigned long)data
);
168 while (!fair_read_trylock(&frwlock
))
172 for (i
= 1; i
< NR_VARS
; i
++) {
176 "Unequal cur %d/prev %d at i %d, iter %lu "
177 "in thread\n", cur
, prev
, i
, iter
);
179 fair_read_unlock(&frwlock
);
180 if (THREAD_READER_DELAY
)
181 msleep(THREAD_READER_DELAY
);
182 } while (!kthread_should_stop());
183 printk("trylock_reader_thread/%lu iterations : %lu, "
184 "successful iterations : %lu\n",
185 (unsigned long)data
, iter
, success_iter
);
189 DEFINE_PER_CPU(cycles_t
, int_delaymin
);
190 DEFINE_PER_CPU(cycles_t
, int_delayavg
);
191 DEFINE_PER_CPU(cycles_t
, int_delaymax
);
192 DEFINE_PER_CPU(cycles_t
, int_ipi_nr
);
194 static void interrupt_reader_ipi(void *data
)
198 cycles_t time1
, time2
;
199 cycles_t
*delaymax
, *delaymin
, *delayavg
, *ipi_nr
, delay
;
202 * Skip the ipi caller, not in irq context.
207 delaymax
= &per_cpu(int_delaymax
, smp_processor_id());
208 delaymin
= &per_cpu(int_delaymin
, smp_processor_id());
209 delayavg
= &per_cpu(int_delayavg
, smp_processor_id());
210 ipi_nr
= &per_cpu(int_ipi_nr
, smp_processor_id());
213 time1
= get_cycles();
216 fair_read_lock(&frwlock
);
219 time2
= get_cycles();
221 delay
= time2
- time1
;
222 *delaymax
= max(*delaymax
, delay
);
223 *delaymin
= min(*delaymin
, delay
);
227 for (i
= 1; i
< NR_VARS
; i
++) {
231 "Unequal cur %d/prev %d at i %d in interrupt\n",
234 fair_read_unlock(&frwlock
);
237 DEFINE_PER_CPU(unsigned long, trylock_int_iter
);
238 DEFINE_PER_CPU(unsigned long, trylock_int_success
);
240 static void trylock_interrupt_reader_ipi(void *data
)
246 * Skip the ipi caller, not in irq context.
251 per_cpu(trylock_int_iter
, smp_processor_id())++;
252 while (!fair_read_trylock(&frwlock
))
253 per_cpu(trylock_int_iter
, smp_processor_id())++;
254 per_cpu(trylock_int_success
, smp_processor_id())++;
256 for (i
= 1; i
< NR_VARS
; i
++) {
260 "Unequal cur %d/prev %d at i %d in interrupt\n",
263 fair_read_unlock(&frwlock
);
267 static int interrupt_reader_thread(void *data
)
269 unsigned long iter
= 0;
272 for_each_online_cpu(i
) {
273 per_cpu(int_delaymax
, i
) = 0;
274 per_cpu(int_delaymin
, i
) = ULLONG_MAX
;
275 per_cpu(int_delayavg
, i
) = 0;
276 per_cpu(int_ipi_nr
, i
) = 0;
280 on_each_cpu(interrupt_reader_ipi
, NULL
, 0);
281 if (INTERRUPT_READER_DELAY
)
282 msleep(INTERRUPT_READER_DELAY
);
283 } while (!kthread_should_stop());
284 printk("interrupt_reader_thread/%lu iterations : %lu\n",
285 (unsigned long)data
, iter
);
286 for_each_online_cpu(i
) {
287 if (!per_cpu(int_ipi_nr
, i
))
289 per_cpu(int_delayavg
, i
) /= per_cpu(int_ipi_nr
, i
);
290 printk("interrupt readers on CPU %i, "
291 "lock delay [min,avg,max] %llu,%llu,%llu cycles\n",
293 calibrate_cycles(per_cpu(int_delaymin
, i
)),
294 calibrate_cycles(per_cpu(int_delayavg
, i
)),
295 calibrate_cycles(per_cpu(int_delaymax
, i
)));
300 static int trylock_interrupt_reader_thread(void *data
)
302 unsigned long iter
= 0;
307 on_each_cpu(trylock_interrupt_reader_ipi
, NULL
, 0);
308 if (INTERRUPT_READER_DELAY
)
309 msleep(INTERRUPT_READER_DELAY
);
310 } while (!kthread_should_stop());
311 printk("trylock_interrupt_reader_thread/%lu iterations : %lu\n",
312 (unsigned long)data
, iter
);
313 for_each_online_cpu(i
) {
314 printk("trylock interrupt readers on CPU %i, "
316 "successful iterations : %lu\n",
317 i
, per_cpu(trylock_int_iter
, i
),
318 per_cpu(trylock_int_success
, i
));
319 per_cpu(trylock_int_iter
, i
) = 0;
320 per_cpu(trylock_int_success
, i
) = 0;
325 static int writer_thread(void *data
)
329 unsigned long iter
= 0;
330 cycles_t time1
, time2
, delay
, delaymax
= 0, delaymin
= ULLONG_MAX
,
333 printk("writer_thread/%lu runnning\n", (unsigned long)data
);
336 preempt_disable(); /* for get_cycles accuracy */
338 time1
= get_cycles();
341 #if (TEST_INTERRUPTS)
342 fair_write_lock_irq(&frwlock
);
344 fair_write_lock(&frwlock
);
348 time2
= get_cycles();
350 delay
= time2
- time1
;
351 delaymax
= max(delaymax
, delay
);
352 delaymin
= min(delaymin
, delay
);
354 new = (int)get_cycles();
355 for (i
= 0; i
< NR_VARS
; i
++) {
358 #if (TEST_INTERRUPTS)
359 fair_write_unlock_irq(&frwlock
);
361 fair_write_unlock(&frwlock
);
363 preempt_enable(); /* for get_cycles accuracy */
364 if (WRITER_DELAY
> 0)
365 udelay(WRITER_DELAY
);
366 } while (!kthread_should_stop());
368 printk("writer_thread/%lu iterations : %lu, "
369 "lock delay [min,avg,max] %llu,%llu,%llu cycles\n",
370 (unsigned long)data
, iter
,
371 calibrate_cycles(delaymin
),
372 calibrate_cycles(delayavg
),
373 calibrate_cycles(delaymax
));
377 static int trylock_writer_thread(void *data
)
381 unsigned long iter
= 0, success
= 0, fail
= 0;
383 printk("trylock_writer_thread/%lu runnning\n", (unsigned long)data
);
386 #if (TEST_INTERRUPTS)
387 if (fair_write_trylock_irq_else_subscribe(&frwlock
))
389 if (fair_write_trylock_else_subscribe(&frwlock
))
392 #if (TRYLOCK_WRITERS_FAIL_ITER == -1)
395 #if (TEST_INTERRUPTS)
396 if (fair_write_trylock_irq_subscribed(&frwlock
))
398 if (fair_write_trylock_subscribed(&frwlock
))
403 for (i
= 0; i
< TRYLOCK_WRITERS_FAIL_ITER
; i
++) {
405 #if (TEST_INTERRUPTS)
406 if (fair_write_trylock_irq_subscribed(&frwlock
))
408 if (fair_write_trylock_subscribed(&frwlock
))
414 fair_write_unsubscribe(&frwlock
);
418 new = (int)get_cycles();
419 for (i
= 0; i
< NR_VARS
; i
++) {
422 #if (TEST_INTERRUPTS)
423 fair_write_unlock_irq(&frwlock
);
425 fair_write_unlock(&frwlock
);
428 if (TRYLOCK_WRITER_DELAY
> 0)
429 udelay(TRYLOCK_WRITER_DELAY
);
430 } while (!kthread_should_stop());
431 printk("trylock_writer_thread/%lu iterations : "
432 "[try,success,fail after %d try], "
434 (unsigned long)data
, TRYLOCK_WRITERS_FAIL_ITER
,
435 iter
, success
, fail
);
439 static void fair_rwlock_create(void)
443 for (i
= 0; i
< NR_READERS
; i
++) {
444 printk("starting reader thread %lu\n", i
);
445 reader_threads
[i
] = kthread_run(reader_thread
, (void *)i
,
447 BUG_ON(!reader_threads
[i
]);
450 for (i
= 0; i
< NR_TRYLOCK_READERS
; i
++) {
451 printk("starting trylock reader thread %lu\n", i
);
452 trylock_reader_threads
[i
] = kthread_run(trylock_reader_thread
,
453 (void *)i
, "frwlock_trylock_reader");
454 BUG_ON(!trylock_reader_threads
[i
]);
456 for (i
= 0; i
< NR_INTERRUPT_READERS
; i
++) {
457 printk("starting interrupt reader %lu\n", i
);
458 interrupt_reader
[i
] = kthread_run(interrupt_reader_thread
,
460 "frwlock_interrupt_reader");
462 for (i
= 0; i
< NR_TRYLOCK_INTERRUPT_READERS
; i
++) {
463 printk("starting trylock interrupt reader %lu\n", i
);
464 trylock_interrupt_reader
[i
] =
465 kthread_run(trylock_interrupt_reader_thread
,
466 (void *)i
, "frwlock_trylock_interrupt_reader");
468 for (i
= 0; i
< NR_WRITERS
; i
++) {
469 printk("starting writer thread %lu\n", i
);
470 writer_threads
[i
] = kthread_run(writer_thread
, (void *)i
,
472 BUG_ON(!writer_threads
[i
]);
474 for (i
= 0; i
< NR_TRYLOCK_WRITERS
; i
++) {
475 printk("starting trylock writer thread %lu\n", i
);
476 trylock_writer_threads
[i
] = kthread_run(trylock_writer_thread
,
477 (void *)i
, "frwlock_trylock_writer");
478 BUG_ON(!trylock_writer_threads
[i
]);
482 static void fair_rwlock_stop(void)
486 for (i
= 0; i
< NR_WRITERS
; i
++)
487 kthread_stop(writer_threads
[i
]);
488 for (i
= 0; i
< NR_TRYLOCK_WRITERS
; i
++)
489 kthread_stop(trylock_writer_threads
[i
]);
490 for (i
= 0; i
< NR_READERS
; i
++)
491 kthread_stop(reader_threads
[i
]);
492 for (i
= 0; i
< NR_TRYLOCK_READERS
; i
++)
493 kthread_stop(trylock_reader_threads
[i
]);
494 for (i
= 0; i
< NR_INTERRUPT_READERS
; i
++)
495 kthread_stop(interrupt_reader
[i
]);
496 for (i
= 0; i
< NR_TRYLOCK_INTERRUPT_READERS
; i
++)
497 kthread_stop(trylock_interrupt_reader
[i
]);
501 static void perform_test(const char *name
, void (*callback
)(void))
503 printk("%s\n", name
);
507 static int my_open(struct inode
*inode
, struct file
*file
)
510 cycles_t time1
, time2
, delay
;
512 printk("** get_cycles calibration **\n");
513 cycles_calibration_min
= ULLONG_MAX
;
514 cycles_calibration_avg
= 0;
515 cycles_calibration_max
= 0;
518 for (i
= 0; i
< 10; i
++) {
520 time1
= get_cycles();
523 time2
= get_cycles();
525 delay
= time2
- time1
;
526 cycles_calibration_min
= min(cycles_calibration_min
, delay
);
527 cycles_calibration_avg
+= delay
;
528 cycles_calibration_max
= max(cycles_calibration_max
, delay
);
530 cycles_calibration_avg
/= 10;
533 printk("get_cycles takes [min,avg,max] %llu,%llu,%llu cycles, "
534 "results calibrated on avg\n",
535 cycles_calibration_min
,
536 cycles_calibration_avg
,
537 cycles_calibration_max
);
539 printk("** Single writer test, no contention **\n");
540 writer_threads
[0] = kthread_run(writer_thread
, (void *)0,
542 BUG_ON(!writer_threads
[0]);
543 ssleep(SINGLE_WRITER_TEST_DURATION
);
544 kthread_stop(writer_threads
[0]);
546 printk("** Single trylock writer test, no contention **\n");
547 trylock_writer_threads
[0] = kthread_run(trylock_writer_thread
,
549 "trylock_frwlock_writer");
550 BUG_ON(!trylock_writer_threads
[0]);
551 ssleep(SINGLE_WRITER_TEST_DURATION
);
552 kthread_stop(trylock_writer_threads
[0]);
554 printk("** Single reader test, no contention **\n");
555 reader_threads
[0] = kthread_run(reader_thread
, (void *)0,
557 BUG_ON(!reader_threads
[0]);
558 ssleep(SINGLE_READER_TEST_DURATION
);
559 kthread_stop(reader_threads
[0]);
561 printk("** Multiple readers test, no contention **\n");
562 for (i
= 0; i
< NR_READERS
; i
++) {
563 printk("starting reader thread %lu\n", i
);
564 reader_threads
[i
] = kthread_run(reader_thread
, (void *)i
,
566 BUG_ON(!reader_threads
[i
]);
568 ssleep(SINGLE_READER_TEST_DURATION
);
569 for (i
= 0; i
< NR_READERS
; i
++)
570 kthread_stop(reader_threads
[i
]);
572 printk("** High contention test **\n");
573 perform_test("fair-rwlock-create", fair_rwlock_create
);
574 ssleep(TEST_DURATION
);
575 perform_test("fair-rwlock-stop", fair_rwlock_stop
);
581 static struct file_operations my_operations
= {
585 int init_module(void)
587 pentry
= create_proc_entry("testfrwlock", 0444, NULL
);
589 pentry
->proc_fops
= &my_operations
;
591 printk("NR_CPUS : %d\n", NR_CPUS
);
592 printk("THREAD_ROFFSET : %lX\n", THREAD_ROFFSET
);
593 printk("THREAD_RMASK : %lX\n", THREAD_RMASK
);
594 printk("SOFTIRQ_ROFFSET : %lX\n", SOFTIRQ_ROFFSET
);
595 printk("SOFTIRQ_RMASK : %lX\n", SOFTIRQ_RMASK
);
596 printk("HARDIRQ_ROFFSET : %lX\n", HARDIRQ_ROFFSET
);
597 printk("HARDIRQ_RMASK : %lX\n", HARDIRQ_RMASK
);
598 printk("SUBSCRIBERS_WOFFSET : %lX\n", SUBSCRIBERS_WOFFSET
);
599 printk("SUBSCRIBERS_WMASK : %lX\n", SUBSCRIBERS_WMASK
);
600 printk("WRITER_MUTEX : %lX\n", WRITER_MUTEX
);
601 printk("SOFTIRQ_WMASK : %lX\n", SOFTIRQ_WMASK
);
602 printk("HARDIRQ_WMASK : %lX\n", HARDIRQ_WMASK
);
607 void cleanup_module(void)
609 remove_proc_entry("testfrwlock", NULL
);
612 MODULE_LICENSE("GPL");
613 MODULE_AUTHOR("Mathieu Desnoyers");
614 MODULE_DESCRIPTION("Fair rwlock test");