5 #include <linux/module.h>
6 #include <linux/proc_fs.h>
7 #include <linux/sched.h>
8 #include <linux/timex.h>
9 #include <linux/fair-rwlock.h>
10 #include <linux/kthread.h>
11 #include <linux/delay.h>
12 #include <linux/hardirq.h>
13 #include <linux/module.h>
14 #include <linux/percpu.h>
15 #include <asm/ptrace.h>
17 #if (NR_CPUS > 64 && (BITS_PER_LONG == 32 || NR_CPUS > 32768))
18 #error "fair rwlock needs more bits per long to deal with that many CPUs"
21 /* Test duration, in seconds */
22 #define TEST_DURATION 60
24 #define THREAD_ROFFSET 1UL
25 #define THREAD_RMASK ((NR_CPUS - 1) * THREAD_ROFFSET)
26 #define SOFTIRQ_ROFFSET (THREAD_RMASK + 1)
27 #define SOFTIRQ_RMASK ((NR_CPUS - 1) * SOFTIRQ_ROFFSET)
28 #define HARDIRQ_ROFFSET ((SOFTIRQ_RMASK | THREAD_RMASK) + 1)
29 #define HARDIRQ_RMASK ((NR_CPUS - 1) * HARDIRQ_ROFFSET)
31 #define SUBSCRIBERS_WOFFSET \
32 ((HARDIRQ_RMASK | SOFTIRQ_RMASK | THREAD_RMASK) + 1)
33 #define SUBSCRIBERS_WMASK \
34 ((NR_CPUS - 1) * SUBSCRIBERS_WOFFSET)
35 #define WRITER_MUTEX \
36 ((SUBSCRIBERS_WMASK | HARDIRQ_RMASK | SOFTIRQ_RMASK | THREAD_RMASK) + 1)
37 #define SOFTIRQ_WMASK (WRITER_MUTEX << 1)
38 #define SOFTIRQ_WOFFSET SOFTIRQ_WMASK
39 #define HARDIRQ_WMASK (SOFTIRQ_WMASK << 1)
40 #define HARDIRQ_WOFFSET HARDIRQ_WMASK
44 #define NR_TRYLOCK_WRITERS 2
46 #define NR_TRYLOCK_READERS 2
47 #define NR_INTERRUPT_READERS 1
48 #define NR_TRYLOCK_INTERRUPT_READERS 1
51 * Writer iteration delay, in us. 0 for busy loop. Caution : writers can
54 #define WRITER_DELAY 10
56 static int var
[NR_VARS
];
57 static struct task_struct
*reader_threads
[NR_READERS
];
58 static struct task_struct
*trylock_reader_threads
[NR_TRYLOCK_READERS
];
59 static struct task_struct
*writer_threads
[NR_WRITERS
];
60 static struct task_struct
*trylock_writer_threads
[NR_TRYLOCK_WRITERS
];
61 static struct task_struct
*interrupt_reader
;
62 static struct task_struct
*trylock_interrupt_reader
;
64 static struct fair_rwlock frwlock
= {
65 .value
= ATOMIC_LONG_INIT(0),
68 struct proc_dir_entry
*pentry
= NULL
;
70 static int reader_thread(void *data
)
74 unsigned long iter
= 0;
75 cycles_t time1
, time2
, delaymax
= 0;
77 printk("reader_thread/%lu runnning\n", (unsigned long)data
);
80 preempt_disable(); /* for get_cycles accuracy */
82 fair_read_lock(&frwlock
);
84 delaymax
= max(delaymax
, time2
- time1
);
86 for (i
= 1; i
< NR_VARS
; i
++) {
90 "Unequal cur %d/prev %d at i %d, iter %lu "
91 "in thread\n", cur
, prev
, i
, iter
);
93 fair_read_unlock(&frwlock
);
94 preempt_enable(); /* for get_cycles accuracy */
96 } while (!kthread_should_stop());
97 printk("reader_thread/%lu iterations : %lu, "
98 "max contention %llu cycles\n",
99 (unsigned long)data
, iter
, delaymax
);
103 static int trylock_reader_thread(void *data
)
107 unsigned long iter
= 0, success_iter
= 0;
109 printk("trylock_reader_thread/%lu runnning\n", (unsigned long)data
);
111 while (!fair_read_trylock(&frwlock
))
115 for (i
= 1; i
< NR_VARS
; i
++) {
119 "Unequal cur %d/prev %d at i %d, iter %lu "
120 "in thread\n", cur
, prev
, i
, iter
);
122 fair_read_unlock(&frwlock
);
124 } while (!kthread_should_stop());
125 printk("trylock_reader_thread/%lu iterations : %lu, "
126 "successful iterations : %lu\n",
127 (unsigned long)data
, iter
, success_iter
);
131 DEFINE_PER_CPU(cycles_t
, int_delaymax
);
133 static void interrupt_reader_ipi(void *data
)
137 cycles_t time1
, time2
;
141 * Skip the ipi caller, not in irq context.
146 delaymax
= &per_cpu(int_delaymax
, smp_processor_id());
147 time1
= get_cycles();
148 fair_read_lock(&frwlock
);
149 time2
= get_cycles();
150 *delaymax
= max(*delaymax
, time2
- time1
);
152 for (i
= 1; i
< NR_VARS
; i
++) {
156 "Unequal cur %d/prev %d at i %d in interrupt\n",
159 fair_read_unlock(&frwlock
);
162 DEFINE_PER_CPU(unsigned long, trylock_int_iter
);
163 DEFINE_PER_CPU(unsigned long, trylock_int_success
);
165 static void trylock_interrupt_reader_ipi(void *data
)
171 * Skip the ipi caller, not in irq context.
176 per_cpu(trylock_int_iter
, smp_processor_id())++;
177 while (!fair_read_trylock(&frwlock
))
178 per_cpu(trylock_int_iter
, smp_processor_id())++;
179 per_cpu(trylock_int_success
, smp_processor_id())++;
181 for (i
= 1; i
< NR_VARS
; i
++) {
185 "Unequal cur %d/prev %d at i %d in interrupt\n",
188 fair_read_unlock(&frwlock
);
192 static int interrupt_reader_thread(void *data
)
194 unsigned long iter
= 0;
199 on_each_cpu(interrupt_reader_ipi
, NULL
, 0);
201 } while (!kthread_should_stop());
202 printk("interrupt_reader_thread/%lu iterations : %lu\n",
203 (unsigned long)data
, iter
);
204 for_each_online_cpu(i
) {
205 printk("interrupt readers on CPU %i, "
206 "max contention : %llu cycles\n",
207 i
, per_cpu(int_delaymax
, i
));
212 static int trylock_interrupt_reader_thread(void *data
)
214 unsigned long iter
= 0;
219 on_each_cpu(trylock_interrupt_reader_ipi
, NULL
, 0);
221 } while (!kthread_should_stop());
222 printk("trylock_interrupt_reader_thread/%lu iterations : %lu\n",
223 (unsigned long)data
, iter
);
224 for_each_online_cpu(i
) {
225 printk("trylock interrupt readers on CPU %i, "
227 "successful iterations : %lu\n",
228 i
, per_cpu(trylock_int_iter
, i
),
229 per_cpu(trylock_int_success
, i
));
234 static int writer_thread(void *data
)
238 unsigned long iter
= 0;
239 cycles_t time1
, time2
, delaymax
= 0;
241 printk("writer_thread/%lu runnning\n", (unsigned long)data
);
244 preempt_disable(); /* for get_cycles accuracy */
245 time1
= get_cycles();
246 fair_write_lock_irq(&frwlock
);
247 //fair_write_lock(&frwlock);
248 time2
= get_cycles();
249 delaymax
= max(delaymax
, time2
- time1
);
250 new = (int)get_cycles();
251 for (i
= 0; i
< NR_VARS
; i
++) {
254 //fair_write_unlock(&frwlock);
255 fair_write_unlock_irq(&frwlock
);
256 preempt_enable(); /* for get_cycles accuracy */
257 if (WRITER_DELAY
> 0)
258 udelay(WRITER_DELAY
);
259 } while (!kthread_should_stop());
260 printk("writer_thread/%lu iterations : %lu, "
261 "max contention %llu cycles\n",
262 (unsigned long)data
, iter
, delaymax
);
266 static int trylock_writer_thread(void *data
)
270 unsigned long iter
= 0, success
= 0;
272 printk("trylock_writer_thread/%lu runnning\n", (unsigned long)data
);
274 fair_write_subscribe(&frwlock
);
275 while (!fair_write_trylock_subscribed_irq(&frwlock
)) {
280 //fair_write_lock(&frwlock);
281 new = (int)get_cycles();
282 for (i
= 0; i
< NR_VARS
; i
++) {
285 //fair_write_unlock(&frwlock);
286 fair_write_unlock_irq(&frwlock
);
287 if (WRITER_DELAY
> 0)
288 udelay(WRITER_DELAY
);
289 } while (!kthread_should_stop());
290 printk("trylock_writer_thread/%lu iterations : %lu, "
291 "successful iterations : %lu\n",
292 (unsigned long)data
, iter
, success
);
296 static void fair_rwlock_create(void)
300 for (i
= 0; i
< NR_READERS
; i
++) {
301 printk("starting reader thread %lu\n", i
);
302 reader_threads
[i
] = kthread_run(reader_thread
, (void *)i
,
304 BUG_ON(!reader_threads
[i
]);
307 for (i
= 0; i
< NR_TRYLOCK_READERS
; i
++) {
308 printk("starting trylock reader thread %lu\n", i
);
309 trylock_reader_threads
[i
] = kthread_run(trylock_reader_thread
,
310 (void *)i
, "frwlock_trylock_reader");
311 BUG_ON(!trylock_reader_threads
[i
]);
315 printk("starting interrupt reader %lu\n", i
);
316 interrupt_reader
= kthread_run(interrupt_reader_thread
, NULL
,
317 "frwlock_interrupt_reader");
318 printk("starting trylock interrupt reader %lu\n", i
);
319 trylock_interrupt_reader
= kthread_run(trylock_interrupt_reader_thread
,
320 NULL
, "frwlock_trylock_interrupt_reader");
322 for (i
= 0; i
< NR_WRITERS
; i
++) {
323 printk("starting writer thread %lu\n", i
);
324 writer_threads
[i
] = kthread_run(writer_thread
, (void *)i
,
326 BUG_ON(!writer_threads
[i
]);
328 for (i
= 0; i
< NR_TRYLOCK_WRITERS
; i
++) {
329 printk("starting trylock writer thread %lu\n", i
);
330 trylock_writer_threads
[i
] = kthread_run(trylock_writer_thread
,
331 (void *)i
, "frwlock_trylock_writer");
332 BUG_ON(!trylock_writer_threads
[i
]);
337 static void fair_rwlock_stop(void)
341 for (i
= 0; i
< NR_WRITERS
; i
++)
342 kthread_stop(writer_threads
[i
]);
343 for (i
= 0; i
< NR_TRYLOCK_WRITERS
; i
++)
344 kthread_stop(trylock_writer_threads
[i
]);
345 for (i
= 0; i
< NR_READERS
; i
++)
346 kthread_stop(reader_threads
[i
]);
347 for (i
= 0; i
< NR_TRYLOCK_READERS
; i
++)
348 kthread_stop(trylock_reader_threads
[i
]);
349 kthread_stop(interrupt_reader
);
350 kthread_stop(trylock_interrupt_reader
);
354 static void perform_test(const char *name
, void (*callback
)(void))
356 printk("%s\n", name
);
360 static int my_open(struct inode
*inode
, struct file
*file
)
362 perform_test("fair-rwlock-create", fair_rwlock_create
);
363 ssleep(TEST_DURATION
);
364 perform_test("fair-rwlock-stop", fair_rwlock_stop
);
370 static struct file_operations my_operations
= {
374 int init_module(void)
376 pentry
= create_proc_entry("testfrwlock", 0444, NULL
);
378 pentry
->proc_fops
= &my_operations
;
380 printk("NR_CPUS : %d\n", NR_CPUS
);
381 printk("THREAD_ROFFSET : %lX\n", THREAD_ROFFSET
);
382 printk("THREAD_RMASK : %lX\n", THREAD_RMASK
);
383 printk("SOFTIRQ_ROFFSET : %lX\n", SOFTIRQ_ROFFSET
);
384 printk("SOFTIRQ_RMASK : %lX\n", SOFTIRQ_RMASK
);
385 printk("HARDIRQ_ROFFSET : %lX\n", HARDIRQ_ROFFSET
);
386 printk("HARDIRQ_RMASK : %lX\n", HARDIRQ_RMASK
);
387 printk("SUBSCRIBERS_WOFFSET : %lX\n", SUBSCRIBERS_WOFFSET
);
388 printk("SUBSCRIBERS_WMASK : %lX\n", SUBSCRIBERS_WMASK
);
389 printk("WRITER_MUTEX : %lX\n", WRITER_MUTEX
);
390 printk("SOFTIRQ_WMASK : %lX\n", SOFTIRQ_WMASK
);
391 printk("HARDIRQ_WMASK : %lX\n", HARDIRQ_WMASK
);
396 void cleanup_module(void)
398 remove_proc_entry("testfrwlock", NULL
);
401 MODULE_LICENSE("GPL");
402 MODULE_AUTHOR("Mathieu Desnoyers");
403 MODULE_DESCRIPTION("Fair rwlock test");