2 * rcutorture.h: simple user-level performance/stress test of RCU.
5 * ./rcu <nreaders> rperf [ <cpustride> ]
6 * Run a read-side performance test with the specified
7 * number of readers spaced by <cpustride>.
8 * Thus "./rcu 16 rperf 2" would run 16 readers on even-numbered
10 * ./rcu <nupdaters> uperf [ <cpustride> ]
11 * Run an update-side performance test with the specified
12 * number of updaters and specified CPU spacing.
13 * ./rcu <nreaders> perf [ <cpustride> ]
14 * Run a combined read/update performance test with the specified
15 * number of readers and one updater and specified CPU spacing.
16 * The readers run on the low-numbered CPUs and the updater
17 * of the highest-numbered CPU.
19 * The above tests produce output as follows:
21 * n_reads: 46008000 n_updates: 146026 nreaders: 2 nupdaters: 1 duration: 1
22 * ns/read: 43.4707 ns/update: 6848.1
24 * The first line lists the total number of RCU reads and updates executed
25 * during the test, the number of reader threads, the number of updater
26 * threads, and the duration of the test in seconds. The second line
27 * lists the average duration of each type of operation in nanoseconds,
28 * or "nan" if the corresponding type of operation was not performed.
30 * ./rcu <nreaders> stress
31 * Run a stress test with the specified number of readers and
32 * one updater. None of the threads are affinitied to any
35 * This test produces output as follows:
37 * n_reads: 114633217 n_updates: 3903415 n_mberror: 0
38 * rcu_stress_count: 114618391 14826 0 0 0 0 0 0 0 0 0
40 * The first line lists the number of RCU read and update operations
41 * executed, followed by the number of memory-ordering violations
42 * (which will be zero in a correct RCU implementation). The second
43 * line lists the number of readers observing progressively more stale
44 * data. A correct RCU implementation will have all but the first two
47 * This program is free software; you can redistribute it and/or modify
48 * it under the terms of the GNU General Public License as published by
49 * the Free Software Foundation; either version 2 of the License, or
50 * (at your option) any later version.
52 * This program is distributed in the hope that it will be useful,
53 * but WITHOUT ANY WARRANTY; without even the implied warranty of
54 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
55 * GNU General Public License for more details.
57 * You should have received a copy of the GNU General Public License
58 * along with this program; if not, write to the Free Software
59 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
61 * Copyright (c) 2008 Paul E. McKenney, IBM Corporation.
73 DEFINE_PER_THREAD(long long, n_reads_pt
);
74 DEFINE_PER_THREAD(long long, n_updates_pt
);
83 WRITER_STATE_SYNC_RCU
,
84 WRITER_STATE_CALL_RCU
,
85 WRITER_STATE_POLL_RCU
,
88 static enum callrcu_type callrcu_type
= CALLRCU_GLOBAL
;
90 long long n_reads
= 0LL;
99 volatile int goflag
__attribute__((__aligned__(CAA_CACHE_LINE_SIZE
)))
102 #define RCU_READ_RUN 1000
105 #define RCU_READ_NESTABLE
107 #ifdef RCU_READ_NESTABLE
108 #define rcu_read_lock_nest() rcu_read_lock()
109 #define rcu_read_unlock_nest() rcu_read_unlock()
110 #else /* #ifdef RCU_READ_NESTABLE */
111 #define rcu_read_lock_nest()
112 #define rcu_read_unlock_nest()
113 #endif /* #else #ifdef RCU_READ_NESTABLE */
116 #define mark_rcu_quiescent_state rcu_quiescent_state
117 #define put_thread_offline rcu_thread_offline
118 #define put_thread_online rcu_thread_online
121 #ifndef mark_rcu_quiescent_state
122 #define mark_rcu_quiescent_state() do {} while (0)
123 #endif /* #ifdef mark_rcu_quiescent_state */
125 #ifndef put_thread_offline
126 #define put_thread_offline() do {} while (0)
127 #define put_thread_online() do {} while (0)
128 #define put_thread_online_delay() do {} while (0)
129 #else /* #ifndef put_thread_offline */
130 #define put_thread_online_delay() synchronize_rcu()
131 #endif /* #else #ifndef put_thread_offline */
138 void *rcu_read_perf_test(void *arg
)
142 long long n_reads_local
= 0;
144 rcu_register_thread();
146 uatomic_inc(&nthreadsrunning
);
147 put_thread_offline();
148 while (goflag
== GOFLAG_INIT
)
149 (void) poll(NULL
, 0, 1);
151 while (goflag
== GOFLAG_RUN
) {
152 for (i
= 0; i
< RCU_READ_RUN
; i
++) {
154 /* rcu_read_lock_nest(); */
155 /* rcu_read_unlock_nest(); */
158 n_reads_local
+= RCU_READ_RUN
;
159 mark_rcu_quiescent_state();
161 __get_thread_var(n_reads_pt
) += n_reads_local
;
162 put_thread_offline();
163 rcu_unregister_thread();
169 void *rcu_update_perf_test(void *arg
__attribute__((unused
)))
171 long long n_updates_local
= 0;
173 if (callrcu_type
== CALLRCU_PERTHREAD
) {
174 struct call_rcu_data
*crdp
;
176 crdp
= create_call_rcu_data(0, -1);
178 diag("Successfully using per-thread call_rcu() worker.");
179 set_thread_call_rcu_data(crdp
);
182 uatomic_inc(&nthreadsrunning
);
183 while (goflag
== GOFLAG_INIT
)
184 (void) poll(NULL
, 0, 1);
185 while (goflag
== GOFLAG_RUN
) {
189 __get_thread_var(n_updates_pt
) += n_updates_local
;
190 if (callrcu_type
== CALLRCU_PERTHREAD
) {
191 struct call_rcu_data
*crdp
;
193 crdp
= get_thread_call_rcu_data();
194 set_thread_call_rcu_data(NULL
);
195 call_rcu_data_free(crdp
);
201 void perftestinit(void)
203 init_per_thread(n_reads_pt
, 0LL);
204 init_per_thread(n_updates_pt
, 0LL);
205 uatomic_set(&nthreadsrunning
, 0);
209 int perftestrun(int nthreads
, int nreaders
, int nupdaters
)
215 while (uatomic_read(&nthreadsrunning
) < nthreads
)
216 (void) poll(NULL
, 0, 1);
221 goflag
= GOFLAG_STOP
;
225 n_reads
+= per_thread(n_reads_pt
, t
);
226 n_updates
+= per_thread(n_updates_pt
, t
);
228 diag("n_reads: %lld n_updates: %ld nreaders: %d nupdaters: %d duration: %d",
229 n_reads
, n_updates
, nreaders
, nupdaters
, duration
);
230 diag("ns/read: %g ns/update: %g",
231 ((duration
* 1000*1000*1000.*(double)nreaders
) /
233 ((duration
* 1000*1000*1000.*(double)nupdaters
) /
235 if (get_cpu_call_rcu_data(0)) {
236 diag("Deallocating per-CPU call_rcu threads.\n");
237 free_all_cpu_call_rcu_data();
243 int perftest(int nreaders
, int cpustride
)
249 for (i
= 0; i
< nreaders
; i
++) {
250 arg
= (long)(i
* cpustride
);
251 create_thread(rcu_read_perf_test
, (void *)arg
);
253 arg
= (long)(i
* cpustride
);
254 create_thread(rcu_update_perf_test
, (void *)arg
);
255 return perftestrun(i
+ 1, nreaders
, 1);
259 int rperftest(int nreaders
, int cpustride
)
265 init_per_thread(n_reads_pt
, 0LL);
266 for (i
= 0; i
< nreaders
; i
++) {
267 arg
= (long)(i
* cpustride
);
268 create_thread(rcu_read_perf_test
, (void *)arg
);
270 return perftestrun(i
, nreaders
, 0);
274 int uperftest(int nupdaters
, int cpustride
)
280 init_per_thread(n_reads_pt
, 0LL);
281 for (i
= 0; i
< nupdaters
; i
++) {
282 arg
= (long)(i
* cpustride
);
283 create_thread(rcu_update_perf_test
, (void *)arg
);
285 return perftestrun(i
, 0, nupdaters
);
292 #define RCU_STRESS_PIPE_LEN 10
299 struct rcu_stress rcu_stress_array
[RCU_STRESS_PIPE_LEN
] = { { 0, 0 } };
300 struct rcu_stress
*rcu_stress_current
;
301 int rcu_stress_idx
= 0;
304 DEFINE_PER_THREAD(long long [RCU_STRESS_PIPE_LEN
+ 1], rcu_stress_count
);
309 void *rcu_read_stress_test(void *arg
__attribute__((unused
)))
313 struct rcu_stress
*p
;
316 rcu_register_thread();
317 put_thread_offline();
318 while (goflag
== GOFLAG_INIT
)
319 (void) poll(NULL
, 0, 1);
321 while (goflag
== GOFLAG_RUN
) {
323 p
= rcu_dereference(rcu_stress_current
);
326 rcu_read_lock_nest();
327 for (i
= 0; i
< 100; i
++)
329 rcu_read_unlock_nest();
332 if ((pc
> RCU_STRESS_PIPE_LEN
) || (pc
< 0))
333 pc
= RCU_STRESS_PIPE_LEN
;
334 __get_thread_var(rcu_stress_count
)[pc
]++;
335 __get_thread_var(n_reads_pt
)++;
336 mark_rcu_quiescent_state();
337 if ((++itercnt
% 0x1000) == 0) {
338 put_thread_offline();
339 put_thread_online_delay();
343 put_thread_offline();
344 rcu_unregister_thread();
349 static pthread_mutex_t call_rcu_test_mutex
= PTHREAD_MUTEX_INITIALIZER
;
350 static pthread_cond_t call_rcu_test_cond
= PTHREAD_COND_INITIALIZER
;
353 void rcu_update_stress_test_rcu(struct rcu_head
*head
__attribute__((unused
)))
357 ret
= pthread_mutex_lock(&call_rcu_test_mutex
);
360 diag("pthread_mutex_lock: %s",
364 ret
= pthread_cond_signal(&call_rcu_test_cond
);
367 diag("pthread_cond_signal: %s",
371 ret
= pthread_mutex_unlock(&call_rcu_test_mutex
);
374 diag("pthread_mutex_unlock: %s",
381 void advance_writer_state(enum writer_state
*state
)
384 case WRITER_STATE_SYNC_RCU
:
385 *state
= WRITER_STATE_CALL_RCU
;
387 case WRITER_STATE_CALL_RCU
:
388 *state
= WRITER_STATE_POLL_RCU
;
390 case WRITER_STATE_POLL_RCU
:
391 *state
= WRITER_STATE_SYNC_RCU
;
397 void *rcu_update_stress_test(void *arg
__attribute__((unused
)))
400 struct rcu_stress
*p
;
402 enum writer_state writer_state
= WRITER_STATE_SYNC_RCU
;
404 while (goflag
== GOFLAG_INIT
)
405 (void) poll(NULL
, 0, 1);
406 while (goflag
== GOFLAG_RUN
) {
407 i
= rcu_stress_idx
+ 1;
408 if (i
>= RCU_STRESS_PIPE_LEN
)
410 p
= &rcu_stress_array
[i
];
415 rcu_assign_pointer(rcu_stress_current
, p
);
417 for (i
= 0; i
< RCU_STRESS_PIPE_LEN
; i
++)
418 if (i
!= rcu_stress_idx
)
419 rcu_stress_array
[i
].pipe_count
++;
420 switch (writer_state
) {
421 case WRITER_STATE_SYNC_RCU
:
424 case WRITER_STATE_CALL_RCU
:
428 ret
= pthread_mutex_lock(&call_rcu_test_mutex
);
431 diag("pthread_mutex_lock: %s",
435 rcu_register_thread();
436 call_rcu(&rh
, rcu_update_stress_test_rcu
);
437 rcu_unregister_thread();
439 * Our MacOS X test machine with the following
441 * 15.6.0 Darwin Kernel Version 15.6.0
442 * root:xnu-3248.60.10~1/RELEASE_X86_64
443 * appears to have issues with liburcu-signal
444 * signal being delivered on top of
445 * pthread_cond_wait. It seems to make the
446 * thread continue, and therefore corrupt the
447 * rcu_head. Work around this issue by
448 * unregistering the RCU read-side thread
449 * immediately after call_rcu (call_rcu needs
450 * us to be registered RCU readers).
452 ret
= pthread_cond_wait(&call_rcu_test_cond
,
453 &call_rcu_test_mutex
);
456 diag("pthread_cond_signal: %s",
460 ret
= pthread_mutex_unlock(&call_rcu_test_mutex
);
463 diag("pthread_mutex_unlock: %s",
469 case WRITER_STATE_POLL_RCU
:
471 struct urcu_gp_poll_state poll_state
;
473 rcu_register_thread();
474 poll_state
= start_poll_synchronize_rcu();
475 rcu_unregister_thread();
476 while (!poll_state_synchronize_rcu(poll_state
))
477 (void) poll(NULL
, 0, 1); /* Wait for 1ms */
482 advance_writer_state(&writer_state
);
489 void *rcu_fake_update_stress_test(void *arg
__attribute__((unused
)))
491 if (callrcu_type
== CALLRCU_PERTHREAD
) {
492 struct call_rcu_data
*crdp
;
494 crdp
= create_call_rcu_data(0, -1);
496 diag("Successfully using per-thread call_rcu() worker.");
497 set_thread_call_rcu_data(crdp
);
500 while (goflag
== GOFLAG_INIT
)
501 (void) poll(NULL
, 0, 1);
502 while (goflag
== GOFLAG_RUN
) {
504 (void) poll(NULL
, 0, 1);
506 if (callrcu_type
== CALLRCU_PERTHREAD
) {
507 struct call_rcu_data
*crdp
;
509 crdp
= get_thread_call_rcu_data();
510 set_thread_call_rcu_data(NULL
);
511 call_rcu_data_free(crdp
);
517 int stresstest(int nreaders
)
524 init_per_thread(n_reads_pt
, 0LL);
526 p
= &per_thread(rcu_stress_count
,t
)[0];
527 for (i
= 0; i
<= RCU_STRESS_PIPE_LEN
; i
++)
530 rcu_stress_current
= &rcu_stress_array
[0];
531 rcu_stress_current
->pipe_count
= 0;
532 rcu_stress_current
->mbtest
= 1;
533 for (i
= 0; i
< nreaders
; i
++)
534 create_thread(rcu_read_stress_test
, NULL
);
535 create_thread(rcu_update_stress_test
, NULL
);
536 for (i
= 0; i
< 5; i
++)
537 create_thread(rcu_fake_update_stress_test
, NULL
);
543 goflag
= GOFLAG_STOP
;
547 n_reads
+= per_thread(n_reads_pt
, t
);
548 diag("n_reads: %lld n_updates: %ld n_mberror: %d",
549 n_reads
, n_updates
, n_mberror
);
551 rdiag("rcu_stress_count:");
552 for (i
= 0; i
<= RCU_STRESS_PIPE_LEN
; i
++) {
555 sum
+= per_thread(rcu_stress_count
, t
)[i
];
560 if (get_cpu_call_rcu_data(0)) {
561 diag("Deallocating per-CPU call_rcu threads.");
562 free_all_cpu_call_rcu_data();
575 void usage(char *argv
[]) __attribute__((__noreturn__
));
578 void usage(char *argv
[])
580 diag("Usage: %s nreaders [ perf | rperf | uperf | stress ] [ stride ] [ callrcu_global | callrcu_percpu | callrcu_perthread ]\n", argv
[0]);
584 int main(int argc
, char *argv
[])
589 plan_tests(NR_TESTS
);
594 const char *callrcu_str
= argv
[4];;
596 if (strcmp(callrcu_str
, "callrcu_global") == 0) {
597 callrcu_type
= CALLRCU_GLOBAL
;
598 } else if (strcmp(callrcu_str
, "callrcu_percpu") == 0) {
599 callrcu_type
= CALLRCU_PERCPU
;
600 } else if (strcmp(callrcu_str
, "callrcu_perthread") == 0) {
601 callrcu_type
= CALLRCU_PERTHREAD
;
608 switch (callrcu_type
) {
610 diag("Using global per-process call_rcu thread.");
613 diag("Using per-CPU call_rcu threads.");
614 if (create_all_cpu_call_rcu_data(0))
615 diag("create_all_cpu_call_rcu_data: %s",
618 case CALLRCU_PERTHREAD
:
619 diag("Using per-thread call_rcu() worker.");
626 yield_active
|= YIELD_READ
;
627 yield_active
|= YIELD_WRITE
;
631 if (strcmp(argv
[1], "-h") == 0
632 || strcmp(argv
[1], "--help") == 0) {
636 nreaders
= strtoul(argv
[1], NULL
, 0);
638 ok(!perftest(nreaders
, cpustride
),
639 "perftest readers: %d, stride: %d",
640 nreaders
, cpustride
);
644 cpustride
= strtoul(argv
[3], NULL
, 0);
645 if (strcmp(argv
[2], "perf") == 0)
646 ok(!perftest(nreaders
, cpustride
),
647 "perftest readers: %d, stride: %d",
648 nreaders
, cpustride
);
649 else if (strcmp(argv
[2], "rperf") == 0)
650 ok(!rperftest(nreaders
, cpustride
),
651 "rperftest readers: %d, stride: %d",
652 nreaders
, cpustride
);
653 else if (strcmp(argv
[2], "uperf") == 0)
654 ok(!uperftest(nreaders
, cpustride
),
655 "uperftest readers: %d, stride: %d",
656 nreaders
, cpustride
);
657 else if (strcmp(argv
[2], "stress") == 0)
658 ok(!stresstest(nreaders
),
659 "stresstest readers: %d, stride: %d",
660 nreaders
, cpustride
);
667 return exit_status();