1 // SPDX-FileCopyrightText: 2008 Paul E. McKenney, IBM Corporation.
3 // SPDX-License-Identifier: GPL-2.0-or-later
6 * rcutorture.h: simple user-level performance/stress test of RCU.
9 * ./rcu <nreaders> rperf [ <cpustride> ]
10 * Run a read-side performance test with the specified
11 * number of readers spaced by <cpustride>.
12 * Thus "./rcu 16 rperf 2" would run 16 readers on even-numbered
14 * ./rcu <nupdaters> uperf [ <cpustride> ]
15 * Run an update-side performance test with the specified
16 * number of updaters and specified CPU spacing.
17 * ./rcu <nreaders> perf [ <cpustride> ]
18 * Run a combined read/update performance test with the specified
19 * number of readers and one updater and specified CPU spacing.
20 * The readers run on the low-numbered CPUs and the updater
21 * of the highest-numbered CPU.
23 * The above tests produce output as follows:
25 * n_reads: 46008000 n_updates: 146026 nreaders: 2 nupdaters: 1 duration: 1
26 * ns/read: 43.4707 ns/update: 6848.1
28 * The first line lists the total number of RCU reads and updates executed
29 * during the test, the number of reader threads, the number of updater
30 * threads, and the duration of the test in seconds. The second line
31 * lists the average duration of each type of operation in nanoseconds,
32 * or "nan" if the corresponding type of operation was not performed.
34 * ./rcu <nreaders> stress
35 * Run a stress test with the specified number of readers and
36 * one updater. None of the threads are affinitied to any
39 * This test produces output as follows:
41 * n_reads: 114633217 n_updates: 3903415 n_mberror: 0
42 * rcu_stress_count: 114618391 14826 0 0 0 0 0 0 0 0 0
44 * The first line lists the number of RCU read and update operations
45 * executed, followed by the number of memory-ordering violations
46 * (which will be zero in a correct RCU implementation). The second
47 * line lists the number of readers observing progressively more stale
48 * data. A correct RCU implementation will have all but the first two
61 DEFINE_PER_THREAD(long long, n_reads_pt
);
62 DEFINE_PER_THREAD(long long, n_updates_pt
);
71 WRITER_STATE_SYNC_RCU
,
72 WRITER_STATE_CALL_RCU
,
73 WRITER_STATE_POLL_RCU
,
76 static enum callrcu_type callrcu_type
= CALLRCU_GLOBAL
;
78 long long n_reads
= 0LL;
87 volatile int goflag
__attribute__((__aligned__(CAA_CACHE_LINE_SIZE
)))
90 #define RCU_READ_RUN 1000
93 #define RCU_READ_NESTABLE
95 #ifdef RCU_READ_NESTABLE
96 #define rcu_read_lock_nest() rcu_read_lock()
97 #define rcu_read_unlock_nest() rcu_read_unlock()
98 #else /* #ifdef RCU_READ_NESTABLE */
99 #define rcu_read_lock_nest()
100 #define rcu_read_unlock_nest()
101 #endif /* #else #ifdef RCU_READ_NESTABLE */
104 #define mark_rcu_quiescent_state rcu_quiescent_state
105 #define put_thread_offline rcu_thread_offline
106 #define put_thread_online rcu_thread_online
109 #ifndef mark_rcu_quiescent_state
110 #define mark_rcu_quiescent_state() do {} while (0)
111 #endif /* #ifdef mark_rcu_quiescent_state */
113 #ifndef put_thread_offline
114 #define put_thread_offline() do {} while (0)
115 #define put_thread_online() do {} while (0)
116 #define put_thread_online_delay() do {} while (0)
117 #else /* #ifndef put_thread_offline */
118 #define put_thread_online_delay() synchronize_rcu()
119 #endif /* #else #ifndef put_thread_offline */
126 void *rcu_read_perf_test(void *arg
)
130 long long n_reads_local
= 0;
132 rcu_register_thread();
134 uatomic_inc(&nthreadsrunning
);
135 put_thread_offline();
136 while (goflag
== GOFLAG_INIT
)
137 (void) poll(NULL
, 0, 1);
139 while (goflag
== GOFLAG_RUN
) {
140 for (i
= 0; i
< RCU_READ_RUN
; i
++) {
142 /* rcu_read_lock_nest(); */
143 /* rcu_read_unlock_nest(); */
146 n_reads_local
+= RCU_READ_RUN
;
147 mark_rcu_quiescent_state();
149 __get_thread_var(n_reads_pt
) += n_reads_local
;
150 put_thread_offline();
151 rcu_unregister_thread();
157 void *rcu_update_perf_test(void *arg
__attribute__((unused
)))
159 long long n_updates_local
= 0;
161 if (callrcu_type
== CALLRCU_PERTHREAD
) {
162 struct call_rcu_data
*crdp
;
164 crdp
= create_call_rcu_data(0, -1);
166 diag("Successfully using per-thread call_rcu() worker.");
167 set_thread_call_rcu_data(crdp
);
170 uatomic_inc(&nthreadsrunning
);
171 while (goflag
== GOFLAG_INIT
)
172 (void) poll(NULL
, 0, 1);
173 while (goflag
== GOFLAG_RUN
) {
177 __get_thread_var(n_updates_pt
) += n_updates_local
;
178 if (callrcu_type
== CALLRCU_PERTHREAD
) {
179 struct call_rcu_data
*crdp
;
181 crdp
= get_thread_call_rcu_data();
182 set_thread_call_rcu_data(NULL
);
183 call_rcu_data_free(crdp
);
189 void perftestinit(void)
191 init_per_thread(n_reads_pt
, 0LL);
192 init_per_thread(n_updates_pt
, 0LL);
193 uatomic_set(&nthreadsrunning
, 0);
197 int perftestrun(int nthreads
, int nreaders
, int nupdaters
)
203 while (uatomic_read(&nthreadsrunning
) < nthreads
)
204 (void) poll(NULL
, 0, 1);
209 goflag
= GOFLAG_STOP
;
213 n_reads
+= per_thread(n_reads_pt
, t
);
214 n_updates
+= per_thread(n_updates_pt
, t
);
216 diag("n_reads: %lld n_updates: %ld nreaders: %d nupdaters: %d duration: %d",
217 n_reads
, n_updates
, nreaders
, nupdaters
, duration
);
218 diag("ns/read: %g ns/update: %g",
219 ((duration
* 1000*1000*1000.*(double)nreaders
) /
221 ((duration
* 1000*1000*1000.*(double)nupdaters
) /
223 if (get_cpu_call_rcu_data(0)) {
224 diag("Deallocating per-CPU call_rcu threads.\n");
225 free_all_cpu_call_rcu_data();
231 int perftest(int nreaders
, int cpustride
)
237 for (i
= 0; i
< nreaders
; i
++) {
238 arg
= (long)(i
* cpustride
);
239 create_thread(rcu_read_perf_test
, (void *)arg
);
241 arg
= (long)(i
* cpustride
);
242 create_thread(rcu_update_perf_test
, (void *)arg
);
243 return perftestrun(i
+ 1, nreaders
, 1);
247 int rperftest(int nreaders
, int cpustride
)
253 init_per_thread(n_reads_pt
, 0LL);
254 for (i
= 0; i
< nreaders
; i
++) {
255 arg
= (long)(i
* cpustride
);
256 create_thread(rcu_read_perf_test
, (void *)arg
);
258 return perftestrun(i
, nreaders
, 0);
262 int uperftest(int nupdaters
, int cpustride
)
268 init_per_thread(n_reads_pt
, 0LL);
269 for (i
= 0; i
< nupdaters
; i
++) {
270 arg
= (long)(i
* cpustride
);
271 create_thread(rcu_update_perf_test
, (void *)arg
);
273 return perftestrun(i
, 0, nupdaters
);
280 #define RCU_STRESS_PIPE_LEN 10
287 struct rcu_stress rcu_stress_array
[RCU_STRESS_PIPE_LEN
] = { { 0, 0 } };
288 struct rcu_stress
*rcu_stress_current
;
289 int rcu_stress_idx
= 0;
292 DEFINE_PER_THREAD(long long [RCU_STRESS_PIPE_LEN
+ 1], rcu_stress_count
);
297 void *rcu_read_stress_test(void *arg
__attribute__((unused
)))
301 struct rcu_stress
*p
;
304 rcu_register_thread();
305 put_thread_offline();
306 while (goflag
== GOFLAG_INIT
)
307 (void) poll(NULL
, 0, 1);
309 while (goflag
== GOFLAG_RUN
) {
311 p
= rcu_dereference(rcu_stress_current
);
314 rcu_read_lock_nest();
315 for (i
= 0; i
< 100; i
++)
317 rcu_read_unlock_nest();
320 if ((pc
> RCU_STRESS_PIPE_LEN
) || (pc
< 0))
321 pc
= RCU_STRESS_PIPE_LEN
;
322 __get_thread_var(rcu_stress_count
)[pc
]++;
323 __get_thread_var(n_reads_pt
)++;
324 mark_rcu_quiescent_state();
325 if ((++itercnt
% 0x1000) == 0) {
326 put_thread_offline();
327 put_thread_online_delay();
331 put_thread_offline();
332 rcu_unregister_thread();
337 static pthread_mutex_t call_rcu_test_mutex
= PTHREAD_MUTEX_INITIALIZER
;
338 static pthread_cond_t call_rcu_test_cond
= PTHREAD_COND_INITIALIZER
;
341 void rcu_update_stress_test_rcu(struct rcu_head
*head
__attribute__((unused
)))
345 ret
= pthread_mutex_lock(&call_rcu_test_mutex
);
348 diag("pthread_mutex_lock: %s",
352 ret
= pthread_cond_signal(&call_rcu_test_cond
);
355 diag("pthread_cond_signal: %s",
359 ret
= pthread_mutex_unlock(&call_rcu_test_mutex
);
362 diag("pthread_mutex_unlock: %s",
369 void advance_writer_state(enum writer_state
*state
)
372 case WRITER_STATE_SYNC_RCU
:
373 *state
= WRITER_STATE_CALL_RCU
;
375 case WRITER_STATE_CALL_RCU
:
376 *state
= WRITER_STATE_POLL_RCU
;
378 case WRITER_STATE_POLL_RCU
:
379 *state
= WRITER_STATE_SYNC_RCU
;
385 void *rcu_update_stress_test(void *arg
__attribute__((unused
)))
388 struct rcu_stress
*p
;
390 enum writer_state writer_state
= WRITER_STATE_SYNC_RCU
;
392 while (goflag
== GOFLAG_INIT
)
393 (void) poll(NULL
, 0, 1);
394 while (goflag
== GOFLAG_RUN
) {
395 i
= rcu_stress_idx
+ 1;
396 if (i
>= RCU_STRESS_PIPE_LEN
)
398 p
= &rcu_stress_array
[i
];
403 rcu_assign_pointer(rcu_stress_current
, p
);
405 for (i
= 0; i
< RCU_STRESS_PIPE_LEN
; i
++)
406 if (i
!= rcu_stress_idx
)
407 rcu_stress_array
[i
].pipe_count
++;
408 switch (writer_state
) {
409 case WRITER_STATE_SYNC_RCU
:
412 case WRITER_STATE_CALL_RCU
:
416 ret
= pthread_mutex_lock(&call_rcu_test_mutex
);
419 diag("pthread_mutex_lock: %s",
423 rcu_register_thread();
424 call_rcu(&rh
, rcu_update_stress_test_rcu
);
425 rcu_unregister_thread();
427 * Our MacOS X test machine with the following
429 * 15.6.0 Darwin Kernel Version 15.6.0
430 * root:xnu-3248.60.10~1/RELEASE_X86_64
431 * appears to have issues with liburcu-signal
432 * signal being delivered on top of
433 * pthread_cond_wait. It seems to make the
434 * thread continue, and therefore corrupt the
435 * rcu_head. Work around this issue by
436 * unregistering the RCU read-side thread
437 * immediately after call_rcu (call_rcu needs
438 * us to be registered RCU readers).
440 ret
= pthread_cond_wait(&call_rcu_test_cond
,
441 &call_rcu_test_mutex
);
444 diag("pthread_cond_signal: %s",
448 ret
= pthread_mutex_unlock(&call_rcu_test_mutex
);
451 diag("pthread_mutex_unlock: %s",
457 case WRITER_STATE_POLL_RCU
:
459 struct urcu_gp_poll_state poll_state
;
461 rcu_register_thread();
462 poll_state
= start_poll_synchronize_rcu();
463 rcu_unregister_thread();
464 while (!poll_state_synchronize_rcu(poll_state
))
465 (void) poll(NULL
, 0, 1); /* Wait for 1ms */
470 advance_writer_state(&writer_state
);
477 void *rcu_fake_update_stress_test(void *arg
__attribute__((unused
)))
479 if (callrcu_type
== CALLRCU_PERTHREAD
) {
480 struct call_rcu_data
*crdp
;
482 crdp
= create_call_rcu_data(0, -1);
484 diag("Successfully using per-thread call_rcu() worker.");
485 set_thread_call_rcu_data(crdp
);
488 while (goflag
== GOFLAG_INIT
)
489 (void) poll(NULL
, 0, 1);
490 while (goflag
== GOFLAG_RUN
) {
492 (void) poll(NULL
, 0, 1);
494 if (callrcu_type
== CALLRCU_PERTHREAD
) {
495 struct call_rcu_data
*crdp
;
497 crdp
= get_thread_call_rcu_data();
498 set_thread_call_rcu_data(NULL
);
499 call_rcu_data_free(crdp
);
505 int stresstest(int nreaders
)
512 init_per_thread(n_reads_pt
, 0LL);
514 p
= &per_thread(rcu_stress_count
,t
)[0];
515 for (i
= 0; i
<= RCU_STRESS_PIPE_LEN
; i
++)
518 rcu_stress_current
= &rcu_stress_array
[0];
519 rcu_stress_current
->pipe_count
= 0;
520 rcu_stress_current
->mbtest
= 1;
521 for (i
= 0; i
< nreaders
; i
++)
522 create_thread(rcu_read_stress_test
, NULL
);
523 create_thread(rcu_update_stress_test
, NULL
);
524 for (i
= 0; i
< 5; i
++)
525 create_thread(rcu_fake_update_stress_test
, NULL
);
531 goflag
= GOFLAG_STOP
;
535 n_reads
+= per_thread(n_reads_pt
, t
);
536 diag("n_reads: %lld n_updates: %ld n_mberror: %d",
537 n_reads
, n_updates
, n_mberror
);
539 rdiag("rcu_stress_count:");
540 for (i
= 0; i
<= RCU_STRESS_PIPE_LEN
; i
++) {
543 sum
+= per_thread(rcu_stress_count
, t
)[i
];
548 if (get_cpu_call_rcu_data(0)) {
549 diag("Deallocating per-CPU call_rcu threads.");
550 free_all_cpu_call_rcu_data();
563 void usage(char *argv
[]) __attribute__((__noreturn__
));
566 void usage(char *argv
[])
568 diag("Usage: %s nreaders [ perf | rperf | uperf | stress ] [ stride ] [ callrcu_global | callrcu_percpu | callrcu_perthread ]\n", argv
[0]);
572 int main(int argc
, char *argv
[])
577 plan_tests(NR_TESTS
);
582 const char *callrcu_str
= argv
[4];;
584 if (strcmp(callrcu_str
, "callrcu_global") == 0) {
585 callrcu_type
= CALLRCU_GLOBAL
;
586 } else if (strcmp(callrcu_str
, "callrcu_percpu") == 0) {
587 callrcu_type
= CALLRCU_PERCPU
;
588 } else if (strcmp(callrcu_str
, "callrcu_perthread") == 0) {
589 callrcu_type
= CALLRCU_PERTHREAD
;
596 switch (callrcu_type
) {
598 diag("Using global per-process call_rcu thread.");
601 diag("Using per-CPU call_rcu threads.");
602 if (create_all_cpu_call_rcu_data(0))
603 diag("create_all_cpu_call_rcu_data: %s",
606 case CALLRCU_PERTHREAD
:
607 diag("Using per-thread call_rcu() worker.");
614 yield_active
|= YIELD_READ
;
615 yield_active
|= YIELD_WRITE
;
619 if (strcmp(argv
[1], "-h") == 0
620 || strcmp(argv
[1], "--help") == 0) {
624 nreaders
= strtoul(argv
[1], NULL
, 0);
626 ok(!perftest(nreaders
, cpustride
),
627 "perftest readers: %d, stride: %d",
628 nreaders
, cpustride
);
632 cpustride
= strtoul(argv
[3], NULL
, 0);
633 if (strcmp(argv
[2], "perf") == 0)
634 ok(!perftest(nreaders
, cpustride
),
635 "perftest readers: %d, stride: %d",
636 nreaders
, cpustride
);
637 else if (strcmp(argv
[2], "rperf") == 0)
638 ok(!rperftest(nreaders
, cpustride
),
639 "rperftest readers: %d, stride: %d",
640 nreaders
, cpustride
);
641 else if (strcmp(argv
[2], "uperf") == 0)
642 ok(!uperftest(nreaders
, cpustride
),
643 "uperftest readers: %d, stride: %d",
644 nreaders
, cpustride
);
645 else if (strcmp(argv
[2], "stress") == 0)
646 ok(!stresstest(nreaders
),
647 "stresstest readers: %d, stride: %d",
648 nreaders
, cpustride
);
655 return exit_status();