1 // SPDX-FileCopyrightText: 2008 Paul E. McKenney, IBM Corporation.
3 // SPDX-License-Identifier: GPL-2.0-or-later
6 * rcutorture.h: simple user-level performance/stress test of RCU.
9 * ./rcu <nreaders> rperf [ <cpustride> ]
10 * Run a read-side performance test with the specified
11 * number of readers spaced by <cpustride>.
12 * Thus "./rcu 16 rperf 2" would run 16 readers on even-numbered
14 * ./rcu <nupdaters> uperf [ <cpustride> ]
15 * Run an update-side performance test with the specified
16 * number of updaters and specified CPU spacing.
17 * ./rcu <nreaders> perf [ <cpustride> ]
18 * Run a combined read/update performance test with the specified
19 * number of readers and one updater and specified CPU spacing.
20 * The readers run on the low-numbered CPUs and the updater
21 * of the highest-numbered CPU.
23 * The above tests produce output as follows:
25 * n_reads: 46008000 n_updates: 146026 nreaders: 2 nupdaters: 1 duration: 1
26 * ns/read: 43.4707 ns/update: 6848.1
28 * The first line lists the total number of RCU reads and updates executed
29 * during the test, the number of reader threads, the number of updater
30 * threads, and the duration of the test in seconds. The second line
31 * lists the average duration of each type of operation in nanoseconds,
32 * or "nan" if the corresponding type of operation was not performed.
34 * ./rcu <nreaders> stress
35 * Run a stress test with the specified number of readers and
36 * one updater. None of the threads are affinitied to any
39 * This test produces output as follows:
41 * n_reads: 114633217 n_updates: 3903415 n_mberror: 0
42 * rcu_stress_count: 114618391 14826 0 0 0 0 0 0 0 0 0
44 * The first line lists the number of RCU read and update operations
45 * executed, followed by the number of memory-ordering violations
46 * (which will be zero in a correct RCU implementation). The second
47 * line lists the number of readers observing progressively more stale
48 * data. A correct RCU implementation will have all but the first two
51 * rcu_stress_count: Histogram of "ages" of structures seen by readers. If any
52 * entries past the first two are non-zero, RCU is broken. The age of a newly
53 * allocated structure is zero, it becomes one when removed from reader
54 * visibility, and is incremented once per grace period subsequently -- and is
55 * freed after passing through (RCU_STRESS_PIPE_LEN-2) grace periods. Since
56 * this tests only has one true writer (there are fake writers), only buckets at
57 * indexes 0 and 1 should be none-zero.
59 * This program is free software; you can redistribute it and/or modify
60 * it under the terms of the GNU General Public License as published by
61 * the Free Software Foundation; either version 2 of the License, or
62 * (at your option) any later version.
64 * This program is distributed in the hope that it will be useful,
65 * but WITHOUT ANY WARRANTY; without even the implied warranty of
66 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
67 * GNU General Public License for more details.
69 * You should have received a copy of the GNU General Public License
70 * along with this program; if not, write to the Free Software
71 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
73 * Copyright (c) 2008 Paul E. McKenney, IBM Corporation.
83 #include <urcu/uatomic.h>
85 #include "urcu-wait.h"
89 DEFINE_PER_THREAD(long long, n_reads_pt
);
90 DEFINE_PER_THREAD(long long, n_updates_pt
);
99 WRITER_STATE_SYNC_RCU
,
100 WRITER_STATE_CALL_RCU
,
101 WRITER_STATE_POLL_RCU
,
104 static enum callrcu_type callrcu_type
= CALLRCU_GLOBAL
;
106 long long n_reads
= 0LL;
111 #define GOFLAG_INIT 0
113 #define GOFLAG_STOP 2
115 volatile int goflag
__attribute__((__aligned__(CAA_CACHE_LINE_SIZE
)))
118 #define RCU_READ_RUN 1000
121 #define RCU_READ_NESTABLE
123 #ifdef RCU_READ_NESTABLE
124 #define rcu_read_lock_nest() rcu_read_lock()
125 #define rcu_read_unlock_nest() rcu_read_unlock()
126 #else /* #ifdef RCU_READ_NESTABLE */
127 #define rcu_read_lock_nest()
128 #define rcu_read_unlock_nest()
129 #endif /* #else #ifdef RCU_READ_NESTABLE */
132 #define mark_rcu_quiescent_state rcu_quiescent_state
133 #define put_thread_offline rcu_thread_offline
134 #define put_thread_online rcu_thread_online
137 #ifndef mark_rcu_quiescent_state
138 #define mark_rcu_quiescent_state() do {} while (0)
139 #endif /* #ifdef mark_rcu_quiescent_state */
141 #ifndef put_thread_offline
142 #define put_thread_offline() do {} while (0)
143 #define put_thread_online() do {} while (0)
144 #define put_thread_online_delay() do {} while (0)
145 #else /* #ifndef put_thread_offline */
146 #define put_thread_online_delay() synchronize_rcu()
147 #endif /* #else #ifndef put_thread_offline */
154 void *rcu_read_perf_test(void *arg
)
158 long long n_reads_local
= 0;
160 rcu_register_thread();
162 uatomic_inc(&nthreadsrunning
);
163 put_thread_offline();
164 while (uatomic_read(&goflag
) == GOFLAG_INIT
)
165 (void) poll(NULL
, 0, 1);
167 while (uatomic_read(&goflag
) == GOFLAG_RUN
) {
168 for (i
= 0; i
< RCU_READ_RUN
; i
++) {
170 /* rcu_read_lock_nest(); */
171 /* rcu_read_unlock_nest(); */
174 n_reads_local
+= RCU_READ_RUN
;
175 mark_rcu_quiescent_state();
177 __get_thread_var(n_reads_pt
) += n_reads_local
;
178 put_thread_offline();
179 rcu_unregister_thread();
185 void *rcu_update_perf_test(void *arg
__attribute__((unused
)))
187 long long n_updates_local
= 0;
189 if (callrcu_type
== CALLRCU_PERTHREAD
) {
190 struct call_rcu_data
*crdp
;
192 crdp
= create_call_rcu_data(0, -1);
194 diag("Successfully using per-thread call_rcu() worker.");
195 set_thread_call_rcu_data(crdp
);
198 uatomic_inc(&nthreadsrunning
);
199 while (uatomic_read(&goflag
) == GOFLAG_INIT
)
200 (void) poll(NULL
, 0, 1);
201 while (uatomic_read(&goflag
) == GOFLAG_RUN
) {
205 __get_thread_var(n_updates_pt
) += n_updates_local
;
206 if (callrcu_type
== CALLRCU_PERTHREAD
) {
207 struct call_rcu_data
*crdp
;
209 crdp
= get_thread_call_rcu_data();
210 set_thread_call_rcu_data(NULL
);
211 call_rcu_data_free(crdp
);
217 void perftestinit(void)
219 init_per_thread(n_reads_pt
, 0LL);
220 init_per_thread(n_updates_pt
, 0LL);
221 uatomic_set(&nthreadsrunning
, 0);
225 int perftestrun(int nthreads
, int nreaders
, int nupdaters
)
230 while (uatomic_read(&nthreadsrunning
) < nthreads
)
231 (void) poll(NULL
, 0, 1);
232 uatomic_set(&goflag
, GOFLAG_RUN
);
234 uatomic_set(&goflag
, GOFLAG_STOP
);
237 n_reads
+= per_thread(n_reads_pt
, t
);
238 n_updates
+= per_thread(n_updates_pt
, t
);
240 diag("n_reads: %lld n_updates: %ld nreaders: %d nupdaters: %d duration: %d",
241 n_reads
, n_updates
, nreaders
, nupdaters
, duration
);
242 diag("ns/read: %g ns/update: %g",
243 ((duration
* 1000*1000*1000.*(double)nreaders
) /
245 ((duration
* 1000*1000*1000.*(double)nupdaters
) /
247 if (get_cpu_call_rcu_data(0)) {
248 diag("Deallocating per-CPU call_rcu threads.\n");
249 free_all_cpu_call_rcu_data();
255 int perftest(int nreaders
, int cpustride
)
261 for (i
= 0; i
< nreaders
; i
++) {
262 arg
= (long)(i
* cpustride
);
263 create_thread(rcu_read_perf_test
, (void *)arg
);
265 arg
= (long)(i
* cpustride
);
266 create_thread(rcu_update_perf_test
, (void *)arg
);
267 return perftestrun(i
+ 1, nreaders
, 1);
271 int rperftest(int nreaders
, int cpustride
)
277 init_per_thread(n_reads_pt
, 0LL);
278 for (i
= 0; i
< nreaders
; i
++) {
279 arg
= (long)(i
* cpustride
);
280 create_thread(rcu_read_perf_test
, (void *)arg
);
282 return perftestrun(i
, nreaders
, 0);
286 int uperftest(int nupdaters
, int cpustride
)
292 init_per_thread(n_reads_pt
, 0LL);
293 for (i
= 0; i
< nupdaters
; i
++) {
294 arg
= (long)(i
* cpustride
);
295 create_thread(rcu_update_perf_test
, (void *)arg
);
297 return perftestrun(i
, 0, nupdaters
);
304 #define RCU_STRESS_PIPE_LEN 10
311 struct rcu_stress rcu_stress_array
[RCU_STRESS_PIPE_LEN
] = { { 0, 0 } };
312 struct rcu_stress
*rcu_stress_current
;
313 int rcu_stress_idx
= 0;
316 * How many time a reader has seen something that should not be visible. It is
317 * an error if this value is different than zero at the end of the stress test.
319 * Here, the something that should not be visibile is an old pipe that has been
320 * freed (mbtest = 0).
323 DEFINE_PER_THREAD(long long [RCU_STRESS_PIPE_LEN
+ 1], rcu_stress_count
);
328 void *rcu_read_stress_test(void *arg
__attribute__((unused
)))
332 struct rcu_stress
*p
;
335 rcu_register_thread();
336 put_thread_offline();
337 while (uatomic_read(&goflag
) == GOFLAG_INIT
)
338 (void) poll(NULL
, 0, 1);
340 while (uatomic_read(&goflag
) == GOFLAG_RUN
) {
342 p
= rcu_dereference(rcu_stress_current
);
344 uatomic_inc_mo(&n_mberror
, CMM_RELAXED
);
345 rcu_read_lock_nest();
347 * The value of garbage is nothing important. This is
348 * essentially a busy loop. The atomic operation -- while not
349 * important here -- helps tools such as TSAN to not flag this
350 * as a race condition.
352 for (i
= 0; i
< 100; i
++)
353 uatomic_inc(&garbage
);
354 rcu_read_unlock_nest();
355 pc
= uatomic_read(&p
->pipe_count
);
357 if ((pc
> RCU_STRESS_PIPE_LEN
) || (pc
< 0))
358 pc
= RCU_STRESS_PIPE_LEN
;
359 __get_thread_var(rcu_stress_count
)[pc
]++;
360 __get_thread_var(n_reads_pt
)++;
361 mark_rcu_quiescent_state();
362 if ((++itercnt
% 0x1000) == 0) {
363 put_thread_offline();
364 put_thread_online_delay();
368 put_thread_offline();
369 rcu_unregister_thread();
374 static DEFINE_URCU_WAIT_QUEUE(call_rcu_waiters
);
377 void rcu_update_stress_test_rcu(struct rcu_head
*head
__attribute__((unused
)))
379 struct urcu_waiters waiters
;
381 urcu_move_waiters(&waiters
, &call_rcu_waiters
);
382 urcu_wake_all_waiters(&waiters
);
386 void advance_writer_state(enum writer_state
*state
)
389 case WRITER_STATE_SYNC_RCU
:
390 *state
= WRITER_STATE_CALL_RCU
;
392 case WRITER_STATE_CALL_RCU
:
393 *state
= WRITER_STATE_POLL_RCU
;
395 case WRITER_STATE_POLL_RCU
:
396 *state
= WRITER_STATE_SYNC_RCU
;
402 void *rcu_update_stress_test(void *arg
__attribute__((unused
)))
405 struct rcu_stress
*p
, *old_p
;
407 enum writer_state writer_state
= WRITER_STATE_SYNC_RCU
;
409 rcu_register_thread();
411 /* Offline for poll. */
412 put_thread_offline();
413 while (uatomic_read(&goflag
) == GOFLAG_INIT
)
414 (void) poll(NULL
, 0, 1);
418 while (uatomic_read(&goflag
) == GOFLAG_RUN
) {
419 i
= rcu_stress_idx
+ 1;
420 if (i
>= RCU_STRESS_PIPE_LEN
)
424 old_p
= rcu_dereference(rcu_stress_current
);
428 * Allocate a new pipe.
430 p
= &rcu_stress_array
[i
];
434 rcu_assign_pointer(rcu_stress_current
, p
);
438 * Increment every pipe except the freshly allocated one. A
439 * reader should only see either the old pipe or the new
440 * pipe. This is reflected in the rcu_stress_count histogram.
442 for (i
= 0; i
< RCU_STRESS_PIPE_LEN
; i
++)
443 if (i
!= rcu_stress_idx
)
444 uatomic_inc(&rcu_stress_array
[i
].pipe_count
);
446 switch (writer_state
) {
447 case WRITER_STATE_SYNC_RCU
:
450 case WRITER_STATE_CALL_RCU
:
452 DEFINE_URCU_WAIT_NODE(wait
, URCU_WAIT_WAITING
);
454 urcu_wait_add(&call_rcu_waiters
, &wait
);
456 call_rcu(&rh
, rcu_update_stress_test_rcu
);
458 /* Offline for busy-wait. */
459 put_thread_offline();
460 urcu_adaptative_busy_wait(&wait
);
464 case WRITER_STATE_POLL_RCU
:
466 struct urcu_gp_poll_state poll_state
;
468 poll_state
= start_poll_synchronize_rcu();
470 /* Offline for poll. */
471 put_thread_offline();
472 while (!poll_state_synchronize_rcu(poll_state
))
473 (void) poll(NULL
, 0, 1); /* Wait for 1ms */
479 * No readers should see that old pipe now. Setting mbtest to 0
480 * to mark it as "freed".
487 advance_writer_state(&writer_state
);
490 rcu_unregister_thread();
496 void *rcu_fake_update_stress_test(void *arg
__attribute__((unused
)))
498 if (callrcu_type
== CALLRCU_PERTHREAD
) {
499 struct call_rcu_data
*crdp
;
501 crdp
= create_call_rcu_data(0, -1);
503 diag("Successfully using per-thread call_rcu() worker.");
504 set_thread_call_rcu_data(crdp
);
507 while (uatomic_read(&goflag
) == GOFLAG_INIT
)
508 (void) poll(NULL
, 0, 1);
509 while (uatomic_read(&goflag
) == GOFLAG_RUN
) {
511 (void) poll(NULL
, 0, 1);
513 if (callrcu_type
== CALLRCU_PERTHREAD
) {
514 struct call_rcu_data
*crdp
;
516 crdp
= get_thread_call_rcu_data();
517 set_thread_call_rcu_data(NULL
);
518 call_rcu_data_free(crdp
);
524 int stresstest(int nreaders
)
532 init_per_thread(n_reads_pt
, 0LL);
534 p
= &per_thread(rcu_stress_count
,t
)[0];
535 for (i
= 0; i
<= RCU_STRESS_PIPE_LEN
; i
++)
538 rcu_stress_current
= &rcu_stress_array
[0];
539 rcu_stress_current
->pipe_count
= 0;
540 rcu_stress_current
->mbtest
= 1;
541 for (i
= 0; i
< nreaders
; i
++)
542 create_thread(rcu_read_stress_test
, NULL
);
543 create_thread(rcu_update_stress_test
, NULL
);
544 for (i
= 0; i
< 5; i
++)
545 create_thread(rcu_fake_update_stress_test
, NULL
);
546 uatomic_set(&goflag
, GOFLAG_RUN
);
548 uatomic_set(&goflag
, GOFLAG_STOP
);
551 n_reads
+= per_thread(n_reads_pt
, t
);
552 diag("n_reads: %lld n_updates: %ld n_mberror: %d",
553 n_reads
, n_updates
, n_mberror
);
555 rdiag("rcu_stress_count:");
557 for (i
= 0; i
<= RCU_STRESS_PIPE_LEN
; i
++) {
560 sum
+= per_thread(rcu_stress_count
, t
)[i
];
563 * If any entries past the first two are non-zero, RCU is
564 * broken. See details above about rcu_stress_count.
566 if (i
> 1 && sum
!= 0) {
572 if (get_cpu_call_rcu_data(0)) {
573 diag("Deallocating per-CPU call_rcu threads.");
574 free_all_cpu_call_rcu_data();
586 void usage(char *argv
[]) __attribute__((__noreturn__
));
589 void usage(char *argv
[])
591 diag("Usage: %s nreaders [ perf | rperf | uperf | stress ] [ stride ] [ callrcu_global | callrcu_percpu | callrcu_perthread ]\n", argv
[0]);
595 int main(int argc
, char *argv
[])
600 plan_tests(NR_TESTS
);
605 const char *callrcu_str
= argv
[4];;
607 if (strcmp(callrcu_str
, "callrcu_global") == 0) {
608 callrcu_type
= CALLRCU_GLOBAL
;
609 } else if (strcmp(callrcu_str
, "callrcu_percpu") == 0) {
610 callrcu_type
= CALLRCU_PERCPU
;
611 } else if (strcmp(callrcu_str
, "callrcu_perthread") == 0) {
612 callrcu_type
= CALLRCU_PERTHREAD
;
619 switch (callrcu_type
) {
621 diag("Using global per-process call_rcu thread.");
624 diag("Using per-CPU call_rcu threads.");
625 if (create_all_cpu_call_rcu_data(0))
626 diag("create_all_cpu_call_rcu_data: %s",
629 case CALLRCU_PERTHREAD
:
630 diag("Using per-thread call_rcu() worker.");
637 yield_active
|= YIELD_READ
;
638 yield_active
|= YIELD_WRITE
;
642 if (strcmp(argv
[1], "-h") == 0
643 || strcmp(argv
[1], "--help") == 0) {
647 nreaders
= strtoul(argv
[1], NULL
, 0);
649 ok(!perftest(nreaders
, cpustride
),
650 "perftest readers: %d, stride: %d",
651 nreaders
, cpustride
);
655 cpustride
= strtoul(argv
[3], NULL
, 0);
656 if (strcmp(argv
[2], "perf") == 0)
657 ok(!perftest(nreaders
, cpustride
),
658 "perftest readers: %d, stride: %d",
659 nreaders
, cpustride
);
660 else if (strcmp(argv
[2], "rperf") == 0)
661 ok(!rperftest(nreaders
, cpustride
),
662 "rperftest readers: %d, stride: %d",
663 nreaders
, cpustride
);
664 else if (strcmp(argv
[2], "uperf") == 0)
665 ok(!uperftest(nreaders
, cpustride
),
666 "uperftest readers: %d, stride: %d",
667 nreaders
, cpustride
);
668 else if (strcmp(argv
[2], "stress") == 0)
669 ok(!stresstest(nreaders
),
670 "stresstest readers: %d, stride: %d",
671 nreaders
, cpustride
);
678 return exit_status();