2 * rcutorture.h: simple user-level performance/stress test of RCU.
5 * ./rcu <nreaders> rperf [ <cpustride> ]
6 * Run a read-side performance test with the specified
7 * number of readers spaced by <cpustride>.
8 * Thus "./rcu 16 rperf 2" would run 16 readers on even-numbered
10 * ./rcu <nupdaters> uperf [ <cpustride> ]
11 * Run an update-side performance test with the specified
12 * number of updaters and specified CPU spacing.
13 * ./rcu <nreaders> perf [ <cpustride> ]
14 * Run a combined read/update performance test with the specified
15 * number of readers and one updater and specified CPU spacing.
16 * The readers run on the low-numbered CPUs and the updater
17 * of the highest-numbered CPU.
19 * The above tests produce output as follows:
21 * n_reads: 46008000 n_updates: 146026 nreaders: 2 nupdaters: 1 duration: 1
22 * ns/read: 43.4707 ns/update: 6848.1
24 * The first line lists the total number of RCU reads and updates executed
25 * during the test, the number of reader threads, the number of updater
26 * threads, and the duration of the test in seconds. The second line
27 * lists the average duration of each type of operation in nanoseconds,
28 * or "nan" if the corresponding type of operation was not performed.
30 * ./rcu <nreaders> stress
31 * Run a stress test with the specified number of readers and
32 * one updater. None of the threads are affinitied to any
35 * This test produces output as follows:
37 * n_reads: 114633217 n_updates: 3903415 n_mberror: 0
38 * rcu_stress_count: 114618391 14826 0 0 0 0 0 0 0 0 0
40 * The first line lists the number of RCU read and update operations
41 * executed, followed by the number of memory-ordering violations
42 * (which will be zero in a correct RCU implementation). The second
43 * line lists the number of readers observing progressively more stale
44 * data. A correct RCU implementation will have all but the first two
47 * This program is free software; you can redistribute it and/or modify
48 * it under the terms of the GNU General Public License as published by
49 * the Free Software Foundation; either version 2 of the License, or
50 * (at your option) any later version.
52 * This program is distributed in the hope that it will be useful,
53 * but WITHOUT ANY WARRANTY; without even the implied warranty of
54 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
55 * GNU General Public License for more details.
57 * You should have received a copy of the GNU General Public License
58 * along with this program; if not, write to the Free Software
59 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
61 * Copyright (c) 2008 Paul E. McKenney, IBM Corporation.
69 #include "../urcu-call-rcu.h"
71 DEFINE_PER_THREAD(long long, n_reads_pt
);
72 DEFINE_PER_THREAD(long long, n_updates_pt
);
74 long long n_reads
= 0LL;
83 int goflag
__attribute__((__aligned__(CAA_CACHE_LINE_SIZE
))) = GOFLAG_INIT
;
85 #define RCU_READ_RUN 1000
88 #define RCU_READ_NESTABLE
90 #ifdef RCU_READ_NESTABLE
91 #define rcu_read_lock_nest() rcu_read_lock()
92 #define rcu_read_unlock_nest() rcu_read_unlock()
93 #else /* #ifdef RCU_READ_NESTABLE */
94 #define rcu_read_lock_nest()
95 #define rcu_read_unlock_nest()
96 #endif /* #else #ifdef RCU_READ_NESTABLE */
99 #define mark_rcu_quiescent_state rcu_quiescent_state
100 #define put_thread_offline rcu_thread_offline
101 #define put_thread_online rcu_thread_online
104 #ifndef mark_rcu_quiescent_state
105 #define mark_rcu_quiescent_state() do ; while (0)
106 #endif /* #ifdef mark_rcu_quiescent_state */
108 #ifndef put_thread_offline
109 #define put_thread_offline() do ; while (0)
110 #define put_thread_online() do ; while (0)
111 #define put_thread_online_delay() do ; while (0)
112 #else /* #ifndef put_thread_offline */
113 #define put_thread_online_delay() synchronize_rcu()
114 #endif /* #else #ifndef put_thread_offline */
120 void *rcu_read_perf_test(void *arg
)
124 long long n_reads_local
= 0;
126 rcu_register_thread();
128 uatomic_inc(&nthreadsrunning
);
129 while (goflag
== GOFLAG_INIT
)
131 mark_rcu_quiescent_state();
132 while (goflag
== GOFLAG_RUN
) {
133 for (i
= 0; i
< RCU_READ_RUN
; i
++) {
135 /* rcu_read_lock_nest(); */
136 /* rcu_read_unlock_nest(); */
139 n_reads_local
+= RCU_READ_RUN
;
140 mark_rcu_quiescent_state();
142 __get_thread_var(n_reads_pt
) += n_reads_local
;
143 put_thread_offline();
144 rcu_unregister_thread();
149 void *rcu_update_perf_test(void *arg
)
151 long long n_updates_local
= 0;
153 if ((random() & 0xf00) == 0) {
154 struct call_rcu_data
*crdp
;
156 crdp
= create_call_rcu_data(0);
159 "Using per-thread call_rcu() worker.\n");
160 set_thread_call_rcu_data(crdp
);
163 uatomic_inc(&nthreadsrunning
);
164 while (goflag
== GOFLAG_INIT
)
166 while (goflag
== GOFLAG_RUN
) {
170 __get_thread_var(n_updates_pt
) += n_updates_local
;
174 void perftestinit(void)
176 init_per_thread(n_reads_pt
, 0LL);
177 init_per_thread(n_updates_pt
, 0LL);
178 uatomic_set(&nthreadsrunning
, 0);
181 void perftestrun(int nthreads
, int nreaders
, int nupdaters
)
187 while (uatomic_read(&nthreadsrunning
) < nthreads
)
193 goflag
= GOFLAG_STOP
;
197 n_reads
+= per_thread(n_reads_pt
, t
);
198 n_updates
+= per_thread(n_updates_pt
, t
);
200 printf("n_reads: %lld n_updates: %ld nreaders: %d nupdaters: %d duration: %d\n",
201 n_reads
, n_updates
, nreaders
, nupdaters
, duration
);
202 printf("ns/read: %g ns/update: %g\n",
203 ((duration
* 1000*1000*1000.*(double)nreaders
) /
205 ((duration
* 1000*1000*1000.*(double)nupdaters
) /
210 void perftest(int nreaders
, int cpustride
)
216 for (i
= 0; i
< nreaders
; i
++) {
217 arg
= (long)(i
* cpustride
);
218 create_thread(rcu_read_perf_test
, (void *)arg
);
220 arg
= (long)(i
* cpustride
);
221 create_thread(rcu_update_perf_test
, (void *)arg
);
222 perftestrun(i
+ 1, nreaders
, 1);
225 void rperftest(int nreaders
, int cpustride
)
231 init_per_thread(n_reads_pt
, 0LL);
232 for (i
= 0; i
< nreaders
; i
++) {
233 arg
= (long)(i
* cpustride
);
234 create_thread(rcu_read_perf_test
, (void *)arg
);
236 perftestrun(i
, nreaders
, 0);
239 void uperftest(int nupdaters
, int cpustride
)
245 init_per_thread(n_reads_pt
, 0LL);
246 for (i
= 0; i
< nupdaters
; i
++) {
247 arg
= (long)(i
* cpustride
);
248 create_thread(rcu_update_perf_test
, (void *)arg
);
250 perftestrun(i
, 0, nupdaters
);
257 #define RCU_STRESS_PIPE_LEN 10
264 struct rcu_stress rcu_stress_array
[RCU_STRESS_PIPE_LEN
] = { { 0 } };
265 struct rcu_stress
*rcu_stress_current
;
266 int rcu_stress_idx
= 0;
269 DEFINE_PER_THREAD(long long [RCU_STRESS_PIPE_LEN
+ 1], rcu_stress_count
);
273 void *rcu_read_stress_test(void *arg
)
277 struct rcu_stress
*p
;
280 rcu_register_thread();
281 while (goflag
== GOFLAG_INIT
)
283 mark_rcu_quiescent_state();
284 while (goflag
== GOFLAG_RUN
) {
286 p
= rcu_dereference(rcu_stress_current
);
289 rcu_read_lock_nest();
290 for (i
= 0; i
< 100; i
++)
292 rcu_read_unlock_nest();
295 if ((pc
> RCU_STRESS_PIPE_LEN
) || (pc
< 0))
296 pc
= RCU_STRESS_PIPE_LEN
;
297 __get_thread_var(rcu_stress_count
)[pc
]++;
298 __get_thread_var(n_reads_pt
)++;
299 mark_rcu_quiescent_state();
300 if ((++itercnt
% 0x1000) == 0) {
301 put_thread_offline();
302 put_thread_online_delay();
306 put_thread_offline();
307 rcu_unregister_thread();
312 static pthread_mutex_t call_rcu_test_mutex
= PTHREAD_MUTEX_INITIALIZER
;
313 static pthread_cond_t call_rcu_test_cond
= PTHREAD_COND_INITIALIZER
;
315 void rcu_update_stress_test_rcu(struct rcu_head
*head
)
317 if (pthread_mutex_lock(&call_rcu_test_mutex
) != 0) {
318 perror("pthread_mutex_lock");
321 if (pthread_cond_signal(&call_rcu_test_cond
) != 0) {
322 perror("pthread_cond_signal");
325 if (pthread_mutex_unlock(&call_rcu_test_mutex
) != 0) {
326 perror("pthread_mutex_unlock");
331 void *rcu_update_stress_test(void *arg
)
334 struct rcu_stress
*p
;
337 while (goflag
== GOFLAG_INIT
)
339 while (goflag
== GOFLAG_RUN
) {
340 i
= rcu_stress_idx
+ 1;
341 if (i
>= RCU_STRESS_PIPE_LEN
)
343 p
= &rcu_stress_array
[i
];
348 rcu_assign_pointer(rcu_stress_current
, p
);
350 for (i
= 0; i
< RCU_STRESS_PIPE_LEN
; i
++)
351 if (i
!= rcu_stress_idx
)
352 rcu_stress_array
[i
].pipe_count
++;
356 if (pthread_mutex_lock(&call_rcu_test_mutex
) != 0) {
357 perror("pthread_mutex_lock");
360 call_rcu(&rh
, rcu_update_stress_test_rcu
);
361 if (pthread_cond_wait(&call_rcu_test_cond
,
362 &call_rcu_test_mutex
) != 0) {
363 perror("pthread_cond_wait");
366 if (pthread_mutex_unlock(&call_rcu_test_mutex
) != 0) {
367 perror("pthread_mutex_unlock");
376 void *rcu_fake_update_stress_test(void *arg
)
378 if ((random() & 0xf00) == 0) {
379 struct call_rcu_data
*crdp
;
381 crdp
= create_call_rcu_data(0);
384 "Using per-thread call_rcu() worker.\n");
385 set_thread_call_rcu_data(crdp
);
388 while (goflag
== GOFLAG_INIT
)
390 while (goflag
== GOFLAG_RUN
) {
397 void stresstest(int nreaders
)
404 init_per_thread(n_reads_pt
, 0LL);
406 p
= &per_thread(rcu_stress_count
,t
)[0];
407 for (i
= 0; i
<= RCU_STRESS_PIPE_LEN
; i
++)
410 rcu_stress_current
= &rcu_stress_array
[0];
411 rcu_stress_current
->pipe_count
= 0;
412 rcu_stress_current
->mbtest
= 1;
413 for (i
= 0; i
< nreaders
; i
++)
414 create_thread(rcu_read_stress_test
, NULL
);
415 create_thread(rcu_update_stress_test
, NULL
);
416 for (i
= 0; i
< 5; i
++)
417 create_thread(rcu_fake_update_stress_test
, NULL
);
423 goflag
= GOFLAG_STOP
;
427 n_reads
+= per_thread(n_reads_pt
, t
);
428 printf("n_reads: %lld n_updates: %ld n_mberror: %d\n",
429 n_reads
, n_updates
, n_mberror
);
430 printf("rcu_stress_count:");
431 for (i
= 0; i
<= RCU_STRESS_PIPE_LEN
; i
++) {
434 sum
+= per_thread(rcu_stress_count
, t
)[i
];
436 printf(" %lld", sum
);
446 void usage(int argc
, char *argv
[])
448 fprintf(stderr
, "Usage: %s [nreaders [ perf | stress ] ]\n", argv
[0]);
452 int main(int argc
, char *argv
[])
460 if (random() & 0x100) {
461 fprintf(stderr
, "Allocating per-CPU call_rcu threads.\n");
462 if (create_all_cpu_call_rcu_data(0))
463 perror("create_all_cpu_call_rcu_data");
467 yield_active
|= YIELD_READ
;
468 yield_active
|= YIELD_WRITE
;
472 nreaders
= strtoul(argv
[1], NULL
, 0);
474 perftest(nreaders
, cpustride
);
476 cpustride
= strtoul(argv
[3], NULL
, 0);
477 if (strcmp(argv
[2], "perf") == 0)
478 perftest(nreaders
, cpustride
);
479 else if (strcmp(argv
[2], "rperf") == 0)
480 rperftest(nreaders
, cpustride
);
481 else if (strcmp(argv
[2], "uperf") == 0)
482 uperftest(nreaders
, cpustride
);
483 else if (strcmp(argv
[2], "stress") == 0)
484 stresstest(nreaders
);
487 perftest(nreaders
, cpustride
);