From ad46005890368f9c306f0c510b3d4b08c47b66f8 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Mon, 28 Sep 2015 19:27:02 -0400 Subject: [PATCH] Refactor tests - Migrate benchmarks and regression tests to tap, - Replace the "bench" make target by "short_bench" and "long_bench". The short benchmark is 3 seconds per test, and the long one is 30 seconds per test, - make regtest now invokes the benchmarks with only 1 second per benchmark. - Now use "nproc" command to detect the number of available CPUs rather than hardcoding a value. - rcutorture in "stress" mode is now executed. Signed-off-by: Mathieu Desnoyers --- Makefile.am | 8 +- tests/Makefile.am | 9 +- tests/benchmark/Makefile.am | 14 +- tests/benchmark/long_bench_tests | 2 + tests/benchmark/regression_tests | 2 + .../{runall.sh => run-urcu-tests.sh} | 55 +++---- tests/benchmark/run.sh | 25 +++ tests/benchmark/runhash.sh | 77 +++++---- tests/benchmark/runtests-batch.sh | 26 ++- tests/benchmark/runtests.sh | 24 ++- tests/benchmark/short_bench_tests | 2 + tests/regression/Makefile.am | 39 +++-- tests/regression/rcutorture.h | 153 ++++++++++++------ tests/regression/regression_tests | 21 +++ tests/regression/run.sh | 25 +++ tests/regression/test_urcu_fork.c | 63 ++++++-- tests/unit/run.sh | 7 +- tests/utils/tap.c | 41 +++++ tests/utils/tap.h | 6 + 19 files changed, 428 insertions(+), 171 deletions(-) create mode 100644 tests/benchmark/long_bench_tests create mode 100644 tests/benchmark/regression_tests rename tests/benchmark/{runall.sh => run-urcu-tests.sh} (56%) create mode 100755 tests/benchmark/run.sh create mode 100644 tests/benchmark/short_bench_tests create mode 100644 tests/regression/regression_tests create mode 100755 tests/regression/run.sh diff --git a/Makefile.am b/Makefile.am index eef3947..19a44ea 100644 --- a/Makefile.am +++ b/Makefile.am @@ -86,8 +86,10 @@ dist_doc_DATA = README.md ChangeLog dist_noinst_DATA = CodingStyle -.PHONY: bench regtest -bench: - cd tests && $(MAKE) $(AM_MAKEFLAGS) bench +.PHONY: short_bench long_bench regtest +short_bench: + cd tests && $(MAKE) $(AM_MAKEFLAGS) short_bench +long_bench: + cd tests && $(MAKE) $(AM_MAKEFLAGS) long_bench regtest: cd tests && $(MAKE) $(AM_MAKEFLAGS) regtest diff --git a/tests/Makefile.am b/tests/Makefile.am index a8aecac..03f3f29 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -1,8 +1,11 @@ SUBDIRS = utils common unit benchmark regression -.PHONY: bench regtest +.PHONY: short_bench long_bench regtest -bench: - cd benchmark && $(MAKE) $(AM_MAKEFLAGS) bench +short_bench: + cd benchmark && $(MAKE) $(AM_MAKEFLAGS) short_bench +long_bench: + cd benchmark && $(MAKE) $(AM_MAKEFLAGS) long_bench regtest: cd regression && $(MAKE) $(AM_MAKEFLAGS) regtest + cd benchmark && $(MAKE) $(AM_MAKEFLAGS) regtest diff --git a/tests/benchmark/Makefile.am b/tests/benchmark/Makefile.am index e25dd8f..1c3e443 100644 --- a/tests/benchmark/Makefile.am +++ b/tests/benchmark/Makefile.am @@ -4,7 +4,7 @@ endif AM_CFLAGS=-I$(top_srcdir) -I$(top_builddir) -I$(top_srcdir)/tests/common -g SCRIPT_LIST = common.sh \ - runall.sh \ + run-urcu-tests.sh \ runhash.sh \ runtests.sh \ runpaul-phase1.sh \ @@ -222,7 +222,13 @@ clean-local: done; \ fi -.PHONY: bench +.PHONY: short_bench long_bench regtest -bench: - ./runall.sh +short_bench: + ./run.sh short_bench_tests + +long_bench: + ./run.sh long_bench_tests + +regtest: + ./run.sh regression_tests diff --git a/tests/benchmark/long_bench_tests b/tests/benchmark/long_bench_tests new file mode 100644 index 0000000..244f3c4 --- /dev/null +++ b/tests/benchmark/long_bench_tests @@ -0,0 +1,2 @@ +./run-urcu-tests.sh 30 +./runhash.sh 30 diff --git a/tests/benchmark/regression_tests b/tests/benchmark/regression_tests new file mode 100644 index 0000000..436c330 --- /dev/null +++ b/tests/benchmark/regression_tests @@ -0,0 +1,2 @@ +./run-urcu-tests.sh 1 +./runhash.sh 1 diff --git a/tests/benchmark/runall.sh b/tests/benchmark/run-urcu-tests.sh similarity index 56% rename from tests/benchmark/runall.sh rename to tests/benchmark/run-urcu-tests.sh index ced1765..6e6cf60 100755 --- a/tests/benchmark/runall.sh +++ b/tests/benchmark/run-urcu-tests.sh @@ -1,9 +1,22 @@ -#!/bin/sh +#!/bin/bash + +source ../utils/tap.sh + +NUM_TESTS=103 + +plan_tests ${NUM_TESTS} #run all tests +diag "Executing URCU tests" #set to number of active CPUS -NUM_CPUS=8 +NUM_CPUS=$(nproc) +if [[ ${NUM_CPUS} -lt 4 ]]; then + NUM_CPUS=4 # Floor at 4 due to following assumptions. +fi + +#first parameter: seconds per test +DURATION=$1 #extra options, e.g. for setting affinity on even CPUs : #EXTRA_OPTS=$(for a in $(seq 0 2 127); do echo -n "-a ${a} "; done) @@ -24,42 +37,31 @@ NUM_CPUS=8 #fix number of readers and reader C.S. length, vary delay between updates #y: ops/s -rm -f runall.log -rm -fr runall.detail.log - -echo Executing batch RCU test +diag "Executing batch RCU test" -DURATION=10 BATCH_ARRAY="1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 131072 262144" NR_WRITERS=$((${NUM_CPUS} / 2)) -rm -f batch-rcu.log - NR_READERS=$((${NUM_CPUS} - ${NR_WRITERS})) for BATCH_SIZE in ${BATCH_ARRAY}; do - echo "./runtests-batch.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d 0 -b ${BATCH_SIZE} ${EXTRA_OPTS} | tee -a batch-rcu.log" >> runall.log - (./runtests-batch.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d 0 -b ${BATCH_SIZE} ${EXTRA_OPTS} | tee -a batch-rcu.log) || exit 1 + okx ./runtests-batch.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d 0 -b ${BATCH_SIZE} ${EXTRA_OPTS} done #setting gc each 32768. ** UPDATE FOR YOUR ARCHITECTURE BASED ON TEST ABOVE ** EXTRA_OPTS="${EXTRA_OPTS} -b 32768" -echo Executing update fraction test +diag "Executing update fraction test" -DURATION=10 WDELAY_ARRAY="0 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 131072 262144 524288 1048576 2097152 4194304 8388608 16777216 33554432 67108864 134217728" NR_WRITERS=$((${NUM_CPUS} / 2)) -rm -f update-fraction.log - NR_READERS=$((${NUM_CPUS} - ${NR_WRITERS})) for WDELAY in ${WDELAY_ARRAY}; do - echo "./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d ${WDELAY} ${EXTRA_OPTS} | tee -a update-fraction.log" >> runall.log - (./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d ${WDELAY} ${EXTRA_OPTS} | tee -a update-fraction.log) || exit 1 + okx ./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} -d ${WDELAY} ${EXTRA_OPTS} done #Test scalability : @@ -67,16 +69,12 @@ done # y: ops/s # 0 writer. -echo Executing scalability test +diag "Executing scalability test" NR_WRITERS=0 -DURATION=10 - -rm -f scalability.log for NR_READERS in $(seq 1 ${NUM_CPUS}); do - echo "./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS}| tee -a scalability.log" >> runall.log - (./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS}| tee -a scalability.log) || exit 1 + okx ./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS} done @@ -85,20 +83,13 @@ done # 8 readers # 0 writers -echo Executing reader C.S. length test +diag "Executing reader C.S. length test" NR_READERS=${NUM_CPUS} NR_WRITERS=0 -DURATION=10 #in loops. READERCSLEN_ARRAY="0 1 2 4 8 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 131072 262144 524288 1048576 2097152" -rm -f readercslen.log - for READERCSLEN in ${READERCSLEN_ARRAY}; do - echo "./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS} -c ${READERCSLEN} | tee -a readercslen.log" >> runall.log - (./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS} -c ${READERCSLEN} | tee -a readercslen.log) || exit 1 + okx ./runtests.sh ${NR_READERS} ${NR_WRITERS} ${DURATION} ${EXTRA_OPTS} -c ${READERCSLEN} done - -echo Executing Hash table test -./runhash.sh || exit 1 diff --git a/tests/benchmark/run.sh b/tests/benchmark/run.sh new file mode 100755 index 0000000..d99a493 --- /dev/null +++ b/tests/benchmark/run.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Copyright (C) 2013 - Christian Babeux +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; only version 2 +# of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# + +INPUT=$1 +shift 1 + +[ -z "${INPUT}" ] && echo "Error: No testlist. Please specify a testlist to run." && exit 1 + +prove ${*} --merge --exec '' - < ${INPUT} diff --git a/tests/benchmark/runhash.sh b/tests/benchmark/runhash.sh index 0ba20ef..2ae73e9 100755 --- a/tests/benchmark/runhash.sh +++ b/tests/benchmark/runhash.sh @@ -1,17 +1,30 @@ -#!/bin/sh +#!/bin/bash + +source ../utils/tap.sh + +NUM_TESTS=17 + +plan_tests ${NUM_TESTS} + +diag "Executing Hash table test" # TODO: missing tests: # - send kill signals during tests to change the behavior between # add/remove/random # - validate that "nr_leaked" is always 0 in SUMMARY for all tests -# 30 seconds per test -TIME_UNITS=30 +# 1st parameter: seconds per test +TIME_UNITS=$1 TESTPROG=./test_urcu_hash -#thread multiplier -THREAD_MUL=1 +#thread multiplier: number of processors divided by 4. +NUM_CPUS=$(nproc) +if [[ ${NUM_CPUS} -lt 4 ]]; then + NUM_CPUS=4 # Floor at 4 due to following assumptions. +fi + +THREAD_MUL=$(( ${NUM_CPUS} / 4 )) EXTRA_PARAMS=-v @@ -19,19 +32,19 @@ EXTRA_PARAMS=-v # rw test, single key, replace and del randomly, 4 threads, auto resize. # key range: init, lookup, and update: 0 to 0 -${TESTPROG} 0 $((4*${THREAD_MUL})) ${TIME_UNITS} -A -s -M 1 -N 1 -O 1 ${EXTRA_PARAMS} || exit 1 +okx ${TESTPROG} 0 $((4*${THREAD_MUL})) ${TIME_UNITS} -A -s -M 1 -N 1 -O 1 ${EXTRA_PARAMS} # rw test, single key, add unique and del randomly, 4 threads, auto resize. # key range: init, lookup, and update: 0 to 0 -${TESTPROG} 0 $((4*${THREAD_MUL})) ${TIME_UNITS} -A -u -M 1 -N 1 -O 1 ${EXTRA_PARAMS} || exit 1 +okx ${TESTPROG} 0 $((4*${THREAD_MUL})) ${TIME_UNITS} -A -u -M 1 -N 1 -O 1 ${EXTRA_PARAMS} # rw test, single key, replace and del randomly, 2 lookup threads, 2 update threads, auto resize. # key range: init, lookup, and update: 0 to 0 -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -s -M 1 -N 1 -O 1 ${EXTRA_PARAMS} || exit 1 +okx ${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -s -M 1 -N 1 -O 1 ${EXTRA_PARAMS} # rw test, single key, add and del randomly, 2 lookup threads, 2 update threads, auto resize. # key range: init, lookup, and update: 0 to 0 -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -M 1 -N 1 -O 1 ${EXTRA_PARAMS} || exit 1 +okx ${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -M 1 -N 1 -O 1 ${EXTRA_PARAMS} # ** test updates vs lookups with default table @@ -39,17 +52,17 @@ ${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -M 1 -N 1 # rw test, 2 lookup, 2 update threads, add and del randomly, auto resize. # max 1048576 buckets # key range: init, lookup, and update: 0 to 999999 -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A ${EXTRA_PARAMS} || exit 1 +okx ${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A ${EXTRA_PARAMS} # rw test, 2 lookup, 2 update threads, add_replace and del randomly, auto resize. # max 1048576 buckets # key range: init, lookup, and update: 0 to 999999 -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -s ${EXTRA_PARAMS} || exit 1 +okx ${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -s ${EXTRA_PARAMS} # rw test, 2 lookup, 2 update threads, add_unique and del randomly, auto resize. # max 1048576 buckets # key range: init, lookup, and update: 0 to 999999 -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -u ${EXTRA_PARAMS} || exit 1 +okx ${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -u ${EXTRA_PARAMS} # test memory management backends @@ -58,22 +71,22 @@ ${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -u ${EXTR # max buckets: 1048576 # key range: init, lookup, and update: 0 to 99999999 # mm backend: "order" -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -m 1 -n 1048576 -i \ - -M 100000000 -N 100000000 -O 100000000 -B order ${EXTRA_PARAMS} || exit 1 +okx ${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -m 1 -n 1048576 -i \ + -M 100000000 -N 100000000 -O 100000000 -B order ${EXTRA_PARAMS} # rw test, 2 lookup, 2 update threads, add only, auto resize. # max buckets: 1048576 # key range: init, lookup, and update: 0 to 99999999 # mm backend: "chunk" -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -m 1 -n 1048576 -i \ - -M 100000000 -N 100000000 -O 100000000 -B chunk ${EXTRA_PARAMS} || exit 1 +okx ${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -m 1 -n 1048576 -i \ + -M 100000000 -N 100000000 -O 100000000 -B chunk ${EXTRA_PARAMS} # rw test, 2 lookup, 2 update threads, add only, auto resize. # max buckets: 1048576 # key range: init, lookup, and update: 0 to 99999999 # mm backend: "mmap" -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -m 1 -n 1048576 -i \ - -M 100000000 -N 100000000 -O 100000000 -B mmap ${EXTRA_PARAMS} || exit 1 +okx ${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -m 1 -n 1048576 -i \ + -M 100000000 -N 100000000 -O 100000000 -B mmap ${EXTRA_PARAMS} # ** key range tests @@ -84,28 +97,28 @@ ${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A -m 1 -n 1 # key range: lookup: 1000000 to 1999999 # NOTE: reader threads in this test should never have a successful # lookup. TODO -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ - -R 1000000 ${EXTRA_PARAMS} || exit 1 +okx ${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ + -R 1000000 ${EXTRA_PARAMS} # ** small key range # rw test, 2 lookup, 2 update threads, add and del randomly, auto resize. # max 1048576 buckets # key range: init, update, and lookups: 0 to 9 -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ - -M 10 -N 10 -O 10 ${EXTRA_PARAMS} || exit 1 +okx ${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ + -M 10 -N 10 -O 10 ${EXTRA_PARAMS} # rw test, 2 lookup, 2 update threads, add_unique and del randomly, auto resize. # max 1048576 buckets # key range: init, update, and lookups: 0 to 9 -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ - -M 10 -N 10 -O 10 -u ${EXTRA_PARAMS} || exit 1 +okx ${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ + -M 10 -N 10 -O 10 -u ${EXTRA_PARAMS} # rw test, 2 lookup, 2 update threads, add_replace and del randomly, auto resize. # max 1048576 buckets # key range: init, update, and lookups: 0 to 9 -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ - -M 10 -N 10 -O 10 -s ${EXTRA_PARAMS} || exit 1 +okx ${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ + -M 10 -N 10 -O 10 -s ${EXTRA_PARAMS} # ** lookup for known keys @@ -116,8 +129,8 @@ ${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ # key range: updates: 10 to 19 # NOTE: reader threads in this test should always have successful # lookups. TODO -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ - -M 10 -N 10 -O 10 -R 0 -T 0 -S 10 -k 10 -s ${EXTRA_PARAMS} || exit 1 +okx ${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ + -M 10 -N 10 -O 10 -R 0 -T 0 -S 10 -k 10 -s ${EXTRA_PARAMS} # ** Uniqueness test @@ -125,12 +138,12 @@ ${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ # max 1048576 buckets # asserts that no duplicates are observed by reader threads # standard length hash chains -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ - -U ${EXTRA_PARAMS} || exit 1 +okx ${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ + -U ${EXTRA_PARAMS} # rw test, 2 lookup, 2 update threads, add_unique, add_replace and del randomly, auto resize. # max 1048576 buckets # asserts that no duplicates are observed by reader threads # create long hash chains: using modulo 4 on keys as hash -${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ - -U -C 4 ${EXTRA_PARAMS} || exit 1 +okx ${TESTPROG} $((2*${THREAD_MUL})) $((2*${THREAD_MUL})) ${TIME_UNITS} -A \ + -U -C 4 ${EXTRA_PARAMS} diff --git a/tests/benchmark/runtests-batch.sh b/tests/benchmark/runtests-batch.sh index 3d295fb..0e5577b 100755 --- a/tests/benchmark/runtests-batch.sh +++ b/tests/benchmark/runtests-batch.sh @@ -1,8 +1,23 @@ -#!/bin/sh +#!/bin/bash + +source ../utils/tap.sh + +NUM_TESTS=1 + +plan_tests ${NUM_TESTS} . ./common.sh -log_file="runall.detail.log" +function cleanup() +{ + if [ x"$tmpfile" != x"" ]; then + rm -f $tmpfile + fi +} + +tmpfile= +trap cleanup SIGINT SIGTERM EXIT +tmpfile=$(mktemp) # Check if time bin is non-empty if [ -n "$test_time_bin" ]; then @@ -11,9 +26,10 @@ else time_command="" fi +tmpfile=$(mktemp) + #for a in test_urcu_gc test_urcu_gc_mb test_urcu_qsbr_gc; do for a in test_urcu_gc; do - echo "./${a} $*" | tee -a "$log_file" - $time_command ./${a} $* 2>> $log_file + okx $time_command -o $tmpfile ./${a} $* + diag "time: $(cat $tmpfile)" done - diff --git a/tests/benchmark/runtests.sh b/tests/benchmark/runtests.sh index 38e798c..781e8f7 100755 --- a/tests/benchmark/runtests.sh +++ b/tests/benchmark/runtests.sh @@ -1,8 +1,23 @@ -#!/bin/sh +#!/bin/bash + +source ../utils/tap.sh + +NUM_TESTS=15 + +plan_tests ${NUM_TESTS} . ./common.sh -log_file="runall.detail.log" +function cleanup() +{ + if [ x"$tmpfile" != x"" ]; then + rm -f $tmpfile + fi +} + +tmpfile= +trap cleanup SIGINT SIGTERM EXIT +tmpfile=$(mktemp) # Check if time bin is non-empty if [ -n "$test_time_bin" ]; then @@ -15,7 +30,6 @@ for a in test_urcu_gc test_urcu_signal_gc test_urcu_mb_gc test_urcu_qsbr_gc \ test_urcu_lgc test_urcu_signal_lgc test_urcu_mb_lgc test_urcu_qsbr_lgc \ test_urcu test_urcu_signal test_urcu_mb test_urcu_qsbr \ test_rwlock test_perthreadlock test_mutex; do - echo "./${a} $*" | tee -a "$log_file" - $time_command ./${a} $* 2>> $log_file + okx $time_command -o $tmpfile ./${a} $* + diag "time: $(cat $tmpfile)" done - diff --git a/tests/benchmark/short_bench_tests b/tests/benchmark/short_bench_tests new file mode 100644 index 0000000..9d01067 --- /dev/null +++ b/tests/benchmark/short_bench_tests @@ -0,0 +1,2 @@ +./run-urcu-tests.sh 3 +./runhash.sh 3 diff --git a/tests/regression/Makefile.am b/tests/regression/Makefile.am index 8dfe542..05c0375 100644 --- a/tests/regression/Makefile.am +++ b/tests/regression/Makefile.am @@ -1,7 +1,7 @@ if !LIBC_INCLUDES_PTHREAD AM_LDFLAGS=-lpthread endif -AM_CFLAGS=-I$(top_srcdir) -I$(top_builddir) -I$(top_srcdir)/tests/common -g +AM_CFLAGS=-I$(top_srcdir) -I$(top_builddir) -I$(top_srcdir)/tests/utils -I$(top_srcdir)/tests/common -g noinst_PROGRAMS = test_urcu_fork \ rcutorture_urcu \ @@ -19,41 +19,48 @@ URCU_MB_LIB=$(top_builddir)/liburcu-mb.la URCU_SIGNAL_LIB=$(top_builddir)/liburcu-signal.la URCU_BP_LIB=$(top_builddir)/liburcu-bp.la URCU_CDS_LIB=$(top_builddir)/liburcu-cds.la +TAP_LIB=$(top_builddir)/tests/utils/libtap.a test_urcu_fork_SOURCES = test_urcu_fork.c -test_urcu_fork_LDADD = $(URCU_LIB) +test_urcu_fork_LDADD = $(URCU_LIB) $(TAP_LIB) rcutorture_urcu_SOURCES = urcutorture.c rcutorture_urcu_CFLAGS = -DRCU_MEMBARRIER $(AM_CFLAGS) -rcutorture_urcu_LDADD = $(URCU_LIB) +rcutorture_urcu_LDADD = $(URCU_LIB) $(TAP_LIB) rcutorture_urcu_mb_SOURCES = urcutorture.c rcutorture_urcu_mb_CFLAGS = -DRCU_MB $(AM_CFLAGS) -rcutorture_urcu_mb_LDADD = $(URCU_MB_LIB) +rcutorture_urcu_mb_LDADD = $(URCU_MB_LIB) $(TAP_LIB) rcutorture_urcu_qsbr_SOURCES = urcutorture.c rcutorture_urcu_qsbr_CFLAGS = -DTORTURE_QSBR -DRCU_QSBR $(AM_CFLAGS) -rcutorture_urcu_qsbr_LDADD = $(URCU_QSBR_LIB) +rcutorture_urcu_qsbr_LDADD = $(URCU_QSBR_LIB) $(TAP_LIB) rcutorture_urcu_signal_SOURCES = urcutorture.c rcutorture_urcu_signal_CFLAGS = -DRCU_SIGNAL $(AM_CFLAGS) -rcutorture_urcu_signal_LDADD = $(URCU_SIGNAL_LIB) +rcutorture_urcu_signal_LDADD = $(URCU_SIGNAL_LIB) $(TAP_LIB) rcutorture_urcu_bp_SOURCES = urcutorture.c rcutorture_urcu_bp_CFLAGS = -DRCU_BP $(AM_CFLAGS) -rcutorture_urcu_bp_LDADD = $(URCU_BP_LIB) +rcutorture_urcu_bp_LDADD = $(URCU_BP_LIB) $(TAP_LIB) urcutorture.c: ../common/api.h +all-local: + @if [ x"$(srcdir)" != x"$(builddir)" ]; then \ + for script in $(SCRIPT_LIST); do \ + cp -f $(srcdir)/$$script $(builddir); \ + done; \ + fi + +clean-local: + @if [ x"$(srcdir)" != x"$(builddir)" ]; then \ + for script in $(SCRIPT_LIST); do \ + rm -f $(builddir)/$$script; \ + done; \ + fi + .PHONY: regtest -# For now, run the benchmarks too as regression tests. -# TODO: split benchmarks from regression tests regtest: - ./test_urcu_fork - ./rcutorture_urcu - ./rcutorture_urcu_signal - ./rcutorture_urcu_mb - ./rcutorture_urcu_bp - ./rcutorture_urcu_qsbr - cd ../benchmark && ./runall.sh && cd .. + ./run.sh regression_tests diff --git a/tests/regression/rcutorture.h b/tests/regression/rcutorture.h index a2411d7..3444f5b 100644 --- a/tests/regression/rcutorture.h +++ b/tests/regression/rcutorture.h @@ -66,6 +66,9 @@ */ #include +#include "tap.h" + +#define NR_TESTS 1 DEFINE_PER_THREAD(long long, n_reads_pt); DEFINE_PER_THREAD(long long, n_updates_pt); @@ -160,8 +163,7 @@ void *rcu_update_perf_test(void *arg) crdp = create_call_rcu_data(0, -1); if (crdp != NULL) { - fprintf(stderr, - "Using per-thread call_rcu() worker.\n"); + diag("Using per-thread call_rcu() worker."); set_thread_call_rcu_data(crdp); } } @@ -183,7 +185,7 @@ void perftestinit(void) uatomic_set(&nthreadsrunning, 0); } -void perftestrun(int nthreads, int nreaders, int nupdaters) +int perftestrun(int nthreads, int nreaders, int nupdaters) { int t; int duration = 1; @@ -202,21 +204,21 @@ void perftestrun(int nthreads, int nreaders, int nupdaters) n_reads += per_thread(n_reads_pt, t); n_updates += per_thread(n_updates_pt, t); } - printf("n_reads: %lld n_updates: %ld nreaders: %d nupdaters: %d duration: %d\n", + diag("n_reads: %lld n_updates: %ld nreaders: %d nupdaters: %d duration: %d", n_reads, n_updates, nreaders, nupdaters, duration); - printf("ns/read: %g ns/update: %g\n", + diag("ns/read: %g ns/update: %g", ((duration * 1000*1000*1000.*(double)nreaders) / (double)n_reads), ((duration * 1000*1000*1000.*(double)nupdaters) / (double)n_updates)); if (get_cpu_call_rcu_data(0)) { - fprintf(stderr, "Deallocating per-CPU call_rcu threads.\n"); + diag("Deallocating per-CPU call_rcu threads.\n"); free_all_cpu_call_rcu_data(); } - exit(0); + return 0; } -void perftest(int nreaders, int cpustride) +int perftest(int nreaders, int cpustride) { int i; long arg; @@ -228,10 +230,10 @@ void perftest(int nreaders, int cpustride) } arg = (long)(i * cpustride); create_thread(rcu_update_perf_test, (void *)arg); - perftestrun(i + 1, nreaders, 1); + return perftestrun(i + 1, nreaders, 1); } -void rperftest(int nreaders, int cpustride) +int rperftest(int nreaders, int cpustride) { int i; long arg; @@ -242,10 +244,10 @@ void rperftest(int nreaders, int cpustride) arg = (long)(i * cpustride); create_thread(rcu_read_perf_test, (void *)arg); } - perftestrun(i, nreaders, 0); + return perftestrun(i, nreaders, 0); } -void uperftest(int nupdaters, int cpustride) +int uperftest(int nupdaters, int cpustride) { int i; long arg; @@ -256,7 +258,7 @@ void uperftest(int nupdaters, int cpustride) arg = (long)(i * cpustride); create_thread(rcu_update_perf_test, (void *)arg); } - perftestrun(i, 0, nupdaters); + return perftestrun(i, 0, nupdaters); } /* @@ -324,17 +326,28 @@ static pthread_cond_t call_rcu_test_cond = PTHREAD_COND_INITIALIZER; void rcu_update_stress_test_rcu(struct rcu_head *head) { - if (pthread_mutex_lock(&call_rcu_test_mutex) != 0) { - perror("pthread_mutex_lock"); - exit(-1); + int ret; + + ret = pthread_mutex_lock(&call_rcu_test_mutex); + if (ret) { + errno = ret; + diag("pthread_mutex_lock: %s", + strerror(errno)); + abort(); } - if (pthread_cond_signal(&call_rcu_test_cond) != 0) { - perror("pthread_cond_signal"); - exit(-1); + ret = pthread_cond_signal(&call_rcu_test_cond); + if (ret) { + errno = ret; + diag("pthread_cond_signal: %s", + strerror(errno)); + abort(); } - if (pthread_mutex_unlock(&call_rcu_test_mutex) != 0) { - perror("pthread_mutex_unlock"); - exit(-1); + ret = pthread_mutex_unlock(&call_rcu_test_mutex); + if (ret) { + errno = ret; + diag("pthread_mutex_unlock: %s", + strerror(errno)); + abort(); } } @@ -363,19 +376,30 @@ void *rcu_update_stress_test(void *arg) if (n_updates & 0x1) synchronize_rcu(); else { - if (pthread_mutex_lock(&call_rcu_test_mutex) != 0) { - perror("pthread_mutex_lock"); - exit(-1); + int ret; + + ret = pthread_mutex_lock(&call_rcu_test_mutex); + if (ret) { + errno = ret; + diag("pthread_mutex_lock: %s", + strerror(errno)); + abort(); } call_rcu(&rh, rcu_update_stress_test_rcu); - if (pthread_cond_wait(&call_rcu_test_cond, - &call_rcu_test_mutex) != 0) { - perror("pthread_cond_wait"); - exit(-1); + ret = pthread_cond_wait(&call_rcu_test_cond, + &call_rcu_test_mutex); + if (ret) { + errno = ret; + diag("pthread_cond_signal: %s", + strerror(errno)); + abort(); } - if (pthread_mutex_unlock(&call_rcu_test_mutex) != 0) { - perror("pthread_mutex_unlock"); - exit(-1); + ret = pthread_mutex_unlock(&call_rcu_test_mutex); + if (ret) { + errno = ret; + diag("pthread_mutex_unlock: %s", + strerror(errno)); + abort(); } } n_updates++; @@ -390,8 +414,7 @@ void *rcu_fake_update_stress_test(void *arg) crdp = create_call_rcu_data(0, -1); if (crdp != NULL) { - fprintf(stderr, - "Using per-thread call_rcu() worker.\n"); + diag("Using per-thread call_rcu() worker."); set_thread_call_rcu_data(crdp); } } @@ -404,7 +427,7 @@ void *rcu_fake_update_stress_test(void *arg) return NULL; } -void stresstest(int nreaders) +int stresstest(int nreaders) { int i; int t; @@ -435,22 +458,26 @@ void stresstest(int nreaders) wait_all_threads(); for_each_thread(t) n_reads += per_thread(n_reads_pt, t); - printf("n_reads: %lld n_updates: %ld n_mberror: %d\n", + diag("n_reads: %lld n_updates: %ld n_mberror: %d", n_reads, n_updates, n_mberror); - printf("rcu_stress_count:"); + rdiag_start(); + rdiag("rcu_stress_count:"); for (i = 0; i <= RCU_STRESS_PIPE_LEN; i++) { sum = 0LL; for_each_thread(t) { sum += per_thread(rcu_stress_count, t)[i]; } - printf(" %lld", sum); + rdiag(" %lld", sum); } - printf("\n"); + rdiag_end(); if (get_cpu_call_rcu_data(0)) { - fprintf(stderr, "Deallocating per-CPU call_rcu threads.\n"); + diag("Deallocating per-CPU call_rcu threads."); free_all_cpu_call_rcu_data(); } - exit(0); + if (!n_mberror) + return 0; + else + return -1; } /* @@ -459,7 +486,7 @@ void stresstest(int nreaders) void usage(int argc, char *argv[]) { - fprintf(stderr, "Usage: %s [nreaders [ perf | stress ] ]\n", argv[0]); + diag("Usage: %s [nreaders [ perf | rperf | uperf | stress ] ]\n", argv[0]); exit(-1); } @@ -468,13 +495,16 @@ int main(int argc, char *argv[]) int nreaders = 1; int cpustride = 1; + plan_tests(NR_TESTS); + smp_init(); //rcu_init(); srandom(time(NULL)); if (random() & 0x100) { - fprintf(stderr, "Allocating per-CPU call_rcu threads.\n"); + diag("Allocating per-CPU call_rcu threads."); if (create_all_cpu_call_rcu_data(0)) - perror("create_all_cpu_call_rcu_data"); + diag("create_all_cpu_call_rcu_data: %s", + strerror(errno)); } #ifdef DEBUG_YIELD @@ -484,20 +514,37 @@ int main(int argc, char *argv[]) if (argc > 1) { nreaders = strtoul(argv[1], NULL, 0); - if (argc == 2) - perftest(nreaders, cpustride); + if (argc == 2) { + ok(!perftest(nreaders, cpustride), + "perftest readers: %d, stride: %d", + nreaders, cpustride); + goto end; + } if (argc > 3) cpustride = strtoul(argv[3], NULL, 0); if (strcmp(argv[2], "perf") == 0) - perftest(nreaders, cpustride); + ok(!perftest(nreaders, cpustride), + "perftest readers: %d, stride: %d", + nreaders, cpustride); else if (strcmp(argv[2], "rperf") == 0) - rperftest(nreaders, cpustride); + ok(!rperftest(nreaders, cpustride), + "rperftest readers: %d, stride: %d", + nreaders, cpustride); else if (strcmp(argv[2], "uperf") == 0) - uperftest(nreaders, cpustride); + ok(!uperftest(nreaders, cpustride), + "uperftest readers: %d, stride: %d", + nreaders, cpustride); else if (strcmp(argv[2], "stress") == 0) - stresstest(nreaders); - usage(argc, argv); + ok(!stresstest(nreaders), + "stresstest readers: %d, stride: %d", + nreaders, cpustride); + else + usage(argc, argv); + } else { + ok(!perftest(nreaders, cpustride), + "perftest readers: %d, stride: %d", + nreaders, cpustride); } - perftest(nreaders, cpustride); - return 0; +end: + return exit_status(); } diff --git a/tests/regression/regression_tests b/tests/regression/regression_tests new file mode 100644 index 0000000..cdd7571 --- /dev/null +++ b/tests/regression/regression_tests @@ -0,0 +1,21 @@ +./test_urcu_fork +./rcutorture_urcu $(nproc) perf +./rcutorture_urcu_signal $(nproc) perf +./rcutorture_urcu_mb $(nproc) perf +./rcutorture_urcu_bp $(nproc) perf +./rcutorture_urcu_qsbr $(nproc) perf +./rcutorture_urcu $(nproc) rperf +./rcutorture_urcu_signal $(nproc) rperf +./rcutorture_urcu_mb $(nproc) rperf +./rcutorture_urcu_bp $(nproc) rperf +./rcutorture_urcu_qsbr $(nproc) rperf +./rcutorture_urcu $(nproc) uperf +./rcutorture_urcu_signal $(nproc) uperf +./rcutorture_urcu_mb $(nproc) uperf +./rcutorture_urcu_bp $(nproc) uperf +./rcutorture_urcu_qsbr $(nproc) uperf +./rcutorture_urcu $(nproc) stress +./rcutorture_urcu_signal $(nproc) stress +./rcutorture_urcu_mb $(nproc) stress +./rcutorture_urcu_bp $(nproc) stress +./rcutorture_urcu_qsbr $(nproc) stress diff --git a/tests/regression/run.sh b/tests/regression/run.sh new file mode 100755 index 0000000..d99a493 --- /dev/null +++ b/tests/regression/run.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Copyright (C) 2013 - Christian Babeux +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; only version 2 +# of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# + +INPUT=$1 +shift 1 + +[ -z "${INPUT}" ] && echo "Error: No testlist. Please specify a testlist to run." && exit 1 + +prove ${*} --merge --exec '' - < ${INPUT} diff --git a/tests/regression/test_urcu_fork.c b/tests/regression/test_urcu_fork.c index 06786b0..f614201 100644 --- a/tests/regression/test_urcu_fork.c +++ b/tests/regression/test_urcu_fork.c @@ -44,13 +44,27 @@ #endif #include +#include "tap.h" + /* We generate children 3 levels deep */ #define FORK_DEPTH 3 /* Each generation spawns 10 children */ #define NR_FORK 10 +#define NR_TESTS NR_FORK + static int fork_generation; +/* + * Only print diagnostic for top level parent process, else the console + * has trouble formatting the tap output. + */ +#define diag_gen0(...) \ + do { \ + if (!fork_generation) \ + diag(__VA_ARGS__); \ + } while (0) + struct test_node { int somedata; struct rcu_head head; @@ -60,8 +74,7 @@ static void cb(struct rcu_head *head) { struct test_node *node; - fprintf(stderr, "rcu callback invoked in pid: %d\n", - (int) getpid()); + diag_gen0("rcu callback invoked in pid: %d", (int) getpid()); node = caa_container_of(head, struct test_node, head); free(node); } @@ -94,7 +107,7 @@ static int do_fork(const char *execname) { pid_t pid; - fprintf(stderr, "%s parent pid: %d, before fork\n", + diag_gen0("%s parent pid: %d, before fork", execname, (int) getpid()); call_rcu_before_fork(); @@ -102,12 +115,13 @@ static int do_fork(const char *execname) if (pid == 0) { /* child */ fork_generation++; + tap_disable(); call_rcu_after_fork_child(); - fprintf(stderr, "%s child pid: %d, after fork\n", + diag_gen0("%s child pid: %d, after fork", execname, (int) getpid()); test_rcu(); - fprintf(stderr, "%s child pid: %d, after rcu test\n", + diag_gen0("%s child pid: %d, after rcu test", execname, (int) getpid()); if (fork_generation >= FORK_DEPTH) exit(EXIT_SUCCESS); @@ -117,25 +131,26 @@ static int do_fork(const char *execname) /* parent */ call_rcu_after_fork_parent(); - fprintf(stderr, "%s parent pid: %d, after fork\n", + diag_gen0("%s parent pid: %d, after fork", execname, (int) getpid()); test_rcu(); - fprintf(stderr, "%s parent pid: %d, after rcu test\n", + diag_gen0("%s parent pid: %d, after rcu test", execname, (int) getpid()); for (;;) { pid = wait(&status); if (pid < 0) { - perror("wait"); + if (!fork_generation) + perror("wait"); return -1; } if (WIFEXITED(status)) { - fprintf(stderr, "child %u exited normally with status %u\n", + diag_gen0("child %u exited normally with status %u", pid, WEXITSTATUS(status)); if (WEXITSTATUS(status)) return -1; break; } else if (WIFSIGNALED(status)) { - fprintf(stderr, "child %u was terminated by signal %u\n", + diag_gen0("child %u was terminated by signal %u", pid, WTERMSIG(status)); return -1; } else { @@ -144,7 +159,8 @@ static int do_fork(const char *execname) } return 1; } else { - perror("fork"); + if (!fork_generation) + perror("fork"); return -1; } } @@ -153,6 +169,8 @@ int main(int argc, char **argv) { unsigned int i; + plan_tests(NR_TESTS); + #if 0 /* pthread_atfork does not work with malloc/free in callbacks */ ret = pthread_atfork(call_rcu_before_fork, @@ -172,14 +190,27 @@ restart: test_rcu(); synchronize_rcu(); ret = do_fork(argv[0]); - if (ret == 0) /* child */ + if (!fork_generation) { + ok(ret >= 0, "child status %d", ret); + } + if (ret == 0) { /* child */ goto restart; - else if (ret < 0) + } else if (ret < 0) { goto error; - /* else parent, continue. */ + } else { + /* else parent, continue. */ + } + } + if (!fork_generation) { + return exit_status(); + } else { + exit(EXIT_SUCCESS); } - exit(EXIT_SUCCESS); error: - exit(EXIT_FAILURE); + if (!fork_generation) { + return exit_status(); + } else { + exit(EXIT_FAILURE); + } } diff --git a/tests/unit/run.sh b/tests/unit/run.sh index c6c50fd..d99a493 100755 --- a/tests/unit/run.sh +++ b/tests/unit/run.sh @@ -17,6 +17,9 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # -[ -z "$1" ] && echo "Error: No testlist. Please specify a testlist to run." && exit 1 +INPUT=$1 +shift 1 -prove --merge --exec '' - < $1 +[ -z "${INPUT}" ] && echo "Error: No testlist. Please specify a testlist to run." && exit 1 + +prove ${*} --merge --exec '' - < ${INPUT} diff --git a/tests/utils/tap.c b/tests/utils/tap.c index 8bf72f6..6acab66 100644 --- a/tests/utils/tap.c +++ b/tests/utils/tap.c @@ -42,6 +42,7 @@ static char *todo_msg = NULL; static char *todo_msg_fixed = "libtap malloc issue"; static int todo = 0; static int test_died = 0; +static int tap_is_disabled = 0; /* Encapsulate the pthread code in a conditional. In the absence of libpthread the code does nothing */ @@ -284,6 +285,32 @@ diag(char *fmt, ...) return 0; } +unsigned int +rdiag_start(void) +{ + fputs("# ", stderr); + return 0; +} + +unsigned int +rdiag(char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + + return 0; +} + +unsigned int +rdiag_end(void) +{ + fputs("\n", stderr); + return 0; +} + void _expected_tests(unsigned int tests) { @@ -389,6 +416,11 @@ _cleanup(void) LOCK; + if (tap_is_disabled) { + UNLOCK; + return; + } + /* If plan_no_plan() wasn't called, and we don't have a plan, and we're not skipping everything, then something happened before we could produce any output */ @@ -431,3 +463,12 @@ _cleanup(void) UNLOCK; } + +/* Disable tap for this process. */ +void +tap_disable(void) +{ + LOCK; + tap_is_disabled = 1; + UNLOCK; +} diff --git a/tests/utils/tap.h b/tests/utils/tap.h index 0f05943..9118bf5 100644 --- a/tests/utils/tap.h +++ b/tests/utils/tap.h @@ -87,3 +87,9 @@ void todo_start(char *, ...); void todo_end(void); int exit_status(void); + +void tap_disable(void); + +unsigned int rdiag_start(void); +unsigned int rdiag(char *fmt, ...); +unsigned int rdiag_end(void); -- 2.34.1