Test: add new stress test that kills the relayd randomly

Signed-off-by: David Goulet <dgoulet@efficios.com>

diff --git a/tests/stress/kill_relayd b/tests/stress/kill_relayd
new file mode 100644 (file)
index 0000000..6ed2cb3
--- /dev/null
+++ b/tests/stress/kill_relayd
@@ -0,0 +1,67 @@
+#!/bin/bash
+#
+# Copyright (C) - 2013 David Goulet <dgoulet@efficios.com>
+#
+# This library is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation; version 2.1 of the License.
+#
+# This library is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this library; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
+
+CURDIR=$(dirname $0)/
+TESTDIR=$CURDIR/..
+# We use the .libs/ binary since it's run from the repository.
+REPO_RELAYD_BIN="lt-lttng-relayd"
+
+# Number of seconds the we sleep before killing the relayd. If RANDOM_KILL is
+# defined, it's between 1 and 10 seconds.
+NR_SEC=10
+
+KILL_LOOP=0
+RANDOM_KILL=0
+
+source $TESTDIR/utils/utils.sh
+
+function get_random()
+{
+       return $(echo $RANDOM % $NR_SEC + 1 | bc)
+}
+
+function kill_relayd()
+{
+       if [ $RANDOM_KILL -eq 1 ]; then
+               # Something between 1 and NR_SEC seconds.
+               get_random
+               sleep $?
+       else
+               sleep $NR_SEC
+       fi
+       killall -q -9 $REPO_RELAYD_BIN >/dev/null 2>&1
+}
+
+# Do we have to run in an infinite loop ?
+if [ -n "$1" ]; then
+       KILL_LOOP=1
+fi
+
+# Should it be a random kill or not ?
+if [ -n "$2" ]; then
+       RANDOM_KILL=1
+fi
+
+# MUST set TESTDIR before this point.
+
+if [ $KILL_LOOP -eq 0 ]; then
+       kill_relayd
+else
+       while :; do
+               kill_relayd
+       done
+fi

diff --git a/tests/stress/test_multi_sessions_per_uid_5app_streaming_kill_relayd b/tests/stress/test_multi_sessions_per_uid_5app_streaming_kill_relayd
new file mode 100755 (executable)
index 0000000..4ff9276
--- /dev/null
+++ b/tests/stress/test_multi_sessions_per_uid_5app_streaming_kill_relayd
@@ -0,0 +1,192 @@
+#!/bin/bash
+#
+# Copyright (C) - 2013 David Goulet <dgoulet@efficios.com>
+#
+# This library is free software; you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation; version 2.1 of the License.  details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this library; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
+
+CURDIR=$(dirname $0)/
+TESTDIR=$CURDIR/..
+LAUNCH_APP="launch_ust_app"
+KILL_RELAYD_HELPER="kill_relayd"
+SESSION_NAME="stress"
+EVENT_NAME="tp:tptest"
+LOG_FILE_SESSIOND="sessiond.log"
+LOG_FILE_RELAYD="relayd.log"
+CHANNEL_NAME="channel0"
+NR_APP=5
+NR_SESSION=5
+NR_LOOP=100000
+COREDUMP_FILE=$(cat /proc/sys/kernel/core_pattern)
+NUM_TESTS=16
+
+TEST_DESC="Stress test - $NR_SESSION sessions per UID streaming with $NR_APP apps. The relayd is killed sporadically"
+
+source $TESTDIR/utils/utils.sh
+
+# MUST set TESTDIR before calling those functions
+
+function enable_channel_per_uid()
+{
+    local sess_name=$1
+    local channel_name=$2
+
+    $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel --buffers-uid -u $channel_name -s $sess_name >/dev/null 2>&1
+    ok $? "Enable channel $channel_name per UID for session $sess_name"
+}
+
+function lttng_create_session_uri
+{
+       local name=$1
+
+       # Create session with default path
+       $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $name -U net://localhost >/dev/null 2>&1
+       ok $? "Create session on net://localhost"
+}
+
+function check_sessiond()
+{
+       if [ -z "$(pidof lt-lttng-sessiond)" ]; then
+               local str_date=$(date +%H%M%S-%d%m%Y)
+
+               diag "!!!The session daemon died unexpectedly!!!"
+               mv $LOG_FILE_SESSIOND $LOG_FILE_SESSIOND-$str_date
+               if [ -e $COREDUMP_FILE ]; then
+                       mv $COREDUMP_FILE $COREDUMP_FILE-$str_date
+               fi
+               exit 1
+       fi
+}
+
+function start_sessiond()
+{
+       local SESSIOND_BIN="lttng-sessiond"
+
+       validate_kernel_version
+       if [ $? -ne 0 ]; then
+               fail "Start session daemon"
+               BAIL_OUT "*** Kernel too old for session daemon tests ***"
+       fi
+
+       if [ -z $(pidof lt-$SESSIOND_BIN) ]; then
+               # We have to start it like this so the ulimit -c is used by this
+               # process. Also, we collect any error message printed out.
+               #$TESTDIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --quiet --consumerd32-path="$TESTDIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$TESTDIR/../src/bin/lttng-consumerd/lttng-consumerd" >$LOG_FILE_SESSIOND 2>&1 &
+               $TESTDIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --verbose-consumer -vvv --consumerd32-path="$TESTDIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$TESTDIR/../src/bin/lttng-consumerd/lttng-consumerd" >$LOG_FILE_SESSIOND 2>&1 &
+               #$TESTDIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --consumerd32-path="$TESTDIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$TESTDIR/../src/bin/lttng-consumerd/lttng-consumerd" >$LOG_FILE_SESSIOND 2>&1 &
+               status=$?
+               # Wait for sessiond to bootstrap
+               sleep 2
+               ok $status "Start session daemon"
+       fi
+}
+
+function start_relayd
+{
+       local opt=$1
+       local RELAYD_BIN="lttng-relayd"
+
+       if [ -z $(pidof lt-$RELAYD_BIN) ]; then
+               $TESTDIR/../src/bin/lttng-relayd/$RELAYD_BIN $opt >$LOG_FILE_RELAYD 2>&1 &
+               ok $? "Start lttng-relayd (opt: \"$opt\")"
+       fi
+}
+
+function check_relayd()
+{
+       if [ -z "$(pidof lt-lttng-relayd)" ]; then
+               local str_date=$(date +%H%M%S-%d%m%Y)
+
+               #diag "Relay daemon died. Starting it again"
+               if [ -e $COREDUMP_FILE ]; then
+                       mv $COREDUMP_FILE $COREDUMP_FILE-$str_date
+               fi
+               start_relayd
+       fi
+}
+
+test_stress()
+{
+       for b in $(seq 1 $NR_LOOP); do
+               for a in $(seq 1 $NR_SESSION); do
+                       lttng_create_session_uri $SESSION_NAME-$a
+                       check_sessiond
+                       check_relayd
+                       enable_channel_per_uid $SESSION_NAME-$a $CHANNEL_NAME
+                       check_sessiond
+                       check_relayd
+                       enable_ust_lttng_event $SESSION_NAME-$a $EVENT_NAME
+                       check_sessiond
+                       check_relayd
+                       start_lttng_tracing $SESSION_NAME-$a
+                       check_sessiond
+                       check_relayd
+               done
+
+               for a in $(seq 1 $NR_SESSION); do
+                       stop_lttng_tracing $SESSION_NAME-$a
+                       check_sessiond
+                       check_relayd
+                       destroy_lttng_session $SESSION_NAME-$a
+                       check_sessiond
+                       check_relayd
+               done
+       done
+
+       return 0
+}
+
+function cleanup()
+{
+       diag "Cleaning up!"
+       killall -9 $LAUNCH_APP
+       killall -9 $KILL_RELAYD_HELPER
+       stop_lttng_sessiond
+       stop_lttng_relayd
+}
+
+function sighandler()
+{
+       cleanup
+       #rm $LOG_FILE_SESSIOND $LOG_FILE_RELAYD
+       exit 1
+}
+
+trap sighandler SIGINT
+
+# Make sure we collect a coredump if possible.
+ulimit -c unlimited
+
+# MUST set TESTDIR before calling those functions
+plan_tests $NUM_TESTS
+
+print_test_banner "$TEST_DESC"
+
+TRACE_PATH=$(mktemp -d)
+
+start_relayd "-o $TRACE_PATH"
+start_sessiond
+
+diag "Starting applications launcher"
+
+# Start NR_APP applications script that will spawn apps non stop.
+./$TESTDIR/stress/$LAUNCH_APP $NR_APP &
+# Launch the helper script that will randomly kill the relayd at vitam eternam.
+./$TESTDIR/stress/$KILL_RELAYD_HELPER 1 1 &
+
+test_stress
+out=$?
+if [ $out -ne 0 ]; then
+       cleanup
+       exit $out
+fi
+
+cleanup
+rm -rf $TRACE_PATH
+rm $LOG_FILE_SESSIOND $LOG_FILE_RELAYD
+exit 0