From: David Goulet Date: Wed, 19 Jun 2013 18:45:24 +0000 (-0400) Subject: Test: add new stress test that kills the relayd randomly X-Git-Tag: v2.2.0-rc3~9 X-Git-Url: https://git.lttng.org./?a=commitdiff_plain;h=b3300ee248b2ad8660b2f88a846469740288ebcf;p=lttng-tools.git Test: add new stress test that kills the relayd randomly Signed-off-by: David Goulet --- diff --git a/tests/stress/kill_relayd b/tests/stress/kill_relayd new file mode 100644 index 000000000..6ed2cb3d3 --- /dev/null +++ b/tests/stress/kill_relayd @@ -0,0 +1,67 @@ +#!/bin/bash +# +# Copyright (C) - 2013 David Goulet +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; version 2.1 of the License. +# +# This library is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +CURDIR=$(dirname $0)/ +TESTDIR=$CURDIR/.. +# We use the .libs/ binary since it's run from the repository. +REPO_RELAYD_BIN="lt-lttng-relayd" + +# Number of seconds the we sleep before killing the relayd. If RANDOM_KILL is +# defined, it's between 1 and 10 seconds. +NR_SEC=10 + +KILL_LOOP=0 +RANDOM_KILL=0 + +source $TESTDIR/utils/utils.sh + +function get_random() +{ + return $(echo $RANDOM % $NR_SEC + 1 | bc) +} + +function kill_relayd() +{ + if [ $RANDOM_KILL -eq 1 ]; then + # Something between 1 and NR_SEC seconds. + get_random + sleep $? + else + sleep $NR_SEC + fi + killall -q -9 $REPO_RELAYD_BIN >/dev/null 2>&1 +} + +# Do we have to run in an infinite loop ? +if [ -n "$1" ]; then + KILL_LOOP=1 +fi + +# Should it be a random kill or not ? +if [ -n "$2" ]; then + RANDOM_KILL=1 +fi + +# MUST set TESTDIR before this point. + +if [ $KILL_LOOP -eq 0 ]; then + kill_relayd +else + while :; do + kill_relayd + done +fi diff --git a/tests/stress/test_multi_sessions_per_uid_5app_streaming_kill_relayd b/tests/stress/test_multi_sessions_per_uid_5app_streaming_kill_relayd new file mode 100755 index 000000000..4ff927646 --- /dev/null +++ b/tests/stress/test_multi_sessions_per_uid_5app_streaming_kill_relayd @@ -0,0 +1,192 @@ +#!/bin/bash +# +# Copyright (C) - 2013 David Goulet +# +# This library is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; version 2.1 of the License. details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +CURDIR=$(dirname $0)/ +TESTDIR=$CURDIR/.. +LAUNCH_APP="launch_ust_app" +KILL_RELAYD_HELPER="kill_relayd" +SESSION_NAME="stress" +EVENT_NAME="tp:tptest" +LOG_FILE_SESSIOND="sessiond.log" +LOG_FILE_RELAYD="relayd.log" +CHANNEL_NAME="channel0" +NR_APP=5 +NR_SESSION=5 +NR_LOOP=100000 +COREDUMP_FILE=$(cat /proc/sys/kernel/core_pattern) +NUM_TESTS=16 + +TEST_DESC="Stress test - $NR_SESSION sessions per UID streaming with $NR_APP apps. The relayd is killed sporadically" + +source $TESTDIR/utils/utils.sh + +# MUST set TESTDIR before calling those functions + +function enable_channel_per_uid() +{ + local sess_name=$1 + local channel_name=$2 + + $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel --buffers-uid -u $channel_name -s $sess_name >/dev/null 2>&1 + ok $? "Enable channel $channel_name per UID for session $sess_name" +} + +function lttng_create_session_uri +{ + local name=$1 + + # Create session with default path + $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $name -U net://localhost >/dev/null 2>&1 + ok $? "Create session on net://localhost" +} + +function check_sessiond() +{ + if [ -z "$(pidof lt-lttng-sessiond)" ]; then + local str_date=$(date +%H%M%S-%d%m%Y) + + diag "!!!The session daemon died unexpectedly!!!" + mv $LOG_FILE_SESSIOND $LOG_FILE_SESSIOND-$str_date + if [ -e $COREDUMP_FILE ]; then + mv $COREDUMP_FILE $COREDUMP_FILE-$str_date + fi + exit 1 + fi +} + +function start_sessiond() +{ + local SESSIOND_BIN="lttng-sessiond" + + validate_kernel_version + if [ $? -ne 0 ]; then + fail "Start session daemon" + BAIL_OUT "*** Kernel too old for session daemon tests ***" + fi + + if [ -z $(pidof lt-$SESSIOND_BIN) ]; then + # We have to start it like this so the ulimit -c is used by this + # process. Also, we collect any error message printed out. + #$TESTDIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --quiet --consumerd32-path="$TESTDIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$TESTDIR/../src/bin/lttng-consumerd/lttng-consumerd" >$LOG_FILE_SESSIOND 2>&1 & + $TESTDIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --verbose-consumer -vvv --consumerd32-path="$TESTDIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$TESTDIR/../src/bin/lttng-consumerd/lttng-consumerd" >$LOG_FILE_SESSIOND 2>&1 & + #$TESTDIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --consumerd32-path="$TESTDIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$TESTDIR/../src/bin/lttng-consumerd/lttng-consumerd" >$LOG_FILE_SESSIOND 2>&1 & + status=$? + # Wait for sessiond to bootstrap + sleep 2 + ok $status "Start session daemon" + fi +} + +function start_relayd +{ + local opt=$1 + local RELAYD_BIN="lttng-relayd" + + if [ -z $(pidof lt-$RELAYD_BIN) ]; then + $TESTDIR/../src/bin/lttng-relayd/$RELAYD_BIN $opt >$LOG_FILE_RELAYD 2>&1 & + ok $? "Start lttng-relayd (opt: \"$opt\")" + fi +} + +function check_relayd() +{ + if [ -z "$(pidof lt-lttng-relayd)" ]; then + local str_date=$(date +%H%M%S-%d%m%Y) + + #diag "Relay daemon died. Starting it again" + if [ -e $COREDUMP_FILE ]; then + mv $COREDUMP_FILE $COREDUMP_FILE-$str_date + fi + start_relayd + fi +} + +test_stress() +{ + for b in $(seq 1 $NR_LOOP); do + for a in $(seq 1 $NR_SESSION); do + lttng_create_session_uri $SESSION_NAME-$a + check_sessiond + check_relayd + enable_channel_per_uid $SESSION_NAME-$a $CHANNEL_NAME + check_sessiond + check_relayd + enable_ust_lttng_event $SESSION_NAME-$a $EVENT_NAME + check_sessiond + check_relayd + start_lttng_tracing $SESSION_NAME-$a + check_sessiond + check_relayd + done + + for a in $(seq 1 $NR_SESSION); do + stop_lttng_tracing $SESSION_NAME-$a + check_sessiond + check_relayd + destroy_lttng_session $SESSION_NAME-$a + check_sessiond + check_relayd + done + done + + return 0 +} + +function cleanup() +{ + diag "Cleaning up!" + killall -9 $LAUNCH_APP + killall -9 $KILL_RELAYD_HELPER + stop_lttng_sessiond + stop_lttng_relayd +} + +function sighandler() +{ + cleanup + #rm $LOG_FILE_SESSIOND $LOG_FILE_RELAYD + exit 1 +} + +trap sighandler SIGINT + +# Make sure we collect a coredump if possible. +ulimit -c unlimited + +# MUST set TESTDIR before calling those functions +plan_tests $NUM_TESTS + +print_test_banner "$TEST_DESC" + +TRACE_PATH=$(mktemp -d) + +start_relayd "-o $TRACE_PATH" +start_sessiond + +diag "Starting applications launcher" + +# Start NR_APP applications script that will spawn apps non stop. +./$TESTDIR/stress/$LAUNCH_APP $NR_APP & +# Launch the helper script that will randomly kill the relayd at vitam eternam. +./$TESTDIR/stress/$KILL_RELAYD_HELPER 1 1 & + +test_stress +out=$? +if [ $out -ne 0 ]; then + cleanup + exit $out +fi + +cleanup +rm -rf $TRACE_PATH +rm $LOG_FILE_SESSIOND $LOG_FILE_RELAYD +exit 0