tests/regression/tools/notification/test_notification_multi_app

   1 #!/bin/bash
   2 #
   3 # Copyright (C) 2017 Jonathan Rajotte <jonathan.rajotte-julien@efficiso.com>>
   4 #
   5 # SPDX-License-Identifier: LGPL-2.1-only
   6
   7 TEST_DESC="Notification"
   8
   9 CURDIR=$(dirname $0)/
  10 TESTDIR=$CURDIR/../../../
  11
  12 TESTPOINT=$(readlink -f ${CURDIR}/.libs/libpause_consumer.so)
  13
  14 TESTAPP_PATH="$TESTDIR/utils/testapp"
  15 TESTAPP_NAME="gen-ust-events"
  16 TESTAPP_BIN="$TESTAPP_PATH/$TESTAPP_NAME/$TESTAPP_NAME"
  17 TESTAPP_STATE_FILE="$(mktemp -u -t tmp.test_notification_multi_app_state_file.XXXXXX)"
  18
  19 NR_ITER=1000
  20 NR_USEC_WAIT=5
  21
  22 SESSION_NAME="my_session"
  23 CHANNEL_NAME="my_channel"
  24
  25
  26 DIR=$(readlink -f $TESTDIR)
  27
  28 PAGE_SIZE=$(getconf PAGE_SIZE)
  29
  30 NUM_TEST_UST=50
  31 NUM_TEST_KERNEL=50
  32 NUM_TESTS=$(($NUM_TEST_UST + $NUM_TEST_KERNEL))
  33
  34 source $TESTDIR/utils/utils.sh
  35 source $CURDIR/util_event_generator.sh
  36
  37 consumerd_pipe=()
  38 file_sync_after_first_event=$(mktemp -u -t tmp.test_notification_multi_app_sync_after_first.XXXXXX)
  39
  40 # MUST set TESTDIR before calling those functions
  41 plan_tests $NUM_TESTS
  42
  43 print_test_banner "$TEST_DESC"
  44
  45 app_pids=()
  46
  47 function start_client {
  48         local pid=-1
  49         local output_file=$1
  50         local session_name=$2
  51         local channel_name=$3
  52         local domain_type=$4
  53         local buffer_usage_type=$5
  54         local buffer_usage_threshold_type=$6
  55         local buffer_usage_threshold_value=$7
  56         local nr_expected_notification=$8
  57         local use_action_list=$9
  58
  59         ${CURDIR}/base_client ${session_name} ${channel_name} ${domain_type} ${buffer_usage_type} ${buffer_usage_threshold_type} ${buffer_usage_threshold_value} ${nr_expected_notification} ${use_action_list} > ${output_file} &
  60         pid=$!
  61
  62         app_pids+=("$pid")
  63 }
  64
  65 function wait_for_message ()
  66 {
  67         local directory=$1
  68         local file_pattern=$2
  69         local message=$3
  70
  71         for file in $directory/${file_pattern}*; do
  72                 while(true); do
  73                         # Check for "error" message
  74                         grep -q "error:" ${file}
  75                         app_error=$?
  76                         if [ $app_error -eq "0" ] ; then
  77                                 # An error occurred
  78                                 fail "Waiting message: error logged see file content: ${message}, ${file}"
  79                                 return 1
  80                         fi
  81
  82                         grep -q "${message}" ${file}
  83                         if [[ "$?" -ne "0" ]]; then
  84                                 # Lookup failed restart loop
  85                                 diag "Lookup failed sleep and retry grep for: ${message}, ${file}"
  86                                 sleep 0.25
  87                                 continue
  88                         fi
  89                         break
  90                 done
  91         done
  92         pass "Message received: ${message}"
  93         return 0
  94 }
  95
  96 function print_errors ()
  97 {
  98         local directory=$1
  99         local file_pattern=$2
 100
 101         for file in $directory/${file_pattern}*; do
 102                 # Check for "error" message
 103                 error_message=$(grep "error:" ${file})
 104                 if [[ "${error_message}x" != "x" ]]; then
 105                         diag "Errors for application ${file}:"
 106                         diag "${error_message}"
 107                 fi
 108         done
 109 }
 110
 111 function comm_consumerd ()
 112 {
 113         local message=$1
 114         local pipe=$2
 115         echo -ne "${message}" > "${pipe}"
 116 }
 117
 118 function stop_consumerd ()
 119 {
 120         local pipe=$1
 121         comm_consumerd "1" "$pipe"
 122 }
 123
 124 function resume_consumerd ()
 125 {
 126         local pipe=$1
 127         comm_consumerd "\0" "$pipe"
 128 }
 129
 130 function test_multi_app ()
 131 {
 132         local domain_type=$1
 133         local event_generator_pid=$2
 134
 135         local app_pids=()
 136         local low_output_file_pattern="low_app_output_file_"
 137         local high_output_file_pattern="high_app_output_file_"
 138         local output_dir=$(mktemp -d -t "tmp.test_${FUNCNAME[0]}_output_dir.XXXXXX")
 139
 140         local testpoint_base_path=$(readlink -f "$output_dir/lttng.t_p_n_multi_app")
 141         local testpoint_pipe_path=$(TMPDIR="$output_dir" mktemp -u -t "lttng.t_p_n_multi_app.XXXXXX")
 142
 143         local nr_notification_expected=5
 144         local nr_client_app=50
 145         local domain_string=""
 146         local event_name=""
 147
 148         case $domain_type in
 149                 ust)
 150                         domain_string=LTTNG_DOMAIN_UST
 151                         event_name="tp:tptest"
 152                         ;;
 153                 kernel)
 154                         domain_string=LTTNG_DOMAIN_KERNEL
 155                         event_name="lttng_test_filter_event"
 156                         ;;
 157                 *)
 158                         fail "Invalid domain type"
 159                         exit 1
 160                         ;;
 161         esac
 162
 163         # Setup
 164         LTTNG_SESSIOND_ENV_VARS="LTTNG_TESTPOINT_ENABLE=1 CONSUMER_PAUSE_PIPE_PATH=${testpoint_pipe_path} LD_PRELOAD=${TESTPOINT}"
 165         start_lttng_sessiond
 166
 167         create_lttng_session_ok $SESSION_NAME $TRACE_PATH
 168         enable_${domain_type}_lttng_channel_ok $SESSION_NAME $CHANNEL_NAME --subbuf-size=$PAGE_SIZE
 169         enable_${domain_type}_lttng_event_ok $SESSION_NAME $event_name $CHANNEL_NAME
 170
 171         # Fetch consumerd testpoint pipe information
 172         # This is needed since the testpoint create a pipe with the consumer type suffixed
 173         consumerd_pipe=()
 174         for f in "$testpoint_pipe_path"*; do
 175                 consumerd_pipe+=("$f")
 176         done
 177
 178         for (( i = 0; i < $nr_client_app; i++ )); do
 179                 low_app_output_file=$output_dir/${low_output_file_pattern}${i}
 180                 high_app_output_file=$output_dir/${high_output_file_pattern}${i}
 181                 start_client $low_app_output_file $SESSION_NAME $CHANNEL_NAME $domain_string LOW RATIO 0.0 $nr_notification_expected $(( $i % 2))
 182                 start_client $high_app_output_file $SESSION_NAME $CHANNEL_NAME $domain_string HIGH RATIO 0.420 $nr_notification_expected $(( $i % 2))
 183         done
 184
 185         wait_for_message $output_dir "${low_output_file_pattern}" "sync: ready"
 186         wait_for_message $output_dir "${high_output_file_pattern}" "sync: ready"
 187
 188         # Test notification reception
 189         for (( i = 0; i < $nr_notification_expected; i++ )); do
 190
 191                 # Stop consumerd consumption to force high notification
 192                 start_lttng_tracing_ok $SESSION_NAME
 193                 for pipe in "${consumerd_pipe[@]}"; do
 194                         stop_consumerd "${pipe}"
 195                 done
 196
 197                 wait_for_message $output_dir "${high_output_file_pattern}" "notification: high $i"
 198
 199                 # Put application in suspend mode to prevent double low
 200                 # notification and synchronize on state file.
 201                 kill -s SIGUSR1 $event_generator_pid
 202                 while [ ! -f "${TESTAPP_STATE_FILE}" ]; do
 203                         sleep 0.5
 204                 done
 205
 206                 # Resume consumerd
 207                 for pipe in "${consumerd_pipe[@]}"; do
 208                         resume_consumerd "${pipe}"
 209                 done
 210                 # Stop tracing forcing full buffer consumption
 211                 stop_lttng_tracing_ok $SESSION_NAME
 212
 213                 # Check for notifications reception
 214                 wait_for_message $output_dir "${low_output_file_pattern}" "notification: low $i"
 215                 ret=$?
 216                 ok $ret "Notifications $i received"
 217                 if [[ $ret -ne "0" ]]; then
 218                         # Error occurred bail out
 219                         break;
 220                 fi
 221
 222                 # Put application in active mode and synchronize on state file.
 223                 kill -s SIGUSR1 $event_generator_pid
 224                 while [ -f "${TESTAPP_STATE_FILE}" ]; do
 225                         sleep 0.5
 226                 done
 227         done
 228
 229         wait_for_message $output_dir "${low_output_file_pattern}" "exit: 0"
 230         ret=$?
 231         ok $ret "Application for low notification terminated normally"
 232         if [[ $ret -ne "0" ]]; then
 233                 print_errors $output_dir "${low_output_file_pattern}"
 234         fi
 235
 236         wait_for_message $output_dir "${high_output_file_pattern}" "exit: 0"
 237         ret=$?
 238         ok $ret "Application for high notification terminated normally"
 239         if [[ $ret -ne "0" ]]; then
 240                 print_errors $output_dir "${high_output_file_pattern}"
 241         fi
 242
 243
 244         destroy_lttng_session_ok $SESSION_NAME
 245         stop_lttng_sessiond
 246
 247         for pipe in "${consumerd_pipe[@]}"; do
 248                 rm -rf "${pipe}"
 249         done
 250
 251         rm -rf $output_dir
 252 }
 253
 254 function test_multi_app_ust ()
 255 {
 256         diag "Multi client app UST notification"
 257         ust_event_generator "$TESTAPP_BIN" "$TESTAPP_STATE_FILE" &
 258         local generator_pid=$!
 259
 260         test_multi_app ust $generator_pid
 261
 262         kill -s SIGUSR2 $generator_pid 2> /dev/null
 263         wait $generator_pid 2> /dev/null
 264         rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null
 265 }
 266
 267 function test_multi_app_kernel ()
 268 {
 269         diag "Multi client app kernel notification"
 270         modprobe lttng-test
 271
 272         kernel_event_generator generate_filter_events $TESTAPP_STATE_FILE &
 273         local generator_pid=$!
 274
 275         test_multi_app kernel $generator_pid
 276
 277
 278         kill -s SIGUSR2 $generator_pid 2> /dev/null
 279         wait $generator_pid 2> /dev/null
 280         rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null
 281
 282         modprobe --remove lttng-test
 283 }
 284
 285 function test_on_register_evaluation_ust ()
 286 {
 287         diag "On register notification UST"
 288
 289         # Start app in infinite loop
 290         ust_event_generator "$TESTAPP_BIN" "$TESTAPP_STATE_FILE" &
 291         local generator_pid=$!
 292
 293         test_on_register_evaluation ust $generator_pid
 294
 295         kill -s SIGUSR2 $generator_pid 2> /dev/null
 296         wait $generator_pid 2> /dev/null
 297         rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null
 298
 299 }
 300
 301 function test_on_register_evaluation_kernel()
 302 {
 303         diag "On register notification kernel"
 304
 305         modprobe lttng-test
 306
 307         kernel_event_generator generate_filter_events $TESTAPP_STATE_FILE &
 308         local generator_pid=$!
 309
 310         test_on_register_evaluation kernel $generator_pid
 311
 312
 313         kill -s SIGUSR2 $generator_pid 2> /dev/null
 314         wait $generator_pid 2> /dev/null
 315         rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null
 316
 317         modprobe --remove lttng-test
 318 }
 319
 320 function test_on_register_evaluation ()
 321 {
 322         local domain_type=$1
 323         local event_generator_pid=$2
 324
 325         local app_pids=()
 326         local high_output_file_pattern="high_app_output_file_on_register_evaluation"
 327
 328         local output_dir=$(mktemp -d -t "tmp.test_${FUNCNAME[0]}_output_dir.XXXXXX")
 329         local testpoint_base_path=$(readlink -f "$output_dir/lttng.t_p_n_register_evaluation")
 330         local testpoint_pipe_path=$(TMPDIR="$output_dir" mktemp -u -t "lttng.t_p_n_register_evaluation.XXXXXX")
 331         local domain_string=""
 332         local event_name=""
 333
 334         case $domain_type in
 335                 ust)
 336                         domain_string=LTTNG_DOMAIN_UST
 337                         event_name="tp:tptest"
 338                         ;;
 339                 kernel)
 340                         domain_string=LTTNG_DOMAIN_KERNEL
 341                         event_name="lttng_test_filter_event"
 342                         ;;
 343                 *)
 344                         fail "Invalid domain type"
 345                         exit 1
 346                         ;;
 347         esac
 348
 349         # Setup
 350         LTTNG_SESSIOND_ENV_VARS="LTTNG_TESTPOINT_ENABLE=1 CONSUMER_PAUSE_PIPE_PATH=${testpoint_pipe_path} LD_PRELOAD=${TESTPOINT}"
 351         start_lttng_sessiond
 352
 353         create_lttng_session_ok $SESSION_NAME $TRACE_PATH
 354         enable_${domain_type}_lttng_channel_ok $SESSION_NAME $CHANNEL_NAME --subbuf-size=$PAGE_SIZE
 355         enable_${domain_type}_lttng_event_ok $SESSION_NAME $event_name $CHANNEL_NAME
 356
 357         # Fetch consumerd testpoint pipe information
 358         # This is needed since the testpoint create a pipe with the consumer type suffixed
 359         consumerd_pipe=()
 360         for f in "$testpoint_pipe_path"*; do
 361                 consumerd_pipe+=("$f")
 362         done
 363
 364         high_app_output_file=${high_output_file_pattern}.first_receiver
 365         high_app_output_path=$output_dir/${high_app_output_file}
 366         start_client $high_app_output_path $SESSION_NAME $CHANNEL_NAME $domain_string HIGH RATIO 0.420 1 0
 367
 368         wait_for_message $output_dir "${high_app_output_file}" "sync: ready"
 369
 370         # Stop consumerd consumption to force high notification
 371         start_lttng_tracing_ok $SESSION_NAME
 372
 373         for pipe in "${consumerd_pipe[@]}"; do
 374                 stop_consumerd "${pipe}"
 375         done
 376
 377         wait_for_message $output_dir "${high_app_output_file}" "notification: high 0"
 378
 379         # Start a second receiver, the receiver should receive a high
 380         # notification on subscription
 381         high_app_output_file=${high_output_file_pattern}.second_receiver
 382         high_app_output_path=$output_dir/${high_app_output_file}
 383         start_client $high_app_output_path $SESSION_NAME $CHANNEL_NAME $domain_string HIGH RATIO 0.420 1 0
 384         wait_for_message $output_dir "${high_app_output_file}" "sync: ready"
 385         wait_for_message $output_dir "${high_app_output_file}" "notification: high 0"
 386
 387         # Resume consumerd
 388         for pipe in "${consumerd_pipe[@]}"; do
 389                 resume_consumerd "${pipe}"
 390         done
 391
 392         wait_for_message $output_dir "${high_output_file_pattern}" "exit: 0"
 393         ret=$?
 394         ok $ret "Application for high notification terminated normally"
 395         if [[ $ret -ne "0" ]]; then
 396                 # Keep the file
 397                 print_errors "${high_output_file_pattern}"
 398         fi
 399
 400
 401         destroy_lttng_session_ok $SESSION_NAME
 402         stop_lttng_sessiond
 403
 404         kill -s SIGUSR2 $generator_pid 2> /dev/null
 405         wait $generator_pid 2> /dev/null
 406
 407         for pipe in "${consumerd_pipe[@]}"; do
 408                 rm -rf "${pipe}"
 409         done
 410
 411         rm -rf "$output_dir"
 412 }
 413
 414 TESTS=(
 415         test_multi_app_ust
 416         test_on_register_evaluation_ust
 417 )
 418
 419 check_skip_kernel_test "$NUM_TEST_KERNEL" "Skipping kernel multi-app notification tests." || {
 420         validate_lttng_modules_present
 421         TESTS+=(
 422                 test_multi_app_kernel
 423                 test_on_register_evaluation_kernel
 424         )
 425 }
 426
 427 for fct_test in ${TESTS[@]};
 428 do
 429         TRACE_PATH=$(mktemp -d -t tmp.test_notification_multi_app_trace_path.XXXXXX)
 430
 431         ${fct_test}
 432         if [ $? -ne 0 ]; then
 433                 break;
 434         fi
 435
 436         # Only delete if successful
 437         rm -rf $TRACE_PATH
 438 done