From a33d2d4ae3b4bd640b5f535b4213e6606e7d0ddd Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Sun, 26 Jan 2014 23:53:21 -0500 Subject: [PATCH] health check tests: test relayd and consumerd - Cover relayd and consumerd, - Add a test_thread_ok test (no issue found by test) to fast_regression, - Merge duplicated code into test_health.sh. Signed-off-by: Mathieu Desnoyers --- tests/fast_regression | 1 + tests/regression/tools/health/Makefile.am | 6 +- tests/regression/tools/health/health_check.c | 1 + tests/regression/tools/health/health_exit.c | 170 ++++++++++ tests/regression/tools/health/health_fail.c | 185 +++++++++++ tests/regression/tools/health/health_stall.c | 220 ++++++++++++- tests/regression/tools/health/test_health.sh | 310 ++++++++++++++++++ .../regression/tools/health/test_thread_exit | 118 +------ tests/regression/tools/health/test_thread_ok | 2 +- .../regression/tools/health/test_thread_stall | 146 +-------- tests/regression/tools/health/test_tp_fail | 109 +----- 11 files changed, 887 insertions(+), 381 deletions(-) create mode 100644 tests/regression/tools/health/test_health.sh diff --git a/tests/fast_regression b/tests/fast_regression index e969c3d76..40acf5cd7 100644 --- a/tests/fast_regression +++ b/tests/fast_regression @@ -2,6 +2,7 @@ regression/tools/filtering/test_invalid_filter regression/tools/filtering/test_unsupported_op regression/tools/filtering/test_valid_filter regression/tools/streaming/test_ust +regression/tools/health/test_thread_ok regression/tools/live/test_ust regression/tools/live/test_ust_tracefile_count regression/tools/live/test_lttng_ust diff --git a/tests/regression/tools/health/Makefile.am b/tests/regression/tools/health/Makefile.am index c22d5ee39..ed13743a4 100644 --- a/tests/regression/tools/health/Makefile.am +++ b/tests/regression/tools/health/Makefile.am @@ -37,8 +37,10 @@ health_check_LDADD = $(top_builddir)/src/lib/lttng-ctl/liblttng-ctl.la \ $(top_builddir)/src/common/libcommon.la endif -dist_noinst_SCRIPTS = test_thread_exit test_thread_stall test_tp_fail -EXTRA_DIST = test_thread_exit test_thread_stall test_tp_fail +dist_noinst_SCRIPTS = test_thread_exit test_thread_stall test_tp_fail \ + test_health.sh +EXTRA_DIST = test_thread_exit test_thread_stall test_tp_fail \ + test_health.sh all-local: @if [ x"$(srcdir)" != x"$(builddir)" ]; then \ diff --git a/tests/regression/tools/health/health_check.c b/tests/regression/tools/health/health_check.c index dc975de99..9109b1824 100644 --- a/tests/regression/tools/health/health_check.c +++ b/tests/regression/tools/health/health_check.c @@ -1,5 +1,6 @@ /* * Copyright (C) 2012 - Christian Babeux + * Copyright (C) 2014 - Mathieu Desnoyers * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License, version 2 only, as diff --git a/tests/regression/tools/health/health_exit.c b/tests/regression/tools/health/health_exit.c index 55cff4611..8a9932644 100644 --- a/tests/regression/tools/health/health_exit.c +++ b/tests/regression/tools/health/health_exit.c @@ -1,5 +1,6 @@ /* * Copyright (C) 2012 - Christian Babeux + * Copyright (C) 2014 - Mathieu Desnoyers * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License, version 2 only, as @@ -24,6 +25,7 @@ * Check if the specified environment variable is set. * Return 1 if set, otherwise 0. */ +static int check_env_var(const char *env) { if (env) { @@ -36,6 +38,8 @@ int check_env_var(const char *env) return 0; } +/* Session daemon */ + int __testpoint_sessiond_thread_manage_clients(void) { const char *var = "LTTNG_SESSIOND_THREAD_MANAGE_CLIENTS_EXIT"; @@ -92,3 +96,169 @@ int __testpoint_sessiond_thread_manage_consumer(void) return 0; } + +int __testpoint_sessiond_thread_ht_cleanup(void) +{ + const char *var = "LTTNG_SESSIOND_THREAD_HT_CLEANUP_EXIT"; + + if (check_env_var(var)) { + rcu_unregister_thread(); + pthread_exit(NULL); + } + + return 0; +} + +int __testpoint_sessiond_thread_app_manage_notify(void) +{ + const char *var = "LTTNG_SESSIOND_THREAD_APP_MANAGE_NOTIFY_EXIT"; + + if (check_env_var(var)) { + rcu_unregister_thread(); + pthread_exit(NULL); + } + + return 0; +} + +int __testpoint_sessiond_thread_app_reg_dispatch(void) +{ + const char *var = "LTTNG_SESSIOND_THREAD_APP_REG_DISPATCH_EXIT"; + + if (check_env_var(var)) { + pthread_exit(NULL); + } + + return 0; +} + +/* Consumer daemon */ + +int __testpoint_consumerd_thread_channel(void) +{ + const char *var = "LTTNG_CONSUMERD_THREAD_CHANNEL_EXIT"; + + if (check_env_var(var)) { + rcu_unregister_thread(); + pthread_exit(NULL); + } + + return 0; +} + +int __testpoint_consumerd_thread_metadata(void) +{ + const char *var = "LTTNG_CONSUMERD_THREAD_METADATA_EXIT"; + + if (check_env_var(var)) { + rcu_unregister_thread(); + pthread_exit(NULL); + } + + return 0; +} + +int __testpoint_consumerd_thread_data(void) +{ + const char *var = "LTTNG_CONSUMERD_THREAD_DATA_EXIT"; + + if (check_env_var(var)) { + rcu_unregister_thread(); + pthread_exit(NULL); + } + + return 0; +} + +int __testpoint_consumerd_thread_sessiond(void) +{ + const char *var = "LTTNG_CONSUMERD_THREAD_SESSIOND_EXIT"; + + if (check_env_var(var)) { + rcu_unregister_thread(); + pthread_exit(NULL); + } + + return 0; +} + +int __testpoint_consumerd_thread_metadata_timer(void) +{ + const char *var = "LTTNG_CONSUMERD_THREAD_METADATA_TIMER_EXIT"; + + if (check_env_var(var)) { + pthread_exit(NULL); + } + + return 0; +} + +/* Relay daemon */ + +int __testpoint_relayd_thread_dispatcher(void) +{ + const char *var = "LTTNG_RELAYD_THREAD_DISPATCHER_EXIT"; + + if (check_env_var(var)) { + pthread_exit(NULL); + } + + return 0; +} + +int __testpoint_relayd_thread_worker(void) +{ + const char *var = "LTTNG_RELAYD_THREAD_WORKER_EXIT"; + + if (check_env_var(var)) { + rcu_unregister_thread(); + pthread_exit(NULL); + } + + return 0; +} + +int __testpoint_relayd_thread_listener(void) +{ + const char *var = "LTTNG_RELAYD_THREAD_LISTENER_EXIT"; + + if (check_env_var(var)) { + pthread_exit(NULL); + } + + return 0; +} + +int __testpoint_relayd_thread_live_dispatcher(void) +{ + const char *var = "LTTNG_RELAYD_THREAD_LIVE_DISPATCHER_EXIT"; + + if (check_env_var(var)) { + pthread_exit(NULL); + } + + return 0; +} + +int __testpoint_relayd_thread_live_worker(void) +{ + const char *var = "LTTNG_RELAYD_THREAD_LIVE_WORKER_EXIT"; + + if (check_env_var(var)) { + rcu_unregister_thread(); + pthread_exit(NULL); + } + + return 0; +} + +int __testpoint_relayd_thread_live_listener(void) +{ + const char *var = "LTTNG_RELAYD_THREAD_LIVE_LISTENER_EXIT"; + + if (check_env_var(var)) { + pthread_exit(NULL); + } + + return 0; +} diff --git a/tests/regression/tools/health/health_fail.c b/tests/regression/tools/health/health_fail.c index ca1602af3..3f83a7c73 100644 --- a/tests/regression/tools/health/health_fail.c +++ b/tests/regression/tools/health/health_fail.c @@ -1,5 +1,6 @@ /* * Copyright (C) 2012 - Christian Babeux + * Copyright (C) 2014 - Mathieu Desnoyers * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License, version 2 only, as @@ -18,11 +19,13 @@ #include #include #include +#include /* * Check if the specified environment variable is set. * Return 1 if set, otherwise 0. */ +static int check_env_var(const char *env) { if (env) { @@ -35,6 +38,8 @@ int check_env_var(const char *env) return 0; } +/* Session daemon */ + int __testpoint_sessiond_thread_manage_clients(void) { const char *var = "LTTNG_SESSIOND_THREAD_MANAGE_CLIENTS_TP_FAIL"; @@ -46,6 +51,17 @@ int __testpoint_sessiond_thread_manage_clients(void) return 0; } +int __testpoint_sessiond_thread_registration_apps(void) +{ + const char *var = "LTTNG_SESSIOND_THREAD_REG_APPS_TP_FAIL"; + + if (check_env_var(var)) { + return 1; + } + + return 0; +} + int __testpoint_sessiond_thread_manage_apps(void) { const char *var = "LTTNG_SESSIOND_THREAD_MANAGE_APPS_TP_FAIL"; @@ -67,3 +83,172 @@ int __testpoint_sessiond_thread_manage_kernel(void) return 0; } + +int __testpoint_sessiond_thread_manage_consumer(void) +{ + const char *var = "LTTNG_SESSIOND_THREAD_MANAGE_CONSUMER_TP_FAIL"; + + if (check_env_var(var)) { + return 1; + } + + return 0; +} + +int __testpoint_sessiond_thread_ht_cleanup(void) +{ + const char *var = "LTTNG_SESSIOND_THREAD_HT_CLEANUP_TP_FAIL"; + + if (check_env_var(var)) { + return 1; + } + + return 0; +} + +int __testpoint_sessiond_thread_app_manage_notify(void) +{ + const char *var = "LTTNG_SESSIOND_THREAD_APP_MANAGE_NOTIFY_TP_FAIL"; + + if (check_env_var(var)) { + return 1; + } + + return 0; +} + +int __testpoint_sessiond_thread_app_reg_dispatch(void) +{ + const char *var = "LTTNG_SESSIOND_THREAD_APP_REG_DISPATCH_TP_FAIL"; + + if (check_env_var(var)) { + return 1; + } + + return 0; +} + +/* Consumer daemon */ + +int __testpoint_consumerd_thread_channel(void) +{ + const char *var = "LTTNG_CONSUMERD_THREAD_CHANNEL_TP_FAIL"; + + if (check_env_var(var)) { + return 1; + } + + return 0; +} + +int __testpoint_consumerd_thread_metadata(void) +{ + const char *var = "LTTNG_CONSUMERD_THREAD_METADATA_TP_FAIL"; + + if (check_env_var(var)) { + return 1; + } + + return 0; +} + +int __testpoint_consumerd_thread_data(void) +{ + const char *var = "LTTNG_CONSUMERD_THREAD_DATA_TP_FAIL"; + + if (check_env_var(var)) { + return 1; + } + + return 0; +} + +int __testpoint_consumerd_thread_sessiond(void) +{ + const char *var = "LTTNG_CONSUMERD_THREAD_SESSIOND_TP_FAIL"; + + if (check_env_var(var)) { + return 1; + } + + return 0; +} + +int __testpoint_consumerd_thread_metadata_timer(void) +{ + const char *var = "LTTNG_CONSUMERD_THREAD_METADATA_TIMER_TP_FAIL"; + + if (check_env_var(var)) { + return 1; + } + + return 0; +} + +/* Relay daemon */ + +int __testpoint_relayd_thread_dispatcher(void) +{ + const char *var = "LTTNG_RELAYD_THREAD_DISPATCHER_TP_FAIL"; + + if (check_env_var(var)) { + return 1; + } + + return 0; +} + +int __testpoint_relayd_thread_worker(void) +{ + const char *var = "LTTNG_RELAYD_THREAD_WORKER_TP_FAIL"; + + if (check_env_var(var)) { + return 1; + } + + return 0; +} + +int __testpoint_relayd_thread_listener(void) +{ + const char *var = "LTTNG_RELAYD_THREAD_LISTENER_TP_FAIL"; + + if (check_env_var(var)) { + return 1; + } + + return 0; +} + +int __testpoint_relayd_thread_live_dispatcher(void) +{ + const char *var = "LTTNG_RELAYD_THREAD_LIVE_DISPATCHER_TP_FAIL"; + + if (check_env_var(var)) { + return 1; + } + + return 0; +} + +int __testpoint_relayd_thread_live_worker(void) +{ + const char *var = "LTTNG_RELAYD_THREAD_LIVE_WORKER_TP_FAIL"; + + if (check_env_var(var)) { + return 1; + } + + return 0; +} + +int __testpoint_relayd_thread_live_listener(void) +{ + const char *var = "LTTNG_RELAYD_THREAD_LIVE_LISTENER_TP_FAIL"; + + if (check_env_var(var)) { + return 1; + } + + return 0; +} diff --git a/tests/regression/tools/health/health_stall.c b/tests/regression/tools/health/health_stall.c index 455e51913..4be980934 100644 --- a/tests/regression/tools/health/health_stall.c +++ b/tests/regression/tools/health/health_stall.c @@ -1,5 +1,6 @@ /* * Copyright (C) 2012 - Christian Babeux + * Copyright (C) 2014 - Mathieu Desnoyers * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License, version 2 only, as @@ -17,8 +18,9 @@ #include #include -#include +#include #include +#include #define STALL_TIME 60 @@ -26,6 +28,7 @@ * Check if the specified environment variable is set. * Return 1 if set, otherwise 0. */ +static int check_env_var(const char *env) { if (env) { @@ -38,43 +41,226 @@ int check_env_var(const char *env) return 0; } -int __testpoint_sessiond_thread_manage_clients_before_loop(void) +static +void do_stall(void) +{ + unsigned int sleep_time = STALL_TIME; + + while (sleep_time > 0) { + sleep_time = sleep(sleep_time); + } +} + +/* Session daemon */ + +int __testpoint_sessiond_thread_manage_clients(void) { const char *var = "LTTNG_SESSIOND_THREAD_MANAGE_CLIENTS_STALL"; if (check_env_var(var)) { - unsigned int sleep_time = STALL_TIME; - while (sleep_time > 0) { - sleep_time = sleep(sleep_time); - } + do_stall(); } return 0; } -int __testpoint_sessiond_thread_manage_kernel_before_loop(void) +int __testpoint_sessiond_thread_registration_apps(void) { - const char *var = "LTTNG_SESSIOND_THREAD_MANAGE_KERNEL_STALL"; + const char *var = "LTTNG_SESSIOND_THREAD_REG_APPS_STALL"; if (check_env_var(var)) { - unsigned int sleep_time = STALL_TIME; - while (sleep_time > 0) { - sleep_time = sleep(sleep_time); - } + do_stall(); } return 0; } -int __testpoint_sessiond_thread_manage_apps_before_loop(void) +int __testpoint_sessiond_thread_manage_apps(void) { const char *var = "LTTNG_SESSIOND_THREAD_MANAGE_APPS_STALL"; if (check_env_var(var)) { - unsigned int sleep_time = STALL_TIME; - while (sleep_time > 0) { - sleep_time = sleep(sleep_time); - } + do_stall(); + } + + return 0; +} + +int __testpoint_sessiond_thread_manage_kernel(void) +{ + const char *var = "LTTNG_SESSIOND_THREAD_MANAGE_KERNEL_STALL"; + + if (check_env_var(var)) { + do_stall(); + } + + return 0; +} + +int __testpoint_sessiond_thread_manage_consumer(void) +{ + const char *var = "LTTNG_SESSIOND_THREAD_MANAGE_CONSUMER_STALL"; + + if (check_env_var(var)) { + do_stall(); + } + + return 0; +} + +int __testpoint_sessiond_thread_ht_cleanup(void) +{ + const char *var = "LTTNG_SESSIOND_THREAD_HT_CLEANUP_STALL"; + + if (check_env_var(var)) { + do_stall(); + } + + return 0; +} + +int __testpoint_sessiond_thread_app_manage_notify(void) +{ + const char *var = "LTTNG_SESSIOND_THREAD_APP_MANAGE_NOTIFY_STALL"; + + if (check_env_var(var)) { + do_stall(); + } + + return 0; +} + +int __testpoint_sessiond_thread_app_reg_dispatch(void) +{ + const char *var = "LTTNG_SESSIOND_THREAD_APP_REG_DISPATCH_STALL"; + + if (check_env_var(var)) { + do_stall(); + } + + return 0; +} + +/* Consumer daemon */ + +int __testpoint_consumerd_thread_channel(void) +{ + const char *var = "LTTNG_CONSUMERD_THREAD_CHANNEL_STALL"; + + if (check_env_var(var)) { + do_stall(); + } + + return 0; +} + +int __testpoint_consumerd_thread_metadata(void) +{ + const char *var = "LTTNG_CONSUMERD_THREAD_METADATA_STALL"; + + if (check_env_var(var)) { + do_stall(); + } + + return 0; +} + +int __testpoint_consumerd_thread_data(void) +{ + const char *var = "LTTNG_CONSUMERD_THREAD_DATA_STALL"; + + if (check_env_var(var)) { + do_stall(); + } + + return 0; +} + +int __testpoint_consumerd_thread_sessiond(void) +{ + const char *var = "LTTNG_CONSUMERD_THREAD_SESSIOND_STALL"; + + if (check_env_var(var)) { + do_stall(); + } + + return 0; +} + +int __testpoint_consumerd_thread_metadata_timer(void) +{ + const char *var = "LTTNG_CONSUMERD_THREAD_METADATA_TIMER_STALL"; + + if (check_env_var(var)) { + do_stall(); + } + + return 0; +} + +/* Relay daemon */ + +int __testpoint_relayd_thread_dispatcher(void) +{ + const char *var = "LTTNG_RELAYD_THREAD_DISPATCHER_STALL"; + + if (check_env_var(var)) { + do_stall(); + } + + return 0; +} + +int __testpoint_relayd_thread_worker(void) +{ + const char *var = "LTTNG_RELAYD_THREAD_WORKER_STALL"; + + if (check_env_var(var)) { + do_stall(); + } + + return 0; +} + +int __testpoint_relayd_thread_listener(void) +{ + const char *var = "LTTNG_RELAYD_THREAD_LISTENER_STALL"; + + if (check_env_var(var)) { + do_stall(); + } + + return 0; +} + +int __testpoint_relayd_thread_live_dispatcher(void) +{ + const char *var = "LTTNG_RELAYD_THREAD_LIVE_DISPATCHER_STALL"; + + if (check_env_var(var)) { + do_stall(); + } + + return 0; +} + +int __testpoint_relayd_thread_live_worker(void) +{ + const char *var = "LTTNG_RELAYD_THREAD_LIVE_WORKER_STALL"; + + if (check_env_var(var)) { + do_stall(); + } + + return 0; +} + +int __testpoint_relayd_thread_live_listener(void) +{ + const char *var = "LTTNG_RELAYD_THREAD_LIVE_LISTENER_STALL"; + + if (check_env_var(var)) { + do_stall(); } return 0; diff --git a/tests/regression/tools/health/test_health.sh b/tests/regression/tools/health/test_health.sh new file mode 100644 index 000000000..6ae8885d0 --- /dev/null +++ b/tests/regression/tools/health/test_health.sh @@ -0,0 +1,310 @@ +# Copyright (C) - 2012 Christian Babeux +# Copyright (C) - 2014 Mathieu Desnoyers +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License, version 2 only, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# this program; if not, write to the Free Software Foundation, Inc., 51 +# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +TESTDIR=${CURDIR}/../../.. +LTTNG_BIN="lttng" +UST_EVENT_NAME="tp:tptest" +KERNEL_EVENT_NAME="sched_switch" +CHANNEL_NAME="testchan" +HEALTH_CHECK_BIN="health_check" +NUM_TESTS=90 +SLEEP_TIME=30 + +source $TESTDIR/utils/utils.sh + +if [ ! -f "$CURDIR/$SESSIOND_PRELOAD" ]; then + BAIL_OUT "${CURDIR}/${SESSIOND_PRELOAD} is missing." +fi + +function lttng_create_session_uri +{ + # Create session with default path + $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $SESSION_NAME \ + -U net://localhost >/dev/null 2>&1 + ok $? "Create session with default path" +} + +function report_errors +{ + test_thread_error_string="$1" + test_relayd="$2" + err_no_relayd_match="Error querying relayd health" + + # Check for health errors + # Include inability to contact relayd health as an expected + # error, since this can happen whenever the relayd shutdown due + # to an error in any thread. + out=$(grep "${test_thread_error_string}" ${STDOUT_PATH} | wc -l) + if [ $test_relayd -ne 0 ]; then + outerr=$(grep "${err_no_relayd_match}" ${STDERR_PATH} | wc -l) + else + outerr=0 + fi + if [ $out -eq 0 ] && [ $outerr -eq 0 ]; then + fail "Validation failure" + diag "Health returned:" + diag "stdout:" + file=${STDOUT_PATH} + while read line ; do + diag "$line" + done < ${file} + + diag "stderr:" + file=${STDERR_PATH} + while read line ; do + diag "$line" + done < ${file} + else + pass "Validation OK" + fi +} + +function test_health +{ + test_suffix="$1" + test_thread_name="$2" + test_thread_error_string="$3" + test_needs_root="$4" + test_consumerd="$5" + test_relayd="$6" + + diag "Test health problem detection with ${test_thread_name}" + + # Set the socket timeout to 5 so the health check detection + # happens within 25 s + export LTTNG_NETWORK_SOCKET_TIMEOUT=5 + export LTTNG_RELAYD_HEALTH="${HEALTH_PATH}/test-health" + + # Activate testpoints + export LTTNG_TESTPOINT_ENABLE=1 + + # Activate specific thread test + export ${test_thread_name}_${test_suffix}=1 + + # Spawn sessiond with preload healthexit lib + export LD_PRELOAD="$CURDIR/$SESSIOND_PRELOAD" + + diag "Start session daemon" + start_lttng_sessiond + + if [ ${test_consumerd} -eq 1 ]; then + create_lttng_session_no_output $SESSION_NAME + + diag "With UST consumer daemons" + enable_ust_lttng_event $SESSION_NAME $UST_EVENT_NAME $CHANNEL_NAME + + skip $isroot "Root access is needed. Skipping kernel consumer health check test." "1" || + { + diag "With kernel consumer daemon" + lttng_enable_kernel_event $SESSION_NAME $KERNEL_EVENT_NAME $CHANNEL_NAME + } + start_lttng_tracing $SESSION_NAME + fi + + if [ ${test_relayd} -eq 1 ]; then + diag "With relay daemon" + RELAYD_ARGS="--relayd-path=${LTTNG_RELAYD_HEALTH}" + + start_lttng_relayd "-o $TRACE_PATH" + else + RELAYD_ARGS= + fi + + # Check health status, not caring about result + $CURDIR/$HEALTH_CHECK_BIN ${RELAYD_ARGS} \ + > /dev/null + + # Wait + diag "Check after running for ${SLEEP_TIME} seconds" + sleep ${SLEEP_TIME} + + # Check health status + $CURDIR/$HEALTH_CHECK_BIN ${RELAYD_ARGS} \ + > ${STDOUT_PATH} 2> ${STDERR_PATH} + + + if [ ${test_needs_root} -eq 1 ]; then + skip ${isroot} "Root access needed for test \"${test_thread_name}\"." "1" || + { + report_errors "${test_thread_error_string}" "${test_relayd}" + } + else + report_errors "${test_thread_error_string}" "${test_relayd}" + fi + + if [ ${test_relayd} -eq 1 ]; then + stop_lttng_relayd_nocheck + fi + stop_lttng_sessiond + + unset LTTNG_TESTPOINT_ENABLE + unset ${test_thread_name}_${test_suffix} + unset LD_PRELOAD + unset LTTNG_NETWORK_SOCKET_TIMEOUT + unset LTTNG_RELAYD_HEALTH +} + +plan_tests $NUM_TESTS + +print_test_banner "$TEST_DESC" + +THREAD=("LTTNG_SESSIOND_THREAD_MANAGE_CLIENTS" + "LTTNG_SESSIOND_THREAD_MANAGE_APPS" + "LTTNG_SESSIOND_THREAD_REG_APPS" + "LTTNG_SESSIOND_THREAD_HT_CLEANUP" + "LTTNG_SESSIOND_THREAD_APP_MANAGE_NOTIFY" + "LTTNG_SESSIOND_THREAD_APP_REG_DISPATCH" + "LTTNG_SESSIOND_THREAD_MANAGE_KERNEL" + + "LTTNG_CONSUMERD_THREAD_CHANNEL" + "LTTNG_CONSUMERD_THREAD_METADATA" + "LTTNG_CONSUMERD_THREAD_METADATA_TIMER" + + "LTTNG_RELAYD_THREAD_DISPATCHER" + "LTTNG_RELAYD_THREAD_WORKER" + "LTTNG_RELAYD_THREAD_LISTENER" + "LTTNG_RELAYD_THREAD_LIVE_DISPATCHER" + "LTTNG_RELAYD_THREAD_LIVE_WORKER" + "LTTNG_RELAYD_THREAD_LIVE_LISTENER" +) + +ERROR_STRING=( + "Thread \"Session daemon command\" is not responding in component \"sessiond\"." + "Thread \"Session daemon application manager\" is not responding in component \"sessiond\"." + "Thread \"Session daemon application registration\" is not responding in component \"sessiond\"." + "Thread \"Session daemon hash table cleanup\" is not responding in component \"sessiond\"." + "Thread \"Session daemon application notification manager\" is not responding in component \"sessiond\"." + "Thread \"Session daemon application registration dispatcher\" is not responding in component \"sessiond\"." + "Thread \"Session daemon kernel\" is not responding in component \"sessiond\"." + + "Thread \"Consumer daemon channel\" is not responding" + "Thread \"Consumer daemon metadata\" is not responding" + "Thread \"Consumer daemon metadata timer\" is not responding" + + "Thread \"Relay daemon dispatcher\" is not responding in component \"relayd\"." + "Thread \"Relay daemon worker\" is not responding in component \"relayd\"." + "Thread \"Relay daemon listener\" is not responding in component \"relayd\"." + "Thread \"Relay daemon live dispatcher\" is not responding in component \"relayd\"." + "Thread \"Relay daemon live worker\" is not responding in component \"relayd\"." + "Thread \"Relay daemon live listener\" is not responding in component \"relayd\"." +) + +# TODO +# "LTTNG_SESSIOND_THREAD_MANAGE_CONSUMER" +# "Thread \"Session daemon manage consumer\" is not responding in component \"sessiond\"." + +# TODO: test kernel consumerd specifically in addition to UST consumerd + +# TODO: need refactoring of consumerd teardown +# "LTTNG_CONSUMERD_THREAD_SESSIOND" +# "Thread \"Consumer daemon session daemon command manager\" is not responding" + +# TODO: this thread is responsible for close a file descriptor that +# triggers teardown of metadata thread. We should revisit teardown of +# consumerd. +# "LTTNG_CONSUMERD_THREAD_DATA" +# "Thread \"Consumer daemon data\" is not responding" + +NEEDS_ROOT=( + 0 + 0 + 0 + 0 + 0 + 0 + 1 + + 0 + 0 + 0 + + 0 + 0 + 0 + 0 + 0 + 0 +) + +TEST_CONSUMERD=( + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + 1 + 1 + 1 + + 1 + 1 + 1 + 1 + 1 + 1 +) + +TEST_RELAYD=( + 0 + 0 + 0 + 0 + 0 + 0 + 0 + + 0 + 0 + 0 + + 1 + 1 + 1 + 1 + 1 + 1 +) + +STDOUT_PATH=$(mktemp) +STDERR_PATH=$(mktemp) +TRACE_PATH=$(mktemp -d) +HEALTH_PATH=$(mktemp -d) + +if [ "$(id -u)" == "0" ]; then + isroot=1 +else + isroot=0 +fi + +THREAD_COUNT=${#THREAD[@]} +i=0 +while [ "$i" -lt "$THREAD_COUNT" ]; do + test_health "${TEST_SUFFIX}" \ + "${THREAD[$i]}" \ + "${ERROR_STRING[$i]}" \ + "${NEEDS_ROOT[$i]}" \ + "${TEST_CONSUMERD[$i]}" \ + "${TEST_RELAYD[$i]}" + let "i++" +done + +rm -rf ${HEALTH_PATH} +rm -rf ${TRACE_PATH} +rm -f ${STDOUT_PATH} +rm -f ${STDERR_PATH} diff --git a/tests/regression/tools/health/test_thread_exit b/tests/regression/tools/health/test_thread_exit index 7622e356d..f4c84d483 100755 --- a/tests/regression/tools/health/test_thread_exit +++ b/tests/regression/tools/health/test_thread_exit @@ -1,6 +1,7 @@ #!/bin/bash # # Copyright (C) - 2012 Christian Babeux +# Copyright (C) - 2014 Mathieu Desnoyers # # This program is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License, version 2 only, as @@ -17,120 +18,9 @@ TEST_DESC="Health check - Thread exit" -CURDIR=$(dirname $0)/ -TESTDIR=$CURDIR/../../.. -LTTNG_BIN="lttng" SESSION_NAME="health_thread_exit" -EVENT_NAME="bogus" -HEALTH_CHECK_BIN="health_check" SESSIOND_PRELOAD=".libs/libhealthexit.so" -NUM_TESTS=12 +TEST_SUFFIX="EXIT" +CURDIR=$(dirname $(readlink -f $0)) -source $TESTDIR/utils/utils.sh - -if [ ! -f "$CURDIR/$SESSIOND_PRELOAD" ]; then - BAIL_OUT "libhealthexit.so not available for this test." -fi - -function test_thread_exit -{ - test_thread_exit_name="$1" - test_thread_error_string="$2" - - diag "Test health failure with ${test_thread_exit_name}" - - # Activate testpoints - export LTTNG_TESTPOINT_ENABLE=1 - - # Activate specific thread exit - export ${test_thread_exit_name}_EXIT=1 - - # Spawn sessiond with preload healthexit lib - export LD_PRELOAD="$CURDIR/$SESSIOND_PRELOAD" - # Set the socket timeout to 5 so the health check delta is set to 25. - export LTTNG_NETWORK_SOCKET_TIMEOUT=5 - start_lttng_sessiond - - # Cleanup some env. var. - unset LD_PRELOAD - unset ${test_thread_exit_name}_EXIT - - # Check initial health status - $CURDIR/$HEALTH_CHECK_BIN > /dev/null - - # Wait - sleep 30 - - # Check health status - $CURDIR/$HEALTH_CHECK_BIN > ${STDOUT_PATH} 2> ${STDERR_PATH} - - out=$(grep "${test_thread_error_string}" ${STDOUT_PATH} | wc -l) - if [ $out -eq 0 ]; then - fail "Validate thread ${test_thread_exit_name} failure" - diag "Health returned:" - diag "stdout:" - file=${STDOUT_PATH} - while read line ; do - diag "$line" - done < ${file} - - diag "stderr:" - file=${STDERR_PATH} - while read line ; do - diag "$line" - done < ${file} - - stop_lttng_sessiond - return 1 - else - pass "Validate thread ${test_thread_exit_name} failure" - stop_lttng_sessiond - fi -} - -plan_tests $NUM_TESTS - -print_test_banner "$TEST_DESC" - -THREAD=("LTTNG_SESSIOND_THREAD_MANAGE_CLIENTS" - "LTTNG_SESSIOND_THREAD_MANAGE_APPS" - "LTTNG_SESSIOND_THREAD_REG_APPS") - -ERROR_STRING=( - "Thread \"Session daemon command\" is not responding in component \"sessiond\"." - "Thread \"Session daemon application manager\" is not responding in component \"sessiond\"." - "Thread \"Session daemon application registration\" is not responding in component \"sessiond\"." -) - -STDOUT_PATH=$(mktemp) -STDERR_PATH=$(mktemp) - -THREAD_COUNT=${#THREAD[@]} -i=0 -while [ "$i" -lt "$THREAD_COUNT" ]; do - test_thread_exit "${THREAD[$i]}" "${ERROR_STRING[$i]}" - - if [ $? -eq 1 ]; then - exit 1 - fi - - let "i++" -done - -# The manage kernel thread is only spawned if we are root -if [ "$(id -u)" == "0" ]; then - isroot=1 -else - isroot=0 -fi - -skip $isroot "Root access is needed. Skipping LTTNG_SESSIOND_THREAD_MANAGE_KERNEL tests." "3" || -{ - test_thread_exit "LTTNG_SESSIOND_THREAD_MANAGE_KERNEL" "Thread \"Session daemon kernel\" is not responding in component \"sessiond\"." -} - -rm -f ${STDOUT_PATH} -rm -f ${STDERR_PATH} - -# TODO: Special case manage consumer, need to spawn consumer via commands. -#"LTTNG_SESSIOND_THREAD_MANAGE_CONSUMER" +source ${CURDIR}/test_health.sh diff --git a/tests/regression/tools/health/test_thread_ok b/tests/regression/tools/health/test_thread_ok index 59b3f5764..add9c0ced 100755 --- a/tests/regression/tools/health/test_thread_ok +++ b/tests/regression/tools/health/test_thread_ok @@ -110,7 +110,7 @@ function test_thread_ok report_errors # Wait - diag "Check after running for 30 seconds" + diag "Check after running for ${SLEEP_TIME} seconds" sleep ${SLEEP_TIME} # Check health status diff --git a/tests/regression/tools/health/test_thread_stall b/tests/regression/tools/health/test_thread_stall index a3d582525..93144ac88 100755 --- a/tests/regression/tools/health/test_thread_stall +++ b/tests/regression/tools/health/test_thread_stall @@ -1,6 +1,7 @@ #!/bin/bash # # Copyright (C) - 2012 Christian Babeux +# Copyright (C) - 2014 Mathieu Desnoyers # # This program is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License, version 2 only, as @@ -17,148 +18,9 @@ TEST_DESC="Health check - Thread stall" -CURDIR=$(dirname $0)/ -TESTDIR=$CURDIR/../../.. -LTTNG_BIN="lttng" SESSION_NAME="health_thread_stall" -EVENT_NAME="bogus" -HEALTH_CHECK_BIN="health_check" SESSIOND_PRELOAD=".libs/libhealthstall.so" -NUM_TESTS=12 +TEST_SUFFIX="STALL" +CURDIR=$(dirname $(readlink -f $0)) -source $TESTDIR/utils/utils.sh - -if [ ! -f "$CURDIR/$SESSIOND_PRELOAD" ]; then - BAIL_OUT "libhealthstall.so not available for this test." -fi - -function test_thread_stall -{ - test_thread_stall_name="$1" - test_thread_exit_code="$2" - - diag "Test health failure with ${test_thread_stall_name}" - - # Activate testpoints - export LTTNG_TESTPOINT_ENABLE=1 - - # Activate specific thread exit - export ${test_thread_stall_name}_STALL=1 - - # Spawn sessiond with preload healthexit lib - export LD_PRELOAD="$CURDIR/$SESSIOND_PRELOAD" - # Set the socket timeout to 5 so the health check delta is set to 25. - export LTTNG_NETWORK_SOCKET_TIMEOUT=5 - start_lttng_sessiond - - # Cleanup some env. var. - unset LD_PRELOAD - unset ${test_thread_stall_name}_STALL - - # Check initial health status - $CURDIR/$HEALTH_CHECK_BIN > /dev/null - - # Wait - sleep 30 - - # Check health status, should indicate failure - $CURDIR/$HEALTH_CHECK_BIN > ${STDOUT_PATH} 2> ${STDERR_PATH} - - out=$(grep "${test_thread_error_string}" ${STDOUT_PATH} | wc -l) - if [ $out -eq 0 ]; then - fail "Validate thread ${test_thread_stall_name} is stalled" - diag "Health returned:" - diag "stdout:" - file=${STDOUT_PATH} - while read line ; do - diag "$line" - done < ${file} - - diag "stderr:" - file=${STDERR_PATH} - while read line ; do - diag "$line" - done < ${file} - - stop_lttng_sessiond - return 1 - else - pass "Validate thread ${test_thread_stall_name} is stalled" - fi - - # Wait - sleep 40 - - # Check health status, should now pass - $CURDIR/$HEALTH_CHECK_BIN > ${STDOUT_PATH} 2> ${STDERR_PATH} - - out=$(grep "${test_thread_error_string}" ${STDOUT_PATH} | wc -l) - if [ $out -ne 0 ]; then - fail "Validate thread ${test_thread_stall_name} is not longer stalled" - diag "Health returned:" - diag "stdout:" - file=${STDOUT_PATH} - while read line ; do - diag "$line" - done < ${file} - - diag "stderr:" - file=${STDERR_PATH} - while read line ; do - diag "$line" - done < ${file} - - stop_lttng_sessiond - return 1 - else - pass "Validate thread ${test_thread_stall_name} is not longer stalled" - stop_lttng_sessiond - fi -} - -plan_tests $NUM_TESTS - -print_test_banner "$TEST_DESC" - -THREAD=("LTTNG_SESSIOND_THREAD_MANAGE_CLIENTS" - "LTTNG_SESSIOND_THREAD_MANAGE_APPS" -# This thread is a little bit tricky to stall, -# need to send some commands and setup an app. -# "LTTNG_SESSIOND_THREAD_REG_APPS" -) - -ERROR_STRING=( - "Thread \"Session daemon command\" is not responding in component \"sessiond\"." - "Thread \"Session daemon application manager\" is not responding in component \"sessiond\"." - "Thread \"Session daemon application registration\" is not responding in component \"sessiond\"." -) - -STDOUT_PATH=$(mktemp) -STDERR_PATH=$(mktemp) - -THREAD_COUNT=${#THREAD[@]} -i=0 -while [ "$i" -lt "$THREAD_COUNT" ]; do - test_thread_stall "${THREAD[$i]}" "${ERROR_STRING[$i]}" - - if [ $? -eq 1 ]; then - exit 1 - fi - - let "i++" -done - -# The manage kernel thread is only spawned if we are root -if [ "$(id -u)" == "0" ]; then - isroot=1 -else - isroot=0 -fi - -skip $isroot "Root access is needed. Skipping LTTNG_SESSIOND_THREAD_MANAGE_KERNEL tests." "4" || -{ - test_thread_stall "LTTNG_SESSIOND_THREAD_MANAGE_KERNEL" "Thread \"Session daemon kernel\" is not responding in component \"sessiond\"." -} - -rm -f ${STDOUT_PATH} -rm -f ${STDERR_PATH} +source ${CURDIR}/test_health.sh diff --git a/tests/regression/tools/health/test_tp_fail b/tests/regression/tools/health/test_tp_fail index 0af69b843..1f1529813 100755 --- a/tests/regression/tools/health/test_tp_fail +++ b/tests/regression/tools/health/test_tp_fail @@ -1,6 +1,7 @@ #!/bin/bash # # Copyright (C) - 2012 Christian Babeux +# Copyright (C) - 2014 Mathieu Desnoyers # # This program is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License, version 2 only, as @@ -17,111 +18,9 @@ TEST_DESC="Health check - Testpoint failure" -CURDIR=$(dirname $0)/ -TESTDIR=$CURDIR/../../.. -LTTNG_BIN="lttng" SESSION_NAME="health_tp_fail" -EVENT_NAME="bogus" -HEALTH_CHECK_BIN="health_check" SESSIOND_PRELOAD=".libs/libhealthtpfail.so" -NUM_TESTS=9 +TEST_SUFFIX="TP_FAIL" +CURDIR=$(dirname $(readlink -f $0)) -source $TESTDIR/utils/utils.sh - -if [ ! -f "$CURDIR/$SESSIOND_PRELOAD" ]; then - BAIL_OUT "libhealthtpfail.so not available for this test." -fi - -function test_tp_fail -{ - test_tp_fail_name="$1" - test_tp_error_string="$2" - - diag "Test health failure with ${test_tp_fail_name}" - - # Activate testpoints - export LTTNG_TESTPOINT_ENABLE=1 - - # Activate specific testpoint failure - export ${test_tp_fail_name}_TP_FAIL=1 - - # Spawn sessiond with preload healthexit lib - export LD_PRELOAD="$CURDIR/$SESSIOND_PRELOAD" - start_lttng_sessiond - - # Cleanup some env. var. - unset LD_PRELOAD - unset ${test_tp_fail_name}_TP_FAIL - - # Check health status - $CURDIR/$HEALTH_CHECK_BIN > ${STDOUT_PATH} 2> ${STDERR_PATH} - - out=$(grep "${test_tp_error_string}" ${STDOUT_PATH} | wc -l) - if [ $out -eq 0 ]; then - fail "Validate thread ${test_tp_fail_name} failure" - diag "Health returned:" - - diag "stdout:" - file=${STDOUT_PATH} - while read line ; do - diag "$line" - done < ${file} - - diag "stderr:" - file=${STDERR_PATH} - while read line ; do - diag "$line" - done < ${file} - - stop_lttng_sessiond - return 1 - else - pass "Validate thread ${test_tp_fail_name} failure" - stop_lttng_sessiond - fi -} - -plan_tests $NUM_TESTS - -print_test_banner "$TEST_DESC" - -THREAD=("LTTNG_SESSIOND_THREAD_MANAGE_CLIENTS" - "LTTNG_SESSIOND_THREAD_MANAGE_APPS") - -ERROR_STRING=( - "Thread \"Session daemon command\" is not responding in component \"sessiond\"." - "Thread \"Session daemon application manager\" is not responding in component \"sessiond\"." -) - -STDOUT_PATH=$(mktemp) -STDERR_PATH=$(mktemp) - -THREAD_COUNT=${#THREAD[@]} -i=0 -while [ "$i" -lt "$THREAD_COUNT" ]; do - test_tp_fail "${THREAD[$i]}" "${ERROR_STRING[$i]}" - - if [ $? -eq 1 ]; then - exit 1 - fi - - let "i++" -done - -# The manage kernel thread is only spawned if we are root -if [ "$(id -u)" == "0" ]; then - isroot=1 -else - isroot=0 -fi - -skip $isroot "Root access is needed. Skipping LTTNG_SESSIOND_THREAD_MANAGE_KERNEL tests." "3" || -{ - test_tp_fail "LTTNG_SESSIOND_THREAD_MANAGE_KERNEL" "Thread \"Session daemon kernel\" is not responding in component \"sessiond\"." -} - -rm -f ${STDOUT_PATH} -rm -f ${STDERR_PATH} - -# TODO: Special case manage consumer, need to spawn consumer via commands. -#"LTTNG_SESSIOND_THREAD_MANAGE_CONSUMER" +source ${CURDIR}/test_health.sh -- 2.34.1