X-Git-Url: http://git.lttng.org./?a=blobdiff_plain;f=scripts%2Fsystem-tests%2Frun-test-suites.sh;h=62e956db6372dca20cbe66a0c5d905a9d0b8572c;hb=4955549e270f70e6403bb44fb16016dade5e18c7;hp=2637bc44c1efc018fe6346b90a7c5c0e17ddce47;hpb=9ac936bcec3207cb1e23491909c9c1668033ba95;p=lttng-ci.git

diff --git a/scripts/system-tests/run-test-suites.sh b/scripts/system-tests/run-test-suites.sh
index 2637bc4..62e956d 100755
--- a/scripts/system-tests/run-test-suites.sh
+++ b/scripts/system-tests/run-test-suites.sh
@@ -68,16 +68,78 @@ verne() {
     [ "$res" -ne "0" ]
 }
 
+# shellcheck disable=SC2317
+function cleanup
+{
+    timedatectl set-ntp true
+    # The false dates used in the tests are far in the past
+    # and it may take some time for the ntp update to actually
+    # happen.
+    # If the date is still in the past, it is possible that
+    # subsequent steps will fail (eg. TLS certificates cannot
+    # be validated).
+    while [[ "$(date +%Y)" -lt "2024" ]] ; do
+        sleep 1
+    done
+}
+
+function test_timeout
+{
+    local TIMEOUT=0
+    local TIMEOUT_MINUTES="${1:-90}"
+    shift 1
+    PID=''
+    "${@}" &
+    PID="${!}"
+    while true; do
+        sleep 1m
+        if ! ps -q "${PID}" > /dev/null ; then
+            # The process ID doesn't exist anymore
+            break
+        fi
+        TIMEOUT=$((TIMEOUT+1))
+        if [[ "${TIMEOUT}" -ge "${TIMEOUT_MINUTES}" ]]; then
+            echo "Command '${@}' timed out (${TIMEOUT} minutes) " \
+                 "attempting to get backtraces for lttng/babeltrace binaries"
+            apt-get install -y --force-yes gdb
+            # Abort all lttng-sessiond, lttng, lttng-relayd, lttng-consumerd,
+            # and babeltrace process so there are coredumps available.
+            PIDS=$(pgrep 'babeltrace*|[l]ttng*')
+            for P in ${PIDS}; do
+                OUTFILE=$(mktemp -t "backtrace-${P}.XXXXXX")
+                ps -f "${P}" | tee -a "${OUTFILE}"
+                gdb -p "${P}" --batch -ex 'thread apply all bt' 2>&1 | tee -a "${OUTFILE}"
+                mv "${OUTFILE}" /tmp/coredump/
+            done
+
+            # Send sigterm to make
+            kill "${PID}"
+
+            # Cleanup, to hopefully not interfere with future tests
+            apt-get purge -y gdb
+            apt-get autoremove -y
+        fi
+    done
+    wait "${PID}"
+    return "${?}"
+}
+
+trap cleanup EXIT SIGINT SIGTERM
+
 lttng_version="$1"
 failed_tests=0
 
 export LTTNG_ENABLE_DESTRUCTIVE_TESTS="will-break-my-system"
+timedatectl set-ntp false
 
-make --keep-going check || failed_tests=1
+# When make check is interrupted, the default test driver
+# (`config/test-driver`) will still delete the log and trs
+# files for the currently running test.
+test_timeout 90 make --keep-going check || failed_tests=1
 
 if [ -f "./tests/root_regression" ]; then
     cd "./tests" || exit 1
-    prove --nocolor --verbose --merge --exec '' - < root_regression || failed_tests=1
+    prove --nocolor --verbose --merge --exec '' - < root_regression || failed_tests=2
     cd ..
 fi
 
@@ -85,7 +147,7 @@ fi
 # should be retained until lttng-tools 2.13 is no longer supported
 if [ -f "./tests/root_destructive_tests" ]; then
     cd "./tests" || exit 1
-    prove --nocolor --verbose --merge --exec '' - < root_destructive_tests || failed_tests=2
+    prove --nocolor --verbose --merge --exec '' - < root_destructive_tests || failed_tests=3
     cd ..
 else
     echo 'root_destructive_tests not found'