X-Git-Url: http://git.lttng.org./?a=blobdiff_plain;f=scripts%2Fsystem-tests%2Frun-test-suites.sh;h=62e956db6372dca20cbe66a0c5d905a9d0b8572c;hb=4955549e270f70e6403bb44fb16016dade5e18c7;hp=62ab7d24dcca4abccb6ee03ebb104620d5083a10;hpb=d0ffdf18a1f18f7fd4eaac5882ab18d846b08725;p=lttng-ci.git

diff --git a/scripts/system-tests/run-test-suites.sh b/scripts/system-tests/run-test-suites.sh
index 62ab7d2..62e956d 100755
--- a/scripts/system-tests/run-test-suites.sh
+++ b/scripts/system-tests/run-test-suites.sh
@@ -68,9 +68,60 @@ verne() {
     [ "$res" -ne "0" ]
 }
 
+# shellcheck disable=SC2317
 function cleanup
 {
     timedatectl set-ntp true
+    # The false dates used in the tests are far in the past
+    # and it may take some time for the ntp update to actually
+    # happen.
+    # If the date is still in the past, it is possible that
+    # subsequent steps will fail (eg. TLS certificates cannot
+    # be validated).
+    while [[ "$(date +%Y)" -lt "2024" ]] ; do
+        sleep 1
+    done
+}
+
+function test_timeout
+{
+    local TIMEOUT=0
+    local TIMEOUT_MINUTES="${1:-90}"
+    shift 1
+    PID=''
+    "${@}" &
+    PID="${!}"
+    while true; do
+        sleep 1m
+        if ! ps -q "${PID}" > /dev/null ; then
+            # The process ID doesn't exist anymore
+            break
+        fi
+        TIMEOUT=$((TIMEOUT+1))
+        if [[ "${TIMEOUT}" -ge "${TIMEOUT_MINUTES}" ]]; then
+            echo "Command '${@}' timed out (${TIMEOUT} minutes) " \
+                 "attempting to get backtraces for lttng/babeltrace binaries"
+            apt-get install -y --force-yes gdb
+            # Abort all lttng-sessiond, lttng, lttng-relayd, lttng-consumerd,
+            # and babeltrace process so there are coredumps available.
+            PIDS=$(pgrep 'babeltrace*|[l]ttng*')
+            for P in ${PIDS}; do
+                OUTFILE=$(mktemp -t "backtrace-${P}.XXXXXX")
+                ps -f "${P}" | tee -a "${OUTFILE}"
+                gdb -p "${P}" --batch -ex 'thread apply all bt' 2>&1 | tee -a "${OUTFILE}"
+                mv "${OUTFILE}" /tmp/coredump/
+            done
+
+            # Send sigterm to make
+            kill "${PID}"
+
+            # Cleanup, to hopefully not interfere with future tests
+            apt-get purge -y gdb
+            apt-get autoremove -y
+        fi
+    done
+    wait "${PID}"
+    return "${?}"
 }
 
 trap cleanup EXIT SIGINT SIGTERM
@@ -84,8 +135,7 @@ timedatectl set-ntp false
 # When make check is interrupted, the default test driver
 # (`config/test-driver`) will still delete the log and trs
 # files for the currently running test.
-#
-timeout 90m make --keep-going check || failed_tests=1
+test_timeout 90 make --keep-going check || failed_tests=1
 
 if [ -f "./tests/root_regression" ]; then
     cd "./tests" || exit 1
@@ -107,5 +157,4 @@ if [[ "${failed_tests}" != "0" ]] ; then
     find tests/ -iname '*.trs' -print0 -or -iname '*.log' -print0 | tar czf /tmp/coredump/logs.tgz --null -T -
 fi
 
-timedatectl set-ntp true
 exit $failed_tests