From: Jonathan Rajotte <jonathan.rajotte-julien@efficios.com>
Date: Thu, 11 Oct 2018 18:44:27 +0000 (-0400)
Subject: Check for hanging process at the end of a job.
X-Git-Url: https://git.lttng.org./?a=commitdiff_plain;h=0a028cf67adaf9977e910161800ef800b72f2426;p=lttng-ci.git

Check for hanging process at the end of a job.

Run in all scenarios. We force a coredump and archive it to ./build.

Use ldd on /proc/$PID/exe to get actual dependencies.

Signed-off-by: Jonathan Rajotte <jonathan.rajotte-julien@efficios.com>
---

diff --git a/jobs/lttng-tools.yaml b/jobs/lttng-tools.yaml
index a747139..b5d55b1 100644
--- a/jobs/lttng-tools.yaml
+++ b/jobs/lttng-tools.yaml
@@ -281,6 +281,19 @@
 - lttng-tools_build_publishers_dev: &lttng-tools_build_publishers_dev
     name: 'lttng-tools_build_publishers_dev'
     publishers:
+      - postbuildscript:
+          mark-unstable-if-failed: true
+          builders:
+            - role: SLAVE
+              build-on:
+                  - SUCCESS
+                  - UNSTABLE
+                  - NOT_BUILT
+                  - ABORTED
+                  - FAILURE
+              build-steps:
+                  - shell:
+                      !include-raw-escape: scripts/lttng-tools/hang_processes.sh
       - tap:
           results: 'tap/**/*.tap'
           failed-tests-mark-build-as-failure: true
@@ -322,6 +335,19 @@
 - lttng-tools_build_publishers_prod: &lttng-tools_build_publishers_prod
     name: 'lttng-tools_build_publishers_prod'
     publishers:
+      - postbuildscript:
+          mark-unstable-if-failed: true
+          builders:
+            - role: SLAVE
+              build-on:
+                  - SUCCESS
+                  - UNSTABLE
+                  - NOT_BUILT
+                  - ABORTED
+                  - FAILURE
+              build-steps:
+                  - shell:
+                      !include-raw-escape: scripts/lttng-tools/hang_processes.sh
       - tap:
           results: 'tap/**/*.tap'
           failed-tests-mark-build-as-failure: true
diff --git a/scripts/lttng-tools/build.sh b/scripts/lttng-tools/build.sh
index 052eb8d..774f535 100755
--- a/scripts/lttng-tools/build.sh
+++ b/scripts/lttng-tools/build.sh
@@ -342,8 +342,8 @@ if [ "$RUN_TESTS" = "yes" ]; then
     ulimit -c unlimited
 
     # Kill any LTTng-related process
-    lttng_processes="$("$PGREP" -l 'lttng|gen-ust-.+')"
-    if [ $? -eq 0 ]; then
+    lttng_processes="$("$PGREP" -l 'lttng|gen-ust-.+')" || true
+    if [ ! -z "$lttng_processes" ]; then
         echo "The following LTTng processes were detected running on the system and will be killed:"
         echo "$lttng_processes"
 
diff --git a/scripts/lttng-tools/hang_processes.sh b/scripts/lttng-tools/hang_processes.sh
new file mode 100755
index 0000000..d64fa9c
--- /dev/null
+++ b/scripts/lttng-tools/hang_processes.sh
@@ -0,0 +1,68 @@
+#!/bin/bash -exu
+#
+# Copyright (C) 2018 - Jonathan Rajotte-Julien <jonathan.rajotte-julien@efficios.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+PGREP=pgrep
+pids=""
+dependencies=""
+file_list=$(mktemp)
+ret=0
+
+lttng_processes="$("$PGREP" -l 'lttng|gen-ust-.+')" || true
+if [ ! -z "$lttng_processes" ]; then
+
+    pids="$(cut -d ' ' -f 1 <<< "$lttng_processes" | tr '\n' ' ')"
+    comma_pids=$(tr ' ' ',' <<< "$pids")
+    echo "The following LTTng processes were detected running on the system and will be aborted:"
+    echo "$lttng_processes"
+
+    # Stop the processes to make sure everything is frozen
+    kill -SIGSTOP $pids
+
+    # Get dependencies for coredump analysis
+    # Use /proc/$PID/exe and ldd to get all shared libs necessary
+    array=(${pids})
+    # Add the /proc/ prefix using parameter expansion
+    array=("${array[@]/#/\/proc\/}")
+    # Add the /exe suffix using parameter expansion
+    array=("${array[@]/%/\/exe}")
+    dependencies=$(ldd "${array[@]}" | grep -v "not found")
+    dependencies=$(awk '/=>/{print$(NF-1)}' <<< "$dependencies" | sort | uniq)
+
+    kill -SIGABRT $pids
+    kill -SIGCONT $pids
+    ret=1
+fi
+
+core_files=$(find "/tmp" -name "core\.[0-9]*" -type f 2>/dev/null) || true
+if [ ! -z "$core_files" ]; then
+    echo "$core_files" >> "$file_list"
+    echo "$dependencies" >> "$file_list"
+
+    # Make sure the coredump is finished using fuser
+    for core in $core_files; do
+        while fuser "$core"; do
+            sleep 1
+        done
+    done
+
+    tar cfzh "${WORKSPACE}/build/core.tar.gz" -T "$file_list"
+    rm -rf $core_files
+    ret=1
+fi
+
+rm -rf "$file_list"
+exit $ret