From 946bf6a79b4a5433a09fc4259cbe72d42a6d9f73 Mon Sep 17 00:00:00 2001 From: Jonathan Rajotte Date: Wed, 17 Aug 2022 09:06:06 -0400 Subject: [PATCH] jjb: 32-64 integration pipeline Signed-off-by: Jonathan Rajotte Change-Id: I3d462140d238e9fcdc70d3c54c351faf6da1af97 --- jobs/integration.yaml | 96 ++++++ ...tools-_version_-32-64_testsuite.groovy.inc | 318 ++++++++++++++++++ scripts/integration/32-64/build.sh | 130 +++++++ .../integration/32-64/gather_tap_result.sh | 24 ++ 4 files changed, 568 insertions(+) create mode 100644 pipelines/integration/lttng-tools-_version_-32-64_testsuite.groovy.inc create mode 100644 scripts/integration/32-64/build.sh create mode 100644 scripts/integration/32-64/gather_tap_result.sh diff --git a/jobs/integration.yaml b/jobs/integration.yaml index fe93110..fb4cd90 100644 --- a/jobs/integration.yaml +++ b/jobs/integration.yaml @@ -4,6 +4,85 @@ project-type: pipeline dsl: !include-raw: pipelines/integration/ust-2.12-lower-urcu_testsuite.groovy +- job-template: + name: 'integration_lttng-tools-{version}-32-64_testsuite' + description: | + Integration pipeline for multi-bitness scenario (32/64). + This essentially tests frontier scenario where the bitness between component changes. + +

Job is managed by Jenkins Job Builder.

+ project-type: pipeline + dsl: !include-jinja2: pipelines/integration/lttng-tools-_version_-32-64_testsuite.groovy.inc + +- job: + name: integration_lttng-tools-32-64_test + project-type: freestyle + concurrent: true + node: 'amd64-rootnode' + wrappers: + - workspace-cleanup + - timestamps + - ansicolor + - timeout: + timeout: 6 + fail: true + type: no-activity + parameters: + - string: + name: 'ARTIFACT_ID' + default: 'null' + description: 'The ID of the artifact to download.' + - string: + name: 'BT_VERSION' + default: 'stable-2.0' + description: 'The version of babeltrace to use.' + + - choice: + name: 'TEST_TYPE' + description: 'The test to perform.' + choices: + - 32bit-sessiond + - 32bit-relayd + - 32bit-cli + - canary + builders: + - copyartifact: + project: 'babeltrace_${BT_VERSION}_build/platform=bionic-amd64,conf=std,build=std' + which-build: last-successful + stable: false + filter: 'build/**' + target: 'deps-64' + do-not-fingerprint: true + - shell: + !include-raw-escape: scripts/integration/32-64/build.sh + publishers: + - postbuildscript: + mark-unstable-if-failed: true + builders: + - role: SLAVE + build-on: + - SUCCESS + - UNSTABLE + - NOT_BUILT + - ABORTED + - FAILURE + build-steps: + - shell: + !include-raw-escape: scripts/integration/32-64/gather_tap_result.sh + - tap: + results: 'tap/**/*.*' + fail-if-no-results: true + failed-tests-mark-build-as-failure: true + todo-is-failure: false + output-tap-to-console: true + include-comment-diagnostics: true + plan-required: true + show-only-failures: true + verbose: true + - archive: + artifacts: 'tap/**' + allow-empty: false + ## Views - view-template: @@ -19,3 +98,20 @@ - 'Integration' jobs: - 'integration_ust-2.12-lower-urcu_testsuite' + +- project: + name: integration-32-64 + jobs: + - 'integration_lttng-tools-{version}-32-64_testsuite': + version: master + babelversion: stable-2.0 + urcu_version: master + - 'integration_lttng-tools-{version}-32-64_testsuite': + version: stable-2.13 + babelversion: stable-2.0 + urcu_version: stable-0.13 + - 'integration_lttng-tools-{version}-32-64_testsuite': + version: stable-2.12 + babelversion: stable-2.0 + urcu_version: stable-0.13 + - 'integration_lttng-tools-32-64_test' diff --git a/pipelines/integration/lttng-tools-_version_-32-64_testsuite.groovy.inc b/pipelines/integration/lttng-tools-_version_-32-64_testsuite.groovy.inc new file mode 100644 index 0000000..c6490d3 --- /dev/null +++ b/pipelines/integration/lttng-tools-_version_-32-64_testsuite.groovy.inc @@ -0,0 +1,318 @@ +#!groovy + +{% set test_job_name = 'integration_lttng-tools-32-64_test' %} + +pipeline { + agent none + + /* Global options for the pipeline */ + options { + preserveStashes() + buildDiscarder(logRotator(numToKeepStr: '5')) + disableConcurrentBuilds() + timestamps() + skipDefaultCheckout() + } + + triggers { + pollSCM('@hourly') + } + + parameters { + string(name: 'LTTNG_TOOLS_GIT_REPO', defaultValue: 'https://github.com/lttng/lttng-tools', description: 'The http git repo to be cloned for the lttng-tools project') + string(name: 'LTTNG_TOOLS_GIT_BRANCH', defaultValue: '{{ version }}', description: 'The {{ version }} equivalent branch to be checkout-ed') + } + + stages { + /* + * i386 artefact cannot be reused for 2 reason. + * The first one is that the amd64 and i386 node do not run the same distro and libc version. + * The second, is that we need to ensure that the lttng-ust used + * is the same for the amd64 and i386 and that for now we do not + * have this guarantee since we used 2 different jobs + */ + stage('Checkout all sources') { + agent { label 'bionic-amd64' } + steps { + dir("src/userspace-rcu") { + checkout([$class: 'GitSCM', branches: [[name: '{{ urcu_version }}']], extensions: [], gitTool: 'Default', userRemoteConfigs: [[url: 'https://github.com/urcu/userspace-rcu']]]) + } + dir("src/lttng-modules") { + checkout([$class: 'GitSCM', branches: [[name: '{{ version }}']], extensions: [], gitTool: 'Default', userRemoteConfigs: [[url: 'https://github.com/lttng/lttng-modules']]]) + } + dir("src/lttng-ust") { + checkout([$class: 'GitSCM', branches: [[name: '{{ version }}']], extensions: [], gitTool: 'Default', userRemoteConfigs: [[url: 'https://github.com/lttng/lttng-ust']]]) + } + dir("src/lttng-tools") { + checkout([$class: 'GitSCM', branches: [[name: "${params.LTTNG_TOOLS_GIT_BRANCH}"]], extensions: [], gitTool: 'Default', userRemoteConfigs: [[url: "${params.LTTNG_TOOLS_GIT_REPO}"]]]) + } + stash name: 'dep-sources', includes: 'src/userspace-rcu/**,src/lttng-ust/**' + stash name: 'modules-sources', includes: 'src/lttng-modules/**' + stash name: 'tools-sources', includes: 'src/lttng-tools/**,' + } + } + + stage('Build') { + parallel { + stage('Build amd64 deps') { + agent { label 'bionic-amd64' } + stages { + stage('Clean workspace') { + steps { + cleanWs() + } + } + + stage('Build') { + environment { + BASEDIR_64 = "$WORKSPACE/deps-64" + BASEDIR_BUILD_64 = "$BASEDIR_64/build" + INCLUDE_DIR_64 = "$BASEDIR_BUILD_64/include" + CPPFLAGS = "-I$INCLUDE_DIR_64" + LDFLAGS = "-L$BASEDIR_BUILD_64/lib" + PKG_CONFIG_PATH = "$BASEDIR_BUILD_64/lib/pkgconfig" + LD_LIBRARY_PATH ="$BASEDIR_BUILD_64/lib:$LD_LIBRARY_PATH" + PATH ="$PATH:$BASEDIR_64/bin" + PREFIX = "/build" + JAVA_HOME="/usr/lib/jvm/default-java" + CLASSPATH="/usr/share/java/log4j-api.jar:/usr/share/java/log4j-core.jar:/usr/share/java/log4j-1.2.jar" + PYTHON3="python3" + PYTHON_CONFIG="python3-config" + CXXFLAGS="-g -O0" + CFLAGS="-g -O0" + } + steps { + unstash name: 'dep-sources' + + /* Prevent non existent include dir warnings */ + sh 'mkdir -p $INCLUDE_DIR_64' + + dir("src/userspace-rcu") { + sh ''' + ./bootstrap + ./configure --prefix="$PREFIX" + make -j"$(nproc)" V=1 + make install DESTDIR="$WORKSPACE" + ''' + + sh ''' + find "$WORKSPACE/build/lib" -name "*.so" -exec chrpath --delete {} \\; + find "$WORKSPACE/build/lib" -name "*.la" -exec rm -vf {} \\; + ''' + } + + /* Move the artifact of the recently build project to the deps file and remove the "build" directory to prevent `make install` shenanigans for lttng-ust 2.12 */ + sh ''' + cp -rv build/ $BASEDIR_64 + rm -rv build/ + ''' + + dir("src/lttng-ust") { + sh ''' + ./bootstrap + ./configure --prefix="$PREFIX" --enable-java-agent-all --enable-jni-interface --enable-python-agent + make -j"$(nproc)" V=1 + make install DESTDIR="$WORKSPACE" + ''' + + sh ''' + find "$WORKSPACE/build/lib" -name "*.so" -exec chrpath --delete {} \\; + find "$WORKSPACE/build/lib" -name "*.la" -exec rm -vf {} \\; + ''' + } + + /* Move the artifact of the recently build project to the deps file and remove the "build" directory to prevent `make install` shenanigans for lttng-ust 2.12 */ + sh ''' + cp -rv build/ $BASEDIR_64 + rm -rv build/ + ''' + + stash name: "amd64-deps-lttng-tools", includes: 'deps-64/**' + } + } + } + } + stage('Build i386, including lttng-tools') { + agent { label 'bionic-i386' } + stages { + stage('Clean workspace') { + steps { + cleanWs() + } + } + + stage('Build') { + environment { + BASEDIR_32 = "$WORKSPACE/deps-32" + BASEDIR_BUILD_32 = "$BASEDIR_32/build" + INCLUDE_DIR_32 = "$BASEDIR_BUILD_32/include" + CPPFLAGS = "-I$INCLUDE_DIR_32" + LDFLAGS = "-L$BASEDIR_BUILD_32/lib" + PKG_CONFIG_PATH = "$BASEDIR_BUILD_32/lib/pkgconfig" + LD_LIBRARY_PATH ="$BASEDIR_BUILD_32/lib" + PATH ="$PATH:$BASEDIR_32/bin" + PREFIX = "/build" + CXXFLAGS="-g -O0" + CFLAGS="-g -O0" + } + steps { + unstash name: 'dep-sources' + unstash name: 'tools-sources' + + /* Prevent non existent include dir warnings */ + sh 'mkdir -p $INCLUDE_DIR_32' + + dir("src/userspace-rcu") { + sh ''' + ./bootstrap + ./configure --prefix="$PREFIX" + make -j"$(nproc)" V=1 + make install DESTDIR="$WORKSPACE" + ''' + + sh ''' + find "$WORKSPACE/build/lib" -name "*.so" -exec chrpath --delete {} \\; + find "$WORKSPACE/build/lib" -name "*.la" -exec rm -vf {} \\; + ''' + } + + /* Move the artifact of the recently build project to the deps file and remove the "build" directory to prevent `make install` shenanigans for lttng-ust 2.12 */ + sh ''' + cp -rv build/ $BASEDIR_32 + rm -rv build/ + ''' + + + dir("src/lttng-ust") { + sh ''' + ./bootstrap + ./configure --prefix="$PREFIX" + make V=1 + make install DESTDIR="$WORKSPACE" + ''' + + sh ''' + find "$WORKSPACE/build/lib" -name "*.so" -exec chrpath --delete {} \\; + find "$WORKSPACE/build/lib" -name "*.la" -exec rm -vf {} \\; + ''' + } + + /* Move the artifact of the recently build project to the deps file and remove the "build" directory to prevent `make install` shenanigans for lttng-ust 2.12 */ + sh ''' + cp -rv build/ $BASEDIR_32 + rm -rv build/ + ''' + + dir("src/lttng-tools") { + sh ''' + ./bootstrap + ./configure --prefix="$PREFIX" + make V=1 + make install DESTDIR="$WORKSPACE" + ''' + sh ''' + find "$WORKSPACE/build/bin" -type f -perm -0500 -exec chrpath --delete {} \\; + find "$WORKSPACE/build/lib" -name "*.so" -exec chrpath --delete {} \\; + find "$WORKSPACE/build/lib" -name "*.la" -exec rm -vf {} \\; + ''' + } + + /* Move the artifact of the recently build project to the deps file and remove the "build" directory to prevent `make install` shenanigans for lttng-ust 2.12 */ + sh ''' + cp -rv build/ $BASEDIR_32 + rm -rv build/ + ''' + + stash name: "i386-deps-lttng-tools", includes: 'deps-32/**' + + {% if version != 'stable-2.12' %} + /* Save the i386 ld_preloaded libraries for sessiond pausing */ + stash name: "i386-deps-sessiond-notification-pause-lib", includes: 'src/lttng-tools/tests/regression/tools/notification/.libs/libpause_sessiond.so' + {% endif %} + } + } + } + } + } + } + stage('Object store') { + agent { label 'bionic-amd64' } + environment { + S3_PREFIX = "s3://jenkins/32-64-bit-integration" + } + steps { + cleanWs() + dir('tar/sources') { + unstash name: "modules-sources" + unstash name: "tools-sources" + } + dir('tar/deps') { + unstash name: "i386-deps-lttng-tools" + unstash name: "amd64-deps-lttng-tools" + } + dir('tar/testing-overlay/sessiond') { + {% if version == 'stable-2.12' %} + sh ''' + echo "No testing overlay" > testing_overlay.txt + ''' + {% else %} + unstash name: "i386-deps-sessiond-notification-pause-lib" + {% endif %} + } + + dir('tar') { + sh ''' + echo $(pwd) + ls -lah ./ + tar -cf "$WORKSPACE/$BUILD_TAG.tar.bz2" -I pbzip2 . + ''' + } + + sh ''' + echo "# Setup endpoint + host_base = obj.internal.efficios.com + host_bucket = obj.internal.efficios.com + bucket_location = us-east-1 + use_https = True + + # Setup access keys + access_key = jenkins + secret_key = echo123456 + + # Enable S3 v4 signature APIs + signature_v2 = False" > "$WORKSPACE/.s3cfg" + s3cmd -c "$WORKSPACE/.s3cfg" put "$WORKSPACE/$BUILD_TAG.tar.bz2" "$S3_PREFIX/" + ''' + } + + } + stage('Testing') { + parallel { + stage('canary') { + agent none + steps { + build job: '{{ test_job_name }}', parameters: [string(name: 'ARTIFACT_ID', value: "${BUILD_TAG}.tar.bz2"), string(name: 'TEST_TYPE', value: 'canary'), string(name: 'BT_VERSION', value: "{{ babelversion }}")] + } + } + stage('CLI 32-bit') { + agent none + steps { + build job: '{{ test_job_name }}', parameters: [string(name: 'ARTIFACT_ID', value: "${BUILD_TAG}.tar.bz2"), string(name: 'TEST_TYPE', value: '32bit-cli'), string(name: 'BT_VERSION', value: "{{ babelversion }}")] + } + } + stage('sessiond 32-bit') { + agent none + steps { + build job: '{{ test_job_name }}', parameters: [string(name: 'ARTIFACT_ID', value: "${BUILD_TAG}.tar.bz2"), string(name: 'TEST_TYPE', value: '32bit-sessiond'), string(name: 'BT_VERSION', value: "{{ babelversion }}")] + } + } + stage('relayd 32-bit') { + agent none + steps { + build job: '{{ test_job_name }}', parameters: [string(name: 'ARTIFACT_ID', value: "${BUILD_TAG}.tar.bz2"), string(name: 'TEST_TYPE', value: '32bit-relayd'), string(name: 'BT_VERSION', value: "{{ babelversion }}")] + } + } + } + } + } +} diff --git a/scripts/integration/32-64/build.sh b/scripts/integration/32-64/build.sh new file mode 100644 index 0000000..36dce9e --- /dev/null +++ b/scripts/integration/32-64/build.sh @@ -0,0 +1,130 @@ +#!/bin/bash +# shellcheck disable=SC2103 +# +# Copyright (C) 2022 Jonathan Rajotte-Julien +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -eux + +BASEDIR_64="$WORKSPACE/deps-64/build" +BASEDIR_32="$WORKSPACE/deps-32/build" +export CPPFLAGS="-I$BASEDIR_64/include" +export LDFLAGS="-L$BASEDIR_64/lib" +export CFLAGS="-g -O0" +export CXXFLAGS="-g -O0" +export PKG_CONFIG_PATH="$BASEDIR_64/lib/pkgconfig" +export LD_LIBRARY_PATH="$BASEDIR_64/lib:$BASEDIR_32/lib" +export PATH="$PATH:$BASEDIR_64/bin" +export BABELTRACE_PLUGIN_PATH="$BASEDIR_64/lib/babeltrace2/plugins/" +export LIBBABELTRACE2_PLUGIN_PROVIDER_DIR="$BASEDIR_64/lib/babeltrace2/plugin-providers/" +PREFIX="/build" +export JAVA_HOME="/usr/lib/jvm/default-java" +DEPS_JAVA="$WORKSPACE/deps-64/build/share/java" +export CLASSPATH="$DEPS_JAVA/lttng-ust-agent-all.jar:/usr/share/java/log4j-api.jar:/usr/share/java/log4j-core.jar:/usr/share/java/log4j-1.2.jar" +export PYTHON2="python2" +export PYTHON3="python3" +P2_VERSION=$($PYTHON2 -c "import sys;print(sys.version[:3])") +P3_VERSION=$($PYTHON3 -c "import sys;print(sys.version[:3])") +DEPS_PYTHON2="$WORKSPACE/deps-64/build/lib/python$P2_VERSION/site-packages" +DEPS_PYTHON3="$WORKSPACE/deps-64/build/lib/python$P3_VERSION/site-packages" +export PYTHONPATH="$DEPS_PYTHON2:$DEPS_PYTHON3" +export LTTNG_CONSUMERD32_BIN="$BASEDIR_32/lib/lttng/libexec/lttng-consumerd" +export LTTNG_CONSUMERD32_LIBDIR="$BASEDIR_32/lib" +export LTTNG_CONSUMERD64_BIN="$BASEDIR_64/lib/lttng/libexec/lttng-consumerd" +export LTTNG_CONSUMERD64_LIBDIR="$BASEDIR_64/lib/" + +# Stable 2.12 and 2.13 still look for "babeltrace" +ln -s "$BASEDIR_64/bin/babeltrace2" "$BASEDIR_64/bin/babeltrace" + +echo "# Setup endpoint +host_base = obj.internal.efficios.com +host_bucket = obj.internal.efficios.com +bucket_location = us-east-1 +use_https = True + +# Setup access keys +access_key = jenkins +secret_key = echo123456 + +# Enable S3 v4 signature APIs +signature_v2 = False" > "$WORKSPACE/.s3cfg" + + +mkdir artefact +pushd artefact +s3cmd -c "$WORKSPACE/.s3cfg" get "s3://jenkins/32-64-bit-integration/$ARTIFACT_ID" +tar -xvf "$ARTIFACT_ID" +popd + +cp -r artefact/sources/* ./ +cp -r artefact/deps/* ./ + +pushd src/lttng-modules +make -j"$(nproc)" V=1 +make modules_install +depmod -a +popd + +pushd src/lttng-tools + +./bootstrap +./configure --prefix="$PREFIX" --enable-test-java-agent-all --enable-test-python-agent-all +popd + +# Deativate health test, simply because there is little value for this integration testing +# and because de ld_preloaded object is for both lttng-sessiond/consumer leading to difficult ld_preloading +# Deactivate clock plugin test since the app must load the correct biness so and the sessiond its bitness so, +# this is simply not feasible from outside the script. There is little value for this test in this testing context. +pushd src/lttng-tools/tests/regression +sed -i '/tools\/health\/test_thread_ok/d' Makefile.am +sed -i '/ust\/clock-override\/test_clock_override/d' Makefile.am +popd + +pushd src/lttng-tools/ +make -j"$(nproc)" V=1 +popd + +case "$TEST_TYPE" in + "canary") + ;; + "32bit-sessiond") + pushd src/lttng-tools/src/bin/lttng-sessiond + rm lttng-sessiond + ln -s "$BASEDIR_32/bin/lttng-sessiond" lttng-sessiond + popd + + cp -rv "$WORKSPACE/artefact/testing-overlay/sessiond/"* ./ + ;; + "32bit-relayd") + pushd src/lttng-tools/src/bin/lttng-relayd + rm lttng-relayd + ln -s "$BASEDIR_32/bin/lttng-relayd" lttng-relayd + popd + ;; + "32bit-cli") + pushd src/lttng-tools/src/bin/lttng + rm lttng + ln -s "$BASEDIR_32/bin/lttng" lttng + popd + ;; + *) + exit 1 +esac +failed_test=0 +pushd src/lttng-tools/tests +make --keep-going check || failed_test=1 +popd + +exit $failed_test diff --git a/scripts/integration/32-64/gather_tap_result.sh b/scripts/integration/32-64/gather_tap_result.sh new file mode 100644 index 0000000..529acec --- /dev/null +++ b/scripts/integration/32-64/gather_tap_result.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# shellcheck disable=SC2103 +# +# Copyright (C) 2022 Jonathan Rajotte-Julien +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -eux + +pushd src/lttng-tools/tests +mkdir -p "$WORKSPACE/tap" +rsync -a --exclude 'test-suite.log' --include '*/' --include '*.log' --exclude='*' ./ "$WORKSPACE/tap" +popd -- 2.34.1