From e9b4418900c34530a3012c531fee84b4d60c65b0 Mon Sep 17 00:00:00 2001 From: Michael Jeanson Date: Mon, 23 Jan 2017 15:27:15 -0500 Subject: [PATCH] jjb: Update modules based jobs Signed-off-by: Michael Jeanson --- jobs/latency-tracker.yaml | 73 ++++++++++++++++- jobs/lttng-modules.yaml | 79 ++++++++++++++++++- scripts/latency-tracker/master-vanilla.groovy | 55 +++++++++++-- scripts/latency-tracker/param-build.sh | 15 ++-- scripts/lttng-modules/master-vanilla.groovy | 55 +++++++++++-- scripts/lttng-modules/param-build.sh | 27 +++++-- 6 files changed, 270 insertions(+), 34 deletions(-) diff --git a/jobs/latency-tracker.yaml b/jobs/latency-tracker.yaml index 0a27371..b3bcddc 100644 --- a/jobs/latency-tracker.yaml +++ b/jobs/latency-tracker.yaml @@ -68,7 +68,7 @@ - workspace-cleanup - job-template: - name: latency-tracker_{mversion}_build-vanilla + name: latency-tracker_{mversion}_{buildtype}-vanilla defaults: latency-tracker description: | The LTTng modules provide Linux kernel tracing capability to the LTTng @@ -94,13 +94,17 @@ name: 'kverfloor' default: 'v3.12' description: 'The lowest kernel version to build.' + - string: + name: 'kverfilter' + default: 'stable-head' + description: 'Kernel versions list filtering strategy.' - string: name: 'kgitrepo' default: 'git://git-mirror.internal.efficios.com/kernel/stable/linux-stable.git' description: 'The linux kernel git repository url.' - string: name: 'kbuildjob' - default: 'latency-tracker_VERSION_param-build' + default: 'latency-tracker_VERSION_param-{buildtype}' description: 'The parametrized job to use for child builds.' builders: @@ -263,6 +267,64 @@ publishers: - workspace-cleanup +- job-template: + name: latency-tracker_VERSION_param-crossbuild + defaults: latency-tracker + description: | + This is a parametrized job used by 'master' jobs to build any combinations + of latency-tracker and linux kernel versions. + +

Job is managed by Jenkins Job Builder.

+ + project-type: matrix + node: 'master' # Applies only to matrix flyweight task + axes: + - axis: + type: user-defined + name: cross_arch + values: '{obj:cross_arch}' + + properties: + - build-discarder: + days-to-keep: 2 + + parameters: + - string: + name: 'mversion' + default: 'master' + description: 'The latency-tracker branch to build.' + - string: + name: 'kversion' + default: '' + description: 'The linux kernel git tag to build against.' + - string: + name: 'kgitrepo' + default: 'git://git-mirror.internal.efficios.com/kernel/stable/linux-stable.git' + description: 'The linux kernel git repository url.' + + concurrent: true + + scm: + - git: + url: git://github.com/efficios/latency-tracker.git + browser: githubweb + browser-url: https://github.com/efficios/latency-tracker + branches: + - "${{mversion}}" + skip-tag: true + basedir: src/latency-tracker + + triggers: + + builders: + - shell: | + git clone --depth=1 -b "$kversion" --reference $HOME/gitcache/linux-stable.git/ "$kgitrepo" src/linux + - shell: + !include-raw-escape: scripts/latency-tracker/param-build.sh + + publishers: + - workspace-cleanup + - job-template: name: latency-tracker_{mversion}_coverity defaults: latency-tracker @@ -355,13 +417,18 @@ mversion: - master jobs: - - 'latency-tracker_{mversion}_build-vanilla' + - 'latency-tracker_{mversion}_{buildtype}-vanilla': + buildtype: build - 'latency-tracker_{mversion}_build-rt': - 'latency-tracker_{mversion}_build-{uversion}': uversion: - xenial + - 'latency-tracker_{mversion}_{buildtype}-vanilla': + buildtype: crossbuild - 'latency-tracker_VERSION_param-build': arch: !!python/tuple [x86-32, x86-64] + - 'latency-tracker_VERSION_param-crossbuild': + cross_arch: !!python/tuple [armhf, arm64] - 'latency-tracker_{mversion}_cppcheck' - 'latency-tracker_{mversion}_sloccount': mversion: master diff --git a/jobs/lttng-modules.yaml b/jobs/lttng-modules.yaml index c223d5c..828c8b9 100644 --- a/jobs/lttng-modules.yaml +++ b/jobs/lttng-modules.yaml @@ -70,7 +70,7 @@ - workspace-cleanup - job-template: - name: lttng-modules_{mversion}_build-vanilla + name: lttng-modules_{mversion}_{buildtype}-vanilla defaults: lttng-modules description: | The LTTng modules provide Linux kernel tracing capability to the LTTng @@ -94,15 +94,19 @@ description: 'The maximum number of concurrent child build to run.' - string: name: 'kverfloor' - default: 'v2.6.36' + default: '{kverfloor}' description: 'The lowest kernel version to build.' + - string: + name: 'kverfilter' + default: '{kverfilter}' + description: 'Kernel versions list filtering strategy.' - string: name: 'kgitrepo' default: 'git://git-mirror.internal.efficios.com/kernel/stable/linux-stable.git' description: 'The linux kernel git repository url.' - string: name: 'kbuildjob' - default: 'lttng-modules_VERSION_param-build' + default: 'lttng-modules_VERSION_param-{buildtype}' description: 'The parametrized job to use for child builds.' builders: @@ -259,6 +263,64 @@ publishers: - workspace-cleanup +- job-template: + name: lttng-modules_VERSION_param-crossbuild + defaults: lttng-modules + description: | + This is a parametrized job used by 'master' jobs to build any combinations + of lttng-modules and linux kernel versions. + +

Job is managed by Jenkins Job Builder.

+ + project-type: matrix + node: 'master' # Applies only to matrix flyweight task + axes: + - axis: + type: user-defined + name: cross_arch + values: '{obj:cross_arch}' + + properties: + - build-discarder: + days-to-keep: 2 + + parameters: + - string: + name: 'mversion' + default: 'master' + description: 'The lttng-modules branch to build.' + - string: + name: 'kversion' + default: '' + description: 'The linux kernel git tag to build against.' + - string: + name: 'kgitrepo' + default: 'git://git-mirror.internal.efficios.com/kernel/stable/linux-stable.git' + description: 'The linux kernel git repository url.' + + concurrent: true + + scm: + - git: + url: git://github.com/lttng/lttng-modules.git + browser: githubweb + browser-url: https://github.com/lttng/lttng-modules + branches: + - "${{mversion}}" + skip-tag: true + basedir: src/lttng-modules + + triggers: + + builders: + - shell: | + git clone --depth=1 -b "$kversion" --reference $HOME/gitcache/linux-stable.git/ "$kgitrepo" src/linux + - shell: + !include-raw-escape: scripts/lttng-modules/param-build.sh + + publishers: + - workspace-cleanup + - job-template: name: lttng-modules_{mversion}_coverity defaults: lttng-modules @@ -354,15 +416,24 @@ - stable-2.9 - master jobs: - - 'lttng-modules_{mversion}_build-vanilla' + - 'lttng-modules_{mversion}_{buildtype}-vanilla': + buildtype: build + kverfloor: v2.6.36 + kverfilter: none - 'lttng-modules_{mversion}_build-rt': mversion: master - 'lttng-modules_{mversion}_build-{uversion}': uversion: - trusty - xenial + - 'lttng-modules_{mversion}_{buildtype}-vanilla': + buildtype: crossbuild + kverfloor: v3.16 + kverfilter: stable-head - 'lttng-modules_VERSION_param-build': arch: !!python/tuple [x86-32, x86-64] + - 'lttng-modules_VERSION_param-crossbuild': + cross_arch: !!python/tuple [armhf, arm64, powerpc, ppc64el] - 'lttng-modules_{mversion}_cppcheck' - 'lttng-modules_{mversion}_sloccount': mversion: master diff --git a/scripts/latency-tracker/master-vanilla.groovy b/scripts/latency-tracker/master-vanilla.groovy index 95b240e..67da3be 100644 --- a/scripts/latency-tracker/master-vanilla.groovy +++ b/scripts/latency-tracker/master-vanilla.groovy @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016 - Michael Jeanson + * Copyright (C) 2016-2017 - Michael Jeanson * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -79,21 +79,36 @@ class kVersion implements Comparable { return this.rc != Integer.MAX_VALUE } + // Return true if both version are of the same stable branch + Boolean isSameStable(kVersion o) { + if (this.major != o.major) { + return false + } + if (this.majorB != o.majorB) { + return false + } + if (this.minor != o.minor) { + return false + } + + return true + } + @Override int compareTo(kVersion o) { if (this.major != o.major) { - return Integer.compare(this.major, o.major); + return Integer.compare(this.major, o.major) } if (this.majorB != o.majorB) { - return Integer.compare(this.majorB, o.majorB); + return Integer.compare(this.majorB, o.majorB) } if (this.minor != o.minor) { - return Integer.compare(this.minor, o.minor); + return Integer.compare(this.minor, o.minor) } if (this.patch != o.patch) { - return Integer.compare(this.patch, o.patch); + return Integer.compare(this.patch, o.patch) } if (this.rc != o.rc) { - return Integer.compare(this.rc, o.rc); + return Integer.compare(this.rc, o.rc) } // Same version @@ -126,6 +141,7 @@ def mversion = build.buildVariableResolver.resolve('mversion') def maxConcurrentBuild = build.buildVariableResolver.resolve('maxConcurrentBuild') def kgitrepo = build.buildVariableResolver.resolve('kgitrepo') def kverfloor = new kVersion(build.buildVariableResolver.resolve('kverfloor')) +def kverfilter = build.buildVariableResolver.resolve('kverfilter') def job = Hudson.instance.getJob(build.buildVariableResolver.resolve('kbuildjob')) def currentJobName = build.project.getFullDisplayName() @@ -163,6 +179,30 @@ for (ref in refs) { kversions.sort() kversionsRC.sort() +switch (kverfilter) { + case 'stable-head': + // Keep only the head of each stable branch + println('Filter kernel versions to keep only the latest point release of each stable branch.') + + for (i = 0; i < kversions.size(); i++) { + def curr = kversions[i] + def next = i < kversions.size() - 1 ? kversions[i + 1] : null + + if (next != null) { + if (curr.isSameStable(next)) { + kversions.remove(i) + i-- + } + } + } + break + + default: + // No filtering of kernel versions + println('No kernel versions filtering selected.') + break +} + // If the last RC version is newer than the last stable, add it to the build list if (kversionsRC.last() > kversions.last()) { kversions.add(kversionsRC.last()) @@ -217,8 +257,7 @@ while ( kversions.size() != 0 || ongoingBuild.size() != 0 ) { } } - // Check for queued similar job since we only want to run latest - // as Mathieu Desnoyers requirement + // If a newer instance of this job is queued, abort to let it run similarJobQueued = Hudson.instance.queue.items.count{it.task.getFullDisplayName() == currentJobName} if ( similarJobQueued > 0 ) { // Abort since new build is queued diff --git a/scripts/latency-tracker/param-build.sh b/scripts/latency-tracker/param-build.sh index 80a23ca..8788887 100644 --- a/scripts/latency-tracker/param-build.sh +++ b/scripts/latency-tracker/param-build.sh @@ -75,7 +75,7 @@ prepare_lnx_sources() { echo "CONFIG_KALLSYMS_ALL=y" >> "${outdir}"/.config - make ${koutput} silentoldconfig CC=$CC + make ${koutput} olddefconfig CC=$CC make ${koutput} modules_prepare CC=$CC # Version specific tasks @@ -119,6 +119,7 @@ build_modules() { # We expect this build to fail, if it doesn't, fail the job. if [ "$?" -eq 0 ]; then + echo "This build should have failed." exit 1 fi @@ -145,9 +146,6 @@ build_modules() { ## MAIN ## -# Use gcc 4.9, older kernel don't build with gcc 5 -export CC=gcc-4.9 - # Use all CPU cores NPROC=$(nproc) @@ -195,13 +193,16 @@ if [ "x${cross_arch:-}" != "x" ]; then ;; esac + # Use default gcc when cross-compiling + CC="${cross_compile}gcc" + # Export variables used by Kbuild for cross compilation export ARCH="${karch}" export CROSS_COMPILE="${cross_compile}" - # Set arch specific values if we are not cross compiling elif [ "x${arch:-}" != "x" ]; then + case "$arch" in "x86-32") karch="x86" @@ -238,6 +239,10 @@ elif [ "x${arch:-}" != "x" ]; then exit 1 ;; esac + + # Use gcc 4.9, older kernel don't build with gcc 5 + CC=gcc-4.9 + else echo "Not arch or cross_arch specified" exit 1 diff --git a/scripts/lttng-modules/master-vanilla.groovy b/scripts/lttng-modules/master-vanilla.groovy index 95b240e..67da3be 100644 --- a/scripts/lttng-modules/master-vanilla.groovy +++ b/scripts/lttng-modules/master-vanilla.groovy @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016 - Michael Jeanson + * Copyright (C) 2016-2017 - Michael Jeanson * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -79,21 +79,36 @@ class kVersion implements Comparable { return this.rc != Integer.MAX_VALUE } + // Return true if both version are of the same stable branch + Boolean isSameStable(kVersion o) { + if (this.major != o.major) { + return false + } + if (this.majorB != o.majorB) { + return false + } + if (this.minor != o.minor) { + return false + } + + return true + } + @Override int compareTo(kVersion o) { if (this.major != o.major) { - return Integer.compare(this.major, o.major); + return Integer.compare(this.major, o.major) } if (this.majorB != o.majorB) { - return Integer.compare(this.majorB, o.majorB); + return Integer.compare(this.majorB, o.majorB) } if (this.minor != o.minor) { - return Integer.compare(this.minor, o.minor); + return Integer.compare(this.minor, o.minor) } if (this.patch != o.patch) { - return Integer.compare(this.patch, o.patch); + return Integer.compare(this.patch, o.patch) } if (this.rc != o.rc) { - return Integer.compare(this.rc, o.rc); + return Integer.compare(this.rc, o.rc) } // Same version @@ -126,6 +141,7 @@ def mversion = build.buildVariableResolver.resolve('mversion') def maxConcurrentBuild = build.buildVariableResolver.resolve('maxConcurrentBuild') def kgitrepo = build.buildVariableResolver.resolve('kgitrepo') def kverfloor = new kVersion(build.buildVariableResolver.resolve('kverfloor')) +def kverfilter = build.buildVariableResolver.resolve('kverfilter') def job = Hudson.instance.getJob(build.buildVariableResolver.resolve('kbuildjob')) def currentJobName = build.project.getFullDisplayName() @@ -163,6 +179,30 @@ for (ref in refs) { kversions.sort() kversionsRC.sort() +switch (kverfilter) { + case 'stable-head': + // Keep only the head of each stable branch + println('Filter kernel versions to keep only the latest point release of each stable branch.') + + for (i = 0; i < kversions.size(); i++) { + def curr = kversions[i] + def next = i < kversions.size() - 1 ? kversions[i + 1] : null + + if (next != null) { + if (curr.isSameStable(next)) { + kversions.remove(i) + i-- + } + } + } + break + + default: + // No filtering of kernel versions + println('No kernel versions filtering selected.') + break +} + // If the last RC version is newer than the last stable, add it to the build list if (kversionsRC.last() > kversions.last()) { kversions.add(kversionsRC.last()) @@ -217,8 +257,7 @@ while ( kversions.size() != 0 || ongoingBuild.size() != 0 ) { } } - // Check for queued similar job since we only want to run latest - // as Mathieu Desnoyers requirement + // If a newer instance of this job is queued, abort to let it run similarJobQueued = Hudson.instance.queue.items.count{it.task.getFullDisplayName() == currentJobName} if ( similarJobQueued > 0 ) { // Abort since new build is queued diff --git a/scripts/lttng-modules/param-build.sh b/scripts/lttng-modules/param-build.sh index cfc74f8..7e8aa74 100644 --- a/scripts/lttng-modules/param-build.sh +++ b/scripts/lttng-modules/param-build.sh @@ -54,7 +54,7 @@ prepare_lnx_sources() { ;; *) # Que sera sera - make ${koutput} allyesconfig CC=$CC + make ${koutput} ${vanilla_config} CC=$CC ;; esac @@ -75,7 +75,7 @@ prepare_lnx_sources() { echo "CONFIG_KALLSYMS_ALL=y" >> "${outdir}"/.config - make ${koutput} silentoldconfig CC=$CC + make ${koutput} olddefconfig CC=$CC make ${koutput} modules_prepare CC=$CC # Version specific tasks @@ -119,6 +119,7 @@ build_modules() { # We expect this build to fail, if it doesn't, fail the job. if [ "$?" -eq 0 ]; then + echo "This build should have failed." exit 1 fi @@ -145,9 +146,6 @@ build_modules() { ## MAIN ## -# Use gcc 4.9, older kernel don't build with gcc 5 -export CC=gcc-4.9 - # Use all CPU cores NPROC=$(nproc) @@ -168,24 +166,28 @@ if [ "x${cross_arch:-}" != "x" ]; then "armhf") karch="arm" cross_compile="arm-linux-gnueabihf-" + vanilla_config="allyesconfig" ubuntu_config="armhf-config.flavour.generic" ;; "arm64") karch="arm64" cross_compile="aarch64-linux-gnu-" + vanilla_config="allyesconfig" ubuntu_config="arm64-config.flavour.generic" ;; "powerpc") karch="powerpc" cross_compile="powerpc-linux-gnu-" + vanilla_config="ppc44x_defconfig" ubuntu_config="powerpc-config.flavour.powerpc-smp" ;; "ppc64el") karch="powerpc" cross_compile="powerpc64le-linux-gnu-" + vanilla_config="pseries_le_defconfig" ubuntu_config="ppc64el-config.flavour.generic" ;; @@ -195,41 +197,50 @@ if [ "x${cross_arch:-}" != "x" ]; then ;; esac + # Use default gcc when cross-compiling + CC="${cross_compile}gcc" + # Export variables used by Kbuild for cross compilation export ARCH="${karch}" export CROSS_COMPILE="${cross_compile}" - # Set arch specific values if we are not cross compiling elif [ "x${arch:-}" != "x" ]; then + case "$arch" in "x86-32") karch="x86" + vanilla_config="allyesconfig" ubuntu_config="i386-config.flavour.generic" ;; "x86-64") karch="x86" + vanilla_config="allyesconfig" ubuntu_config="amd64-config.flavour.generic" ;; "armhf") karch="arm" + vanilla_config="allyesconfig" ubuntu_config="armhf-config.flavour.generic" ;; "arm64") karch="arm64" + vanilla_config="allyesconfig" ubuntu_config="arm64-config.flavour.generic" ;; "powerpc") karch="powerpc" + vanilla_config="allyesconfig" ubuntu_config="powerpc-config.flavour.powerpc-smp" ;; "ppc64el") karch="powerpc" + vanilla_config="allyesconfig" ubuntu_config="ppc64el-config.flavour.generic" ;; @@ -238,6 +249,10 @@ elif [ "x${arch:-}" != "x" ]; then exit 1 ;; esac + + # Use gcc 4.9, older kernel don't build with gcc 5 + CC=gcc-4.9 + else echo "Not arch or cross_arch specified" exit 1 -- 2.34.1