From: Michael Jeanson <mjeanson@efficios.com>
Date: Mon, 23 Jan 2017 20:27:15 +0000 (-0500)
Subject: jjb: Update modules based jobs
X-Git-Url: https://git.lttng.org./?a=commitdiff_plain;h=e9b4418900c34530a3012c531fee84b4d60c65b0;p=lttng-ci.git

jjb: Update modules based jobs

Signed-off-by: Michael Jeanson <mjeanson@efficios.com>
---

diff --git a/jobs/latency-tracker.yaml b/jobs/latency-tracker.yaml
index 0a27371..b3bcddc 100644
--- a/jobs/latency-tracker.yaml
+++ b/jobs/latency-tracker.yaml
@@ -68,7 +68,7 @@
       - workspace-cleanup
 
 - job-template:
-    name: latency-tracker_{mversion}_build-vanilla
+    name: latency-tracker_{mversion}_{buildtype}-vanilla
     defaults: latency-tracker
     description: |
       The LTTng modules provide Linux kernel tracing capability to the LTTng
@@ -94,13 +94,17 @@
           name: 'kverfloor'
           default: 'v3.12'
           description: 'The lowest kernel version to build.'
+      - string:
+          name: 'kverfilter'
+          default: 'stable-head'
+          description: 'Kernel versions list filtering strategy.'
       - string:
           name: 'kgitrepo'
           default: 'git://git-mirror.internal.efficios.com/kernel/stable/linux-stable.git'
           description: 'The linux kernel git repository url.'
       - string:
           name: 'kbuildjob'
-          default: 'latency-tracker_VERSION_param-build'
+          default: 'latency-tracker_VERSION_param-{buildtype}'
           description: 'The parametrized job to use for child builds.'
 
     builders:
@@ -263,6 +267,64 @@
     publishers:
       - workspace-cleanup
 
+- job-template:
+    name: latency-tracker_VERSION_param-crossbuild
+    defaults: latency-tracker
+    description: |
+      This is a parametrized job used by 'master' jobs to build any combinations
+      of latency-tracker and linux kernel versions.
+
+      <p>Job is managed by Jenkins Job Builder.</p>
+
+    project-type: matrix
+    node: 'master' # Applies only to matrix flyweight task
+    axes:
+      - axis:
+          type: user-defined
+          name: cross_arch
+          values: '{obj:cross_arch}'
+
+    properties:
+      - build-discarder:
+          days-to-keep: 2
+
+    parameters:
+      - string:
+          name: 'mversion'
+          default: 'master'
+          description: 'The latency-tracker branch to build.'
+      - string:
+          name: 'kversion'
+          default: ''
+          description: 'The linux kernel git tag to build against.'
+      - string:
+          name: 'kgitrepo'
+          default: 'git://git-mirror.internal.efficios.com/kernel/stable/linux-stable.git'
+          description: 'The linux kernel git repository url.'
+
+    concurrent: true
+
+    scm:
+      - git:
+          url: git://github.com/efficios/latency-tracker.git
+          browser: githubweb
+          browser-url: https://github.com/efficios/latency-tracker
+          branches:
+            - "${{mversion}}"
+          skip-tag: true
+          basedir: src/latency-tracker
+
+    triggers:
+
+    builders:
+      - shell: |
+          git clone --depth=1 -b "$kversion" --reference $HOME/gitcache/linux-stable.git/ "$kgitrepo" src/linux
+      - shell:
+          !include-raw-escape: scripts/latency-tracker/param-build.sh
+
+    publishers:
+      - workspace-cleanup
+
 - job-template:
     name: latency-tracker_{mversion}_coverity
     defaults: latency-tracker
@@ -355,13 +417,18 @@
     mversion:
       - master
     jobs:
-      - 'latency-tracker_{mversion}_build-vanilla'
+      - 'latency-tracker_{mversion}_{buildtype}-vanilla':
+          buildtype: build
       - 'latency-tracker_{mversion}_build-rt':
       - 'latency-tracker_{mversion}_build-{uversion}':
           uversion:
             - xenial
+      - 'latency-tracker_{mversion}_{buildtype}-vanilla':
+          buildtype: crossbuild
       - 'latency-tracker_VERSION_param-build':
           arch: !!python/tuple [x86-32, x86-64]
+      - 'latency-tracker_VERSION_param-crossbuild':
+          cross_arch: !!python/tuple [armhf, arm64]
       - 'latency-tracker_{mversion}_cppcheck'
       - 'latency-tracker_{mversion}_sloccount':
           mversion: master
diff --git a/jobs/lttng-modules.yaml b/jobs/lttng-modules.yaml
index c223d5c..828c8b9 100644
--- a/jobs/lttng-modules.yaml
+++ b/jobs/lttng-modules.yaml
@@ -70,7 +70,7 @@
       - workspace-cleanup
 
 - job-template:
-    name: lttng-modules_{mversion}_build-vanilla
+    name: lttng-modules_{mversion}_{buildtype}-vanilla
     defaults: lttng-modules
     description: |
       The LTTng modules provide Linux kernel tracing capability to the LTTng
@@ -94,15 +94,19 @@
           description: 'The maximum number of concurrent child build to run.'
       - string:
           name: 'kverfloor'
-          default: 'v2.6.36'
+          default: '{kverfloor}'
           description: 'The lowest kernel version to build.'
+      - string:
+          name: 'kverfilter'
+          default: '{kverfilter}'
+          description: 'Kernel versions list filtering strategy.'
       - string:
           name: 'kgitrepo'
           default: 'git://git-mirror.internal.efficios.com/kernel/stable/linux-stable.git'
           description: 'The linux kernel git repository url.'
       - string:
           name: 'kbuildjob'
-          default: 'lttng-modules_VERSION_param-build'
+          default: 'lttng-modules_VERSION_param-{buildtype}'
           description: 'The parametrized job to use for child builds.'
 
     builders:
@@ -259,6 +263,64 @@
     publishers:
       - workspace-cleanup
 
+- job-template:
+    name: lttng-modules_VERSION_param-crossbuild
+    defaults: lttng-modules
+    description: |
+      This is a parametrized job used by 'master' jobs to build any combinations
+      of lttng-modules and linux kernel versions.
+
+      <p>Job is managed by Jenkins Job Builder.</p>
+
+    project-type: matrix
+    node: 'master' # Applies only to matrix flyweight task
+    axes:
+      - axis:
+          type: user-defined
+          name: cross_arch
+          values: '{obj:cross_arch}'
+
+    properties:
+      - build-discarder:
+          days-to-keep: 2
+
+    parameters:
+      - string:
+          name: 'mversion'
+          default: 'master'
+          description: 'The lttng-modules branch to build.'
+      - string:
+          name: 'kversion'
+          default: ''
+          description: 'The linux kernel git tag to build against.'
+      - string:
+          name: 'kgitrepo'
+          default: 'git://git-mirror.internal.efficios.com/kernel/stable/linux-stable.git'
+          description: 'The linux kernel git repository url.'
+
+    concurrent: true
+
+    scm:
+      - git:
+          url: git://github.com/lttng/lttng-modules.git
+          browser: githubweb
+          browser-url: https://github.com/lttng/lttng-modules
+          branches:
+            - "${{mversion}}"
+          skip-tag: true
+          basedir: src/lttng-modules
+
+    triggers:
+
+    builders:
+      - shell: |
+          git clone --depth=1 -b "$kversion" --reference $HOME/gitcache/linux-stable.git/ "$kgitrepo" src/linux
+      - shell:
+          !include-raw-escape: scripts/lttng-modules/param-build.sh
+
+    publishers:
+      - workspace-cleanup
+
 - job-template:
     name: lttng-modules_{mversion}_coverity
     defaults: lttng-modules
@@ -354,15 +416,24 @@
       - stable-2.9
       - master
     jobs:
-      - 'lttng-modules_{mversion}_build-vanilla'
+      - 'lttng-modules_{mversion}_{buildtype}-vanilla':
+          buildtype: build
+          kverfloor: v2.6.36
+          kverfilter: none
       - 'lttng-modules_{mversion}_build-rt':
           mversion: master
       - 'lttng-modules_{mversion}_build-{uversion}':
           uversion:
             - trusty
             - xenial
+      - 'lttng-modules_{mversion}_{buildtype}-vanilla':
+          buildtype: crossbuild
+          kverfloor: v3.16
+          kverfilter: stable-head
       - 'lttng-modules_VERSION_param-build':
           arch: !!python/tuple [x86-32, x86-64]
+      - 'lttng-modules_VERSION_param-crossbuild':
+          cross_arch: !!python/tuple [armhf, arm64, powerpc, ppc64el]
       - 'lttng-modules_{mversion}_cppcheck'
       - 'lttng-modules_{mversion}_sloccount':
           mversion: master
diff --git a/scripts/latency-tracker/master-vanilla.groovy b/scripts/latency-tracker/master-vanilla.groovy
index 95b240e..67da3be 100644
--- a/scripts/latency-tracker/master-vanilla.groovy
+++ b/scripts/latency-tracker/master-vanilla.groovy
@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016 - Michael Jeanson <mjeanson@efficios.com>
+ * Copyright (C) 2016-2017 - Michael Jeanson <mjeanson@efficios.com>
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -79,21 +79,36 @@ class kVersion implements Comparable<kVersion> {
     return this.rc != Integer.MAX_VALUE
   }
 
+  // Return true if both version are of the same stable branch
+  Boolean isSameStable(kVersion o) {
+    if (this.major != o.major) {
+      return false
+    }
+    if (this.majorB != o.majorB) {
+      return false
+    }
+    if (this.minor != o.minor) {
+      return false
+    }
+
+    return true
+  }
+
   @Override int compareTo(kVersion o) {
     if (this.major != o.major) {
-      return Integer.compare(this.major, o.major);
+      return Integer.compare(this.major, o.major)
     }
     if (this.majorB != o.majorB) {
-      return Integer.compare(this.majorB, o.majorB);
+      return Integer.compare(this.majorB, o.majorB)
     }
     if (this.minor != o.minor) {
-      return Integer.compare(this.minor, o.minor);
+      return Integer.compare(this.minor, o.minor)
     }
     if (this.patch != o.patch) {
-      return Integer.compare(this.patch, o.patch);
+      return Integer.compare(this.patch, o.patch)
     }
     if (this.rc != o.rc) {
-      return Integer.compare(this.rc, o.rc);
+      return Integer.compare(this.rc, o.rc)
     }
 
     // Same version
@@ -126,6 +141,7 @@ def mversion = build.buildVariableResolver.resolve('mversion')
 def maxConcurrentBuild = build.buildVariableResolver.resolve('maxConcurrentBuild')
 def kgitrepo = build.buildVariableResolver.resolve('kgitrepo')
 def kverfloor = new kVersion(build.buildVariableResolver.resolve('kverfloor'))
+def kverfilter = build.buildVariableResolver.resolve('kverfilter')
 def job = Hudson.instance.getJob(build.buildVariableResolver.resolve('kbuildjob'))
 def currentJobName = build.project.getFullDisplayName()
 
@@ -163,6 +179,30 @@ for (ref in refs) {
 kversions.sort()
 kversionsRC.sort()
 
+switch (kverfilter) {
+  case 'stable-head':
+    // Keep only the head of each stable branch
+    println('Filter kernel versions to keep only the latest point release of each stable branch.')
+
+    for (i = 0; i < kversions.size(); i++) {
+      def curr = kversions[i]
+      def next = i < kversions.size() - 1 ? kversions[i + 1] : null
+
+      if (next != null) {
+        if (curr.isSameStable(next)) {
+          kversions.remove(i)
+          i--
+        }
+      }
+    }
+    break
+
+  default:
+    // No filtering of kernel versions
+    println('No kernel versions filtering selected.')
+    break
+}
+
 // If the last RC version is newer than the last stable, add it to the build list
 if (kversionsRC.last() > kversions.last()) {
   kversions.add(kversionsRC.last())
@@ -217,8 +257,7 @@ while ( kversions.size() != 0 || ongoingBuild.size() != 0 ) {
       }
     }
 
-    // Check for queued similar job since we only want to run latest
-    // as Mathieu Desnoyers requirement
+    // If a newer instance of this job is queued, abort to let it run
     similarJobQueued = Hudson.instance.queue.items.count{it.task.getFullDisplayName() == currentJobName}
     if ( similarJobQueued > 0 ) {
 	 // Abort since new build is queued
diff --git a/scripts/latency-tracker/param-build.sh b/scripts/latency-tracker/param-build.sh
index 80a23ca..8788887 100644
--- a/scripts/latency-tracker/param-build.sh
+++ b/scripts/latency-tracker/param-build.sh
@@ -75,7 +75,7 @@ prepare_lnx_sources() {
     echo "CONFIG_KALLSYMS_ALL=y" >> "${outdir}"/.config
 
 
-    make ${koutput} silentoldconfig CC=$CC
+    make ${koutput} olddefconfig CC=$CC
     make ${koutput} modules_prepare CC=$CC
 
     # Version specific tasks
@@ -119,6 +119,7 @@ build_modules() {
 
         # We expect this build to fail, if it doesn't, fail the job.
         if [ "$?" -eq 0 ]; then
+            echo "This build should have failed."
             exit 1
         fi
 
@@ -145,9 +146,6 @@ build_modules() {
 
 ## MAIN ##
 
-# Use gcc 4.9, older kernel don't build with gcc 5
-export CC=gcc-4.9
-
 # Use all CPU cores
 NPROC=$(nproc)
 
@@ -195,13 +193,16 @@ if [ "x${cross_arch:-}" != "x" ]; then
             ;;
     esac
 
+    # Use default gcc when cross-compiling
+    CC="${cross_compile}gcc"
+
     # Export variables used by Kbuild for cross compilation
     export ARCH="${karch}"
     export CROSS_COMPILE="${cross_compile}"
 
-
 # Set arch specific values if we are not cross compiling
 elif [ "x${arch:-}" != "x" ]; then
+
     case "$arch" in
         "x86-32")
             karch="x86"
@@ -238,6 +239,10 @@ elif [ "x${arch:-}" != "x" ]; then
             exit 1
             ;;
     esac
+
+    # Use gcc 4.9, older kernel don't build with gcc 5
+    CC=gcc-4.9
+
 else
     echo "Not arch or cross_arch specified"
     exit 1
diff --git a/scripts/lttng-modules/master-vanilla.groovy b/scripts/lttng-modules/master-vanilla.groovy
index 95b240e..67da3be 100644
--- a/scripts/lttng-modules/master-vanilla.groovy
+++ b/scripts/lttng-modules/master-vanilla.groovy
@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2016 - Michael Jeanson <mjeanson@efficios.com>
+ * Copyright (C) 2016-2017 - Michael Jeanson <mjeanson@efficios.com>
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -79,21 +79,36 @@ class kVersion implements Comparable<kVersion> {
     return this.rc != Integer.MAX_VALUE
   }
 
+  // Return true if both version are of the same stable branch
+  Boolean isSameStable(kVersion o) {
+    if (this.major != o.major) {
+      return false
+    }
+    if (this.majorB != o.majorB) {
+      return false
+    }
+    if (this.minor != o.minor) {
+      return false
+    }
+
+    return true
+  }
+
   @Override int compareTo(kVersion o) {
     if (this.major != o.major) {
-      return Integer.compare(this.major, o.major);
+      return Integer.compare(this.major, o.major)
     }
     if (this.majorB != o.majorB) {
-      return Integer.compare(this.majorB, o.majorB);
+      return Integer.compare(this.majorB, o.majorB)
     }
     if (this.minor != o.minor) {
-      return Integer.compare(this.minor, o.minor);
+      return Integer.compare(this.minor, o.minor)
     }
     if (this.patch != o.patch) {
-      return Integer.compare(this.patch, o.patch);
+      return Integer.compare(this.patch, o.patch)
     }
     if (this.rc != o.rc) {
-      return Integer.compare(this.rc, o.rc);
+      return Integer.compare(this.rc, o.rc)
     }
 
     // Same version
@@ -126,6 +141,7 @@ def mversion = build.buildVariableResolver.resolve('mversion')
 def maxConcurrentBuild = build.buildVariableResolver.resolve('maxConcurrentBuild')
 def kgitrepo = build.buildVariableResolver.resolve('kgitrepo')
 def kverfloor = new kVersion(build.buildVariableResolver.resolve('kverfloor'))
+def kverfilter = build.buildVariableResolver.resolve('kverfilter')
 def job = Hudson.instance.getJob(build.buildVariableResolver.resolve('kbuildjob'))
 def currentJobName = build.project.getFullDisplayName()
 
@@ -163,6 +179,30 @@ for (ref in refs) {
 kversions.sort()
 kversionsRC.sort()
 
+switch (kverfilter) {
+  case 'stable-head':
+    // Keep only the head of each stable branch
+    println('Filter kernel versions to keep only the latest point release of each stable branch.')
+
+    for (i = 0; i < kversions.size(); i++) {
+      def curr = kversions[i]
+      def next = i < kversions.size() - 1 ? kversions[i + 1] : null
+
+      if (next != null) {
+        if (curr.isSameStable(next)) {
+          kversions.remove(i)
+          i--
+        }
+      }
+    }
+    break
+
+  default:
+    // No filtering of kernel versions
+    println('No kernel versions filtering selected.')
+    break
+}
+
 // If the last RC version is newer than the last stable, add it to the build list
 if (kversionsRC.last() > kversions.last()) {
   kversions.add(kversionsRC.last())
@@ -217,8 +257,7 @@ while ( kversions.size() != 0 || ongoingBuild.size() != 0 ) {
       }
     }
 
-    // Check for queued similar job since we only want to run latest
-    // as Mathieu Desnoyers requirement
+    // If a newer instance of this job is queued, abort to let it run
     similarJobQueued = Hudson.instance.queue.items.count{it.task.getFullDisplayName() == currentJobName}
     if ( similarJobQueued > 0 ) {
 	 // Abort since new build is queued
diff --git a/scripts/lttng-modules/param-build.sh b/scripts/lttng-modules/param-build.sh
index cfc74f8..7e8aa74 100644
--- a/scripts/lttng-modules/param-build.sh
+++ b/scripts/lttng-modules/param-build.sh
@@ -54,7 +54,7 @@ prepare_lnx_sources() {
         ;;
       *)
         # Que sera sera
-        make ${koutput} allyesconfig CC=$CC
+        make ${koutput} ${vanilla_config} CC=$CC
         ;;
     esac
 
@@ -75,7 +75,7 @@ prepare_lnx_sources() {
     echo "CONFIG_KALLSYMS_ALL=y" >> "${outdir}"/.config
 
 
-    make ${koutput} silentoldconfig CC=$CC
+    make ${koutput} olddefconfig CC=$CC
     make ${koutput} modules_prepare CC=$CC
 
     # Version specific tasks
@@ -119,6 +119,7 @@ build_modules() {
 
         # We expect this build to fail, if it doesn't, fail the job.
         if [ "$?" -eq 0 ]; then
+            echo "This build should have failed."
             exit 1
         fi
 
@@ -145,9 +146,6 @@ build_modules() {
 
 ## MAIN ##
 
-# Use gcc 4.9, older kernel don't build with gcc 5
-export CC=gcc-4.9
-
 # Use all CPU cores
 NPROC=$(nproc)
 
@@ -168,24 +166,28 @@ if [ "x${cross_arch:-}" != "x" ]; then
         "armhf")
             karch="arm"
             cross_compile="arm-linux-gnueabihf-"
+            vanilla_config="allyesconfig"
             ubuntu_config="armhf-config.flavour.generic"
             ;;
 
         "arm64")
             karch="arm64"
             cross_compile="aarch64-linux-gnu-"
+            vanilla_config="allyesconfig"
             ubuntu_config="arm64-config.flavour.generic"
             ;;
 
         "powerpc")
             karch="powerpc"
             cross_compile="powerpc-linux-gnu-"
+            vanilla_config="ppc44x_defconfig"
             ubuntu_config="powerpc-config.flavour.powerpc-smp"
             ;;
 
         "ppc64el")
             karch="powerpc"
             cross_compile="powerpc64le-linux-gnu-"
+            vanilla_config="pseries_le_defconfig"
             ubuntu_config="ppc64el-config.flavour.generic"
             ;;
 
@@ -195,41 +197,50 @@ if [ "x${cross_arch:-}" != "x" ]; then
             ;;
     esac
 
+    # Use default gcc when cross-compiling
+    CC="${cross_compile}gcc"
+
     # Export variables used by Kbuild for cross compilation
     export ARCH="${karch}"
     export CROSS_COMPILE="${cross_compile}"
 
-
 # Set arch specific values if we are not cross compiling
 elif [ "x${arch:-}" != "x" ]; then
+
     case "$arch" in
         "x86-32")
             karch="x86"
+            vanilla_config="allyesconfig"
             ubuntu_config="i386-config.flavour.generic"
             ;;
 
         "x86-64")
             karch="x86"
+            vanilla_config="allyesconfig"
             ubuntu_config="amd64-config.flavour.generic"
             ;;
 
         "armhf")
             karch="arm"
+            vanilla_config="allyesconfig"
             ubuntu_config="armhf-config.flavour.generic"
             ;;
 
         "arm64")
             karch="arm64"
+            vanilla_config="allyesconfig"
             ubuntu_config="arm64-config.flavour.generic"
             ;;
 
         "powerpc")
             karch="powerpc"
+            vanilla_config="allyesconfig"
             ubuntu_config="powerpc-config.flavour.powerpc-smp"
             ;;
 
         "ppc64el")
             karch="powerpc"
+            vanilla_config="allyesconfig"
             ubuntu_config="ppc64el-config.flavour.generic"
             ;;
 
@@ -238,6 +249,10 @@ elif [ "x${arch:-}" != "x" ]; then
             exit 1
             ;;
     esac
+
+    # Use gcc 4.9, older kernel don't build with gcc 5
+    CC=gcc-4.9
+
 else
     echo "Not arch or cross_arch specified"
     exit 1