From: Michael Jeanson <mjeanson@efficios.com>
Date: Fri, 28 Apr 2017 20:35:26 +0000 (-0400)
Subject: jjb: Revamp lttng-modules jobs
X-Git-Url: https://git.lttng.org./?a=commitdiff_plain;h=3a01c580582e4697b1554ed3695e57ef5911e919;p=lttng-ci.git

jjb: Revamp lttng-modules jobs

Signed-off-by: Michael Jeanson <mjeanson@efficios.com>
---

diff --git a/jobs/lttng-modules.yaml b/jobs/lttng-modules.yaml
index 80897d8..0df201a 100644
--- a/jobs/lttng-modules.yaml
+++ b/jobs/lttng-modules.yaml
@@ -1,4 +1,113 @@
 ---
+## Anchors
+- lttng-modules_build_parameters_defaults: &lttng-modules_build_parameters_defaults
+    name: 'lttng-modules_build_parameters_defaults'
+    parameters:
+      - string:
+          name: 'mversion'
+          default: '{mversion}'
+          description: 'The lttng-modules branch to build.'
+      - string:
+          name: 'maxConcurrentBuild'
+          default: '20'
+          description: 'The maximum number of concurrent child build to run.'
+      - string:
+          name: 'kverfloor'
+          default: '{kverfloor}'
+          description: 'The lowest kernel version to build.'
+      - string:
+          name: 'kverceil'
+          default: '{kverceil}'
+          description: 'The highest kernel version to build. (excluded)'
+      - string:
+          name: 'kverfilter'
+          default: '{kverfilter}'
+          description: 'Kernel versions list filtering strategy.'
+      - string:
+          name: 'kgitrepo'
+          default: 'git://git-mirror.internal.efficios.com/git/linux-all.git'
+          description: 'The linux kernel git repository url.'
+      - string:
+          name: 'kbuildjob'
+          default: 'lttng-modules_VERSION_param-{parambuildtype}'
+          description: 'The parametrized job to use for child builds.'
+
+- lttng-modules_build_parameters_ubuntu: &lttng-modules_build_parameters_ubuntu
+    name: 'lttng-modules_build_parameters_ubuntu'
+    parameters:
+      - string:
+          name: 'mversion'
+          default: '{mversion}'
+          description: 'The lttng-modules branch to build.'
+      - string:
+          name: 'maxConcurrentBuild'
+          default: '20'
+          description: 'The maximum number of concurrent child build to run.'
+      - string:
+          name: 'kverfloor'
+          default: '{kverfloor}'
+          description: 'The lowest kernel version to build.'
+      - string:
+          name: 'kverceil'
+          default: '{kverceil}'
+          description: 'The highest kernel version to build. (excluded)'
+      - string:
+          name: 'kverfilter'
+          default: '{kverfilter}'
+          description: 'Kernel versions list filtering strategy.'
+      - string:
+          name: 'kgitrepo'
+          default: 'git://git-mirror.internal.efficios.com/git/ubuntu-{uversion}.git'
+          description: 'The linux kernel git repository url.'
+      - string:
+          name: 'kbuildjob'
+          default: 'lttng-modules_VERSION_param-{parambuildtype}'
+          description: 'The parametrized job to use for child builds.'
+      - string:
+          name: 'uversion'
+          default: '{uversion}'
+          description: 'The lowest kernel version to build.'
+
+- lttng-modules_build_parameters_rt: &lttng-modules_build_parameters_rt
+    name: 'lttng-modules_build_parameters_rt'
+    parameters:
+      - string:
+          name: 'mversion'
+          default: '{mversion}'
+          description: 'The lttng-modules branch to build.'
+      - string:
+          name: 'maxConcurrentBuild'
+          default: '20'
+          description: 'The maximum number of concurrent child build to run.'
+      - string:
+          name: 'kverfloor'
+          default: 'v2.6.36-rt0-rebase'
+          description: 'The lowest kernel version to build.'
+      - string:
+          name: 'kgitrepo'
+          default: 'git://git-mirror.internal.efficios.com/kernel/rt/linux-rt-devel.git'
+          description: 'The linux kernel git repository url.'
+      - string:
+          name: 'kbuildjob'
+          default: 'lttng-modules_VERSION_param-build'
+          description: 'The parametrized job to use for child builds.'
+
+- lttng-modules_build_builders_defaults: &lttng-modules_build_builders_defaults
+    name: 'lttng-modules_build_builders_defaults'
+    builders:
+      - system-groovy:
+         command:
+           !include-raw-escape: scripts/lttng-modules/master.groovy
+
+- lttng-modules_build_rt_builders_defaults: &lttng-modules_build_rt_builders_defaults
+    name: 'lttng-modules_build_rt_builders_defaults'
+    builders:
+      - system-groovy:
+         command:
+           !include-raw-escape: scripts/lttng-modules/master-rt.groovy
+
+
+## Defaults
 - defaults:
     name: lttng-modules
     description: |
@@ -22,9 +131,10 @@
           branches:
             - "{mversion}"
           shallow-clone: true
-          skip-tag: true
           fastpoll: true
           basedir: src/lttng-modules
+          wipe-workspace: false
+          skip-tag: true
 
     triggers:
       - pollscm:
@@ -36,41 +146,30 @@
       - github:
           url: https://github.com/{github_user}/{github_name}
 
+    publishers:
+      - workspace-cleanup
+
 
 ## Templates
 - job-template:
-    name: lttng-modules_{mversion}_{kversion}_{buildtype}
+    name: lttng-modules_{mversion}_{buildtype}-vanilla
     defaults: lttng-modules
+    description: |
+      The LTTng modules provide Linux kernel tracing capability to the LTTng
+      2.0 tracer toolset.
 
-    project-type: matrix
-    node: 'master' # Applies only to matrix flyweight task
-    axes:
-      - axis:
-         type: slave
-         name: arch
-         values: '{obj:arch}'
+      This job will build the {mversion} branch against stable vanilla
+      kernel tags.
 
-    builders:
-      - copyartifact:
-          project: kernel_{kversion}_{buildtype}/arch=$arch
-          which-build: last-successful
-          stable: false
-          filter: 'build/**'
-          target: 'deps/linux'
-          do-not-fingerprint: true
-      - shell: |
-          git clone --depth=1 -b "v{kversion}" --reference $HOME/gitcache/linux-stable.git/ git://git-mirror.internal.efficios.com/git/linux-all.git src/linux
-      - shell:
-         !include-raw-escape: scripts/lttng-modules/build.sh
+      <p>Job is managed by Jenkins Job Builder.</p>
 
-    publishers:
-      - archive:
-          artifacts: 'build/**'
-          allow-empty: false
-      - workspace-cleanup
+    node: 'master'
+
+    <<: *lttng-modules_build_parameters_defaults
+    <<: *lttng-modules_build_builders_defaults
 
 - job-template:
-    name: lttng-modules_{mversion}_{buildtype}-vanilla
+    name: lttng-modules_{mversion}_fullbuild-vanilla
     defaults: lttng-modules
     description: |
       The LTTng modules provide Linux kernel tracing capability to the LTTng
@@ -83,87 +182,50 @@
 
     node: 'master'
 
-    parameters:
-      - string:
-          name: 'mversion'
-          default: '{mversion}'
-          description: 'The lttng-modules branch to build.'
-      - string:
-          name: 'maxConcurrentBuild'
-          default: '20'
-          description: 'The maximum number of concurrent child build to run.'
-      - string:
-          name: 'kverfloor'
-          default: '{kverfloor}'
-          description: 'The lowest kernel version to build.'
-      - string:
-          name: 'kverceil'
-          default: '{kverceil}'
-          description: 'The highest kernel version to build. (excluded)'
-      - string:
-          name: 'kverfilter'
-          default: '{kverfilter}'
-          description: 'Kernel versions list filtering strategy.'
-      - string:
-          name: 'kgitrepo'
-          default: 'git://git-mirror.internal.efficios.com/git/linux-all.git'
-          description: 'The linux kernel git repository url.'
-      - string:
-          name: 'kbuildjob'
-          default: 'lttng-modules_VERSION_param-{buildtype}'
-          description: 'The parametrized job to use for child builds.'
-
-    builders:
-      - system-groovy:
-         command:
-           !include-raw-escape: scripts/lttng-modules/master-vanilla.groovy
+    <<: *lttng-modules_build_parameters_defaults
+    <<: *lttng-modules_build_builders_defaults
 
-    publishers:
-      - workspace-cleanup
+    triggers:
+      - pollscm:
+          cron: "@midnight"
 
 - job-template:
-    name: lttng-modules_{mversion}_build-{uversion}
+    name: lttng-modules_{mversion}_{buildtype}-{uversion}
     defaults: lttng-modules
     description: |
       The LTTng modules provide Linux kernel tracing capability to the LTTng
       2.0 tracer toolset.
 
-      This job will build the {mversion} branch against all Ubuntu {uversion}
+      This job will build the {mversion} branch against Ubuntu {uversion}
       released kernels, including the LTS backport kernels.
 
       <p>Job is managed by Jenkins Job Builder.</p>
 
     node: 'master'
 
-    parameters:
-      - string:
-          name: 'mversion'
-          default: '{mversion}'
-          description: 'The lttng-modules branch to build.'
-      - string:
-          name: 'maxConcurrentBuild'
-          default: '20'
-          description: 'The maximum number of concurrent child build to run.'
-      - string:
-          name: 'uversion'
-          default: '{uversion}'
-          description: 'The lowest kernel version to build.'
-      - string:
-          name: 'kgitrepo'
-          default: 'git://git-mirror.internal.efficios.com/git/ubuntu-{uversion}.git'
-          description: 'The linux kernel git repository url.'
-      - string:
-          name: 'kbuildjob'
-          default: 'lttng-modules_VERSION_param-build'
-          description: 'The parametrized job to use for child builds.'
+    <<: *lttng-modules_build_parameters_ubuntu
+    <<: *lttng-modules_build_builders_defaults
 
-    builders:
-      - system-groovy:
-         command:
-           !include-raw-escape: scripts/lttng-modules/master-ubuntu.groovy
+- job-template:
+    name: lttng-modules_{mversion}_fullbuild-{uversion}
+    defaults: lttng-modules
+    description: |
+      The LTTng modules provide Linux kernel tracing capability to the LTTng
+      2.0 tracer toolset.
 
-    publishers:
-      - workspace-cleanup
+      This job will build the {mversion} branch against Ubuntu {uversion}
+      released kernels, including the LTS backport kernels.
+
+      <p>Job is managed by Jenkins Job Builder.</p>
+
+    node: 'master'
+
+    <<: *lttng-modules_build_parameters_ubuntu
+    <<: *lttng-modules_build_builders_defaults
+
+    triggers:
+      - pollscm:
+          cron: "@midnight"
 
 - job-template:
     name: lttng-modules_{mversion}_build-rt
@@ -179,35 +241,8 @@
 
     node: 'master'
 
-    parameters:
-      - string:
-          name: 'mversion'
-          default: '{mversion}'
-          description: 'The lttng-modules branch to build.'
-      - string:
-          name: 'maxConcurrentBuild'
-          default: '20'
-          description: 'The maximum number of concurrent child build to run.'
-      - string:
-          name: 'kverfloor'
-          default: 'v2.6.36-rt0-rebase'
-          description: 'The lowest kernel version to build.'
-      - string:
-          name: 'kgitrepo'
-          default: 'git://git-mirror.internal.efficios.com/kernel/rt/linux-rt-devel.git'
-          description: 'The linux kernel git repository url.'
-      - string:
-          name: 'kbuildjob'
-          default: 'lttng-modules_VERSION_param-build'
-          description: 'The parametrized job to use for child builds.'
-
-    builders:
-      - system-groovy:
-         command:
-           !include-raw-escape: scripts/lttng-modules/master-rt.groovy
-
-    publishers:
-      - workspace-cleanup
+    <<: *lttng-modules_build_parameters_rt
+    <<: *lttng-modules_build_rt_builders_defaults
 
 - job-template:
     name: lttng-modules_VERSION_param-build
@@ -264,9 +299,6 @@
       - shell:
           !include-raw-escape: scripts/lttng-modules/param-build.sh
 
-    publishers:
-      - workspace-cleanup
-
 - job-template:
     name: lttng-modules_VERSION_param-crossbuild
     defaults: lttng-modules
@@ -322,9 +354,6 @@
       - shell:
           !include-raw-escape: scripts/lttng-modules/param-build.sh
 
-    publishers:
-      - workspace-cleanup
-
 - job-template:
     name: lttng-modules_{mversion}_coverity
     defaults: lttng-modules
@@ -347,7 +376,7 @@
 
     builders:
       - shell: |
-         git clone --depth=1 -b v4.9 --reference $HOME/gitcache/linux-stable.git/ git://git-mirror.internal.efficios.com/kernel/stable/linux-stable.git src/linux
+         git clone --depth=1 -b v4.10 --reference $HOME/gitcache/linux-stable.git/ git://git-mirror.internal.efficios.com/kernel/stable/linux-stable.git src/linux
          cd src/linux
          make defconfig
          sed -i "s/# CONFIG_KALLSYMS_ALL is not set/CONFIG_KALLSYMS_ALL=y/g" .config
@@ -355,9 +384,6 @@
       - shell:
          !include-raw-escape: scripts/common/coverity.sh
 
-    publishers:
-      - workspace-cleanup
-
 - job-template:
     name: lttng-modules_{mversion}_cppcheck
     defaults: lttng-modules
@@ -377,10 +403,6 @@
           allow-empty: false
       - cppcheck:
           pattern: 'cppcheck.xml'
-      - email:
-          recipients: 'ci-notification@lists.lttng.org'
-          notify-every-unstable-build: true
-          send-to-individuals: false
 
 - job-template:
     name: lttng-modules_{mversion}_sloccount
@@ -407,6 +429,7 @@
           allow-empty: false
       - sloccount:
           report-files: 'cloc.xml'
+      - workspace-cleanup
 
 
 ## Project
@@ -415,29 +438,49 @@
     github_user: lttng
     github_name: lttng-modules
     jobs:
+
+# Vanilla normal builds
       - 'lttng-modules_{mversion}_{buildtype}-vanilla':
           mversion:
+            - stable-2.8
             - stable-2.9
             - master
           buildtype: build
+          parambuildtype: build
           kverfloor: v2.6.36
           kverceil: ''
-          kverfilter: none
+          kverfilter: stable-head
       - 'lttng-modules_{mversion}_{buildtype}-vanilla':
           mversion: stable-2.7
           buildtype: build
+          parambuildtype: build
           kverfloor: v2.6.36
           kverceil: v4.8-rc0
+          kverfilter: stable-head
+      - 'lttng-modules_{mversion}_fullbuild-vanilla':
+          mversion:
+            - stable-2.8
+            - stable-2.9
+            - master
+          buildtype: build
+          parambuildtype: build
+          kverfloor: v2.6.36
+          kverceil: ''
           kverfilter: none
-      - 'lttng-modules_{mversion}_{buildtype}-vanilla':
-          mversion: stable-2.8
+      - 'lttng-modules_{mversion}_fullbuild-vanilla':
+          mversion: stable-2.7
           buildtype: build
+          parambuildtype: build
           kverfloor: v2.6.36
-          kverceil: v4.10-rc0
+          kverceil: v4.8-rc0
           kverfilter: none
+
+# RT normal builds
       - 'lttng-modules_{mversion}_build-rt':
           mversion: master
-      - 'lttng-modules_{mversion}_build-{uversion}':
+
+# Ubuntu normal builds
+      - 'lttng-modules_{mversion}_{buildtype}-{uversion}':
           mversion:
             - stable-2.8
             - stable-2.9
@@ -445,30 +488,53 @@
           uversion:
             - trusty
             - xenial
+          buildtype: build
+          parambuildtype: build
+          kverfloor: ''
+          kverceil: ''
+          kverfilter: stable-head
+
+# Vanilla crossbuilds
       - 'lttng-modules_{mversion}_{buildtype}-vanilla':
           mversion:
+            - stable-2.8
             - stable-2.9
             - master
           buildtype: crossbuild
+          parambuildtype: crossbuild
           kverfloor: v3.16
           kverceil: ''
           kverfilter: stable-head
       - 'lttng-modules_{mversion}_{buildtype}-vanilla':
           mversion: stable-2.7
           buildtype: crossbuild
+          parambuildtype: crossbuild
           kverfloor: v3.16
           kverceil: v4.8-rc0
           kverfilter: stable-head
-      - 'lttng-modules_{mversion}_{buildtype}-vanilla':
-          mversion: stable-2.8
+
+# Ubuntu cross builds
+      - 'lttng-modules_{mversion}_{buildtype}-{uversion}':
+          mversion:
+            - stable-2.8
+            - stable-2.9
+            - master
+          uversion:
+            - trusty
+            - xenial
           buildtype: crossbuild
-          kverfloor: v3.16
-          kverceil: v4.10-rc0
+          parambuildtype: crossbuild
+          kverfloor: ''
+          kverceil: ''
           kverfilter: stable-head
+
+# Parametrized kernel and modules build jobs
       - 'lttng-modules_VERSION_param-build':
           arch: !!python/tuple [x86-32, x86-64]
       - 'lttng-modules_VERSION_param-crossbuild':
           cross_arch: !!python/tuple [armhf, arm64, powerpc, ppc64el]
+
+# Misc jobs
       - 'lttng-modules_{mversion}_cppcheck':
           mversion:
             - stable-2.8
diff --git a/scripts/lttng-modules/master-ubuntu.groovy b/scripts/lttng-modules/master-ubuntu.groovy
deleted file mode 100644
index da5f1fa..0000000
--- a/scripts/lttng-modules/master-ubuntu.groovy
+++ /dev/null
@@ -1,190 +0,0 @@
-/**
- * Copyright (C) 2016 - Michael Jeanson <mjeanson@efficios.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import hudson.model.*
-import hudson.AbortException
-import hudson.console.HyperlinkNote
-import java.util.concurrent.CancellationException
-import org.eclipse.jgit.api.Git
-import org.eclipse.jgit.lib.Ref
-
-
-// Retrieve parameters of the current build
-def mversion = build.buildVariableResolver.resolve('mversion')
-def maxConcurrentBuild = build.buildVariableResolver.resolve('maxConcurrentBuild')
-def kgitrepo = build.buildVariableResolver.resolve('kgitrepo')
-def uversion = build.buildVariableResolver.resolve('uversion')
-def job = Hudson.instance.getJob(build.buildVariableResolver.resolve('kbuildjob'))
-def currentJobName = build.project.getFullDisplayName()
-
-// Get the out variable
-def config = new HashMap()
-def bindings = getBinding()
-config.putAll(bindings.getVariables())
-def out = config['out']
-
-def jlc = new jenkins.model.JenkinsLocationConfiguration()
-def jenkinsUrl = jlc.url
-
-// Get tags from git repository
-def refs = Git.lsRemoteRepository().setTags(true).setRemote(kgitrepo).call();
-
-// Get kernel versions to build
-def kversions = []
-
-def matchStrs = []
-
-switch (uversion) {
-  case 'xenial':
-    matchStrs = [
-      ~/^refs\/tags\/(Ubuntu-4\.4\.0-\d{1,3}\.[\d\.]+)$/,
-      ~/^refs\/tags\/(Ubuntu-lts-.*_16\.04\.\d+)$/,
-    ]
-    break
-
-  case 'trusty':
-    matchStrs = [
-      ~/^refs\/tags\/(Ubuntu-3\.13\.0-[\d\.]+)$/,
-      ~/^refs\/tags\/(Ubuntu-lts-.*_14\.04\.\d+)$/,
-    ]
-    break
-
-  default:
-    println 'Unsupported Ubuntu version: ${uversion}'
-    throw new InterruptedException()
-    break
-}
-
-for (ref in refs) {
-  for (matchStr in matchStrs) {
-    def match = ref.getName() =~ matchStr
-
-    if (match) {
-      kversions.add(match.group(1))
-    }
-  }
-}
-
-kversions.sort()
-
-// Debug
-println "Building the following kernel versions:"
-for (k in kversions) {
-  println k
-}
-
-// Debug: Stop build here
-//throw new InterruptedException()
-
-def joburl = HyperlinkNote.encodeTo('/' + job.url, job.fullDisplayName)
-
-def allBuilds = []
-def ongoingBuild = []
-def failedRuns = []
-def isFailed = false
-
-// Loop while we have kernel versions remaining or jobs running
-while ( kversions.size() != 0 || ongoingBuild.size() != 0 ) {
-
-  if(ongoingBuild.size() < maxConcurrentBuild.toInteger() && kversions.size() != 0) {
-    def kversion = kversions.pop()
-    def job_params = [
-      new StringParameterValue('mversion', mversion),
-      new StringParameterValue('kversion', kversion),
-      new StringParameterValue('kgitrepo', kgitrepo),
-    ]
-
-    // Launch the parametrized build
-    def param_build = job.scheduleBuild2(0, new Cause.UpstreamCause(build), new ParametersAction(job_params))
-    println "triggering ${joburl} for the ${mversion} branch on kernel ${kversion}"
-
-    // Add it to the ongoing build queue
-    ongoingBuild.push(param_build)
-
-  } else {
-
-    println "Waiting... Queued: " + kversions.size() + " Running: " + ongoingBuild.size()
-    try {
-      Thread.sleep(5000)
-    } catch(e) {
-      if (e in InterruptedException) {
-        build.setResult(hudson.model.Result.ABORTED)
-        throw new InterruptedException()
-      } else {
-        throw(e)
-      }
-    }
-
-    // Check for queued similar job since we only want to run latest
-    // as Mathieu Desnoyers requirement
-    similarJobQueued = Hudson.instance.queue.items.count{it.task.getFullDisplayName() == currentJobName}
-    if ( similarJobQueued > 0 ) {
-	 // Abort since new build is queued
-        build.setResult(hudson.model.Result.ABORTED)
-        throw new InterruptedException()
-    }
-
-    def i = ongoingBuild.iterator()
-    while ( i.hasNext() ) {
-      currentBuild = i.next()
-      if ( currentBuild.isCancelled() || currentBuild.isDone() ) {
-        // Remove from queue
-        i.remove()
-
-        // Print results
-        def matrixParent = currentBuild.get()
-        allBuilds.add(matrixParent)
-        def kernelStr = matrixParent.buildVariableResolver.resolve("kversion")
-        println "${matrixParent.fullDisplayName} (${kernelStr}) completed with status ${matrixParent.result}"
-
-        // Process child runs of matrixBuild
-        def childRuns = matrixParent.getRuns()
-        for ( childRun in childRuns ) {
-          println "\t${childRun.fullDisplayName} (${kernelStr}) completed with status ${childRun.result}"
-          if (childRun.result != Result.SUCCESS) {
-            failedRuns.add(childRun)
-            isFailed = true
-          }
-        }
-      }
-    }
-  }
-}
-
-// Get log of failed runs
-for (failedRun in failedRuns) {
-  println "---START---"
-  failedRun.writeWholeLogTo(out)
-  println "---END---"
-}
-
-println "---Build report---"
-for (b in allBuilds) {
-  def kernelStr = b.buildVariableResolver.resolve("kversion")
-  println "${b.fullDisplayName} (${kernelStr}) completed with status ${b.result}"
-  // Cleanup builds
-  try {
-    b.delete()
-  } catch (all) {}
-}
-
-// Mark this build failed if any child build has failed
-if (isFailed) {
-  build.getExecutor().interrupt(Result.FAILURE)
-}
-
-// EOF
diff --git a/scripts/lttng-modules/master-vanilla.groovy b/scripts/lttng-modules/master-vanilla.groovy
deleted file mode 100644
index 7ba8974..0000000
--- a/scripts/lttng-modules/master-vanilla.groovy
+++ /dev/null
@@ -1,342 +0,0 @@
-/**
- * Copyright (C) 2016-2017 - Michael Jeanson <mjeanson@efficios.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-import hudson.model.*
-import hudson.AbortException
-import hudson.console.HyperlinkNote
-import java.util.concurrent.CancellationException
-import org.eclipse.jgit.api.Git
-import org.eclipse.jgit.lib.Ref
-
-
-class InvalidkVersionException extends Exception {
-  public InvalidkVersionException(String message) {
-    super(message)
-  }
-}
-
-class EmptykVersionException extends Exception {
-  public EmptykVersionException(String message) {
-    super(message)
-  }
-}
-
-class kVersion implements Comparable<kVersion> {
-
-  Integer major = 0;
-  Integer majorB = 0;
-  Integer minor = 0;
-  Integer patch = 0;
-  Integer rc = Integer.MAX_VALUE;
-
-  kVersion() {}
-
-  kVersion(version) {
-    this.parse(version)
-  }
-
-  def parse(version) {
-    this.major = 0
-    this.majorB = 0
-    this.minor = 0
-    this.patch = 0
-    this.rc = Integer.MAX_VALUE
-
-    if (!version) {
-      throw new EmptykVersionException("Empty kernel version")
-    }
-
-    def match = version =~ /^v(\d+)\.(\d+)(\.(\d+))?(\.(\d+))?(-rc(\d+))?$/
-    if (!match) {
-      throw new InvalidkVersionException("Invalid kernel version: ${version}")
-    }
-
-    Integer offset = 0;
-
-    // Major
-    this.major = Integer.parseInt(match.group(1))
-    if (this.major <= 2) {
-      offset = 2
-      this.majorB = Integer.parseInt(match.group(2))
-    }
-
-    // Minor
-    if (match.group(2 + offset) != null) {
-      this.minor = Integer.parseInt(match.group(2 + offset))
-    }
-
-    // Patch level
-    if (match.group(4 + offset) != null) {
-      this.patch = Integer.parseInt(match.group(4 + offset))
-    }
-
-    // RC
-    if (match.group(8) != null) {
-      this.rc = Integer.parseInt(match.group(8))
-    }
-  }
-
-  // Return true if this version is a release candidate
-  Boolean isRC() {
-    return this.rc != Integer.MAX_VALUE
-  }
-
-  // Return true if both version are of the same stable branch
-  Boolean isSameStable(kVersion o) {
-    if (this.major != o.major) {
-      return false
-    }
-    if (this.majorB != o.majorB) {
-      return false
-    }
-    if (this.minor != o.minor) {
-      return false
-    }
-
-    return true
-  }
-
-  @Override int compareTo(kVersion o) {
-    if (this.major != o.major) {
-      return Integer.compare(this.major, o.major)
-    }
-    if (this.majorB != o.majorB) {
-      return Integer.compare(this.majorB, o.majorB)
-    }
-    if (this.minor != o.minor) {
-      return Integer.compare(this.minor, o.minor)
-    }
-    if (this.patch != o.patch) {
-      return Integer.compare(this.patch, o.patch)
-    }
-    if (this.rc != o.rc) {
-      return Integer.compare(this.rc, o.rc)
-    }
-
-    // Same version
-    return 0;
-  }
-
-  String toString() {
-    String vString = "v${this.major}"
-
-    if (this.majorB > 0) {
-      vString = vString.concat(".${this.majorB}")
-    }
-
-    vString = vString.concat(".${this.minor}")
-
-    if (this.patch > 0) {
-      vString = vString.concat(".${this.patch}")
-    }
-
-    if (this.rc > 0 && this.rc < Integer.MAX_VALUE) {
-      vString = vString.concat("-rc${this.rc}")
-    }
-    return vString
-  }
-}
-
-
-// Retrieve parameters of the current build
-def mversion = build.buildVariableResolver.resolve('mversion')
-def maxConcurrentBuild = build.buildVariableResolver.resolve('maxConcurrentBuild')
-def kgitrepo = build.buildVariableResolver.resolve('kgitrepo')
-def kverfloor_raw = build.buildVariableResolver.resolve('kverfloor')
-def kverceil_raw = build.buildVariableResolver.resolve('kverceil')
-def kverfilter = build.buildVariableResolver.resolve('kverfilter')
-def job = Hudson.instance.getJob(build.buildVariableResolver.resolve('kbuildjob'))
-def currentJobName = build.project.getFullDisplayName()
-
-// Parse kernel versions
-def kverfloor = new kVersion(kverfloor_raw)
-def kverceil = ""
-
-try {
-    kverceil = new kVersion(kverceil_raw)
-} catch (EmptykVersionException e) {
-    kverceil = new kVersion("v" + Integer.MAX_VALUE + ".0.0")
-}
-
-// Get the out variable
-def config = new HashMap()
-def bindings = getBinding()
-config.putAll(bindings.getVariables())
-def out = config['out']
-
-
-// Get tags from git repository
-def refs = Git.lsRemoteRepository().setTags(true).setRemote(kgitrepo).call();
-
-// Get kernel versions to build
-def kversions = []
-def kversionsRC = []
-for (ref in refs) {
-  def match = ref.getName() =~ /^refs\/tags\/(v[\d\.]+(-rc(\d+))?)$/
-
-  if (match) {
-    def v = new kVersion(match.group(1))
-
-    if ((v >= kverfloor) && (v < kverceil)) {
-      if (v.isRC()) {
-        kversionsRC.add(v)
-      } else {
-        kversions.add(v)
-      }
-    }
-  }
-}
-
-kversions.sort()
-kversionsRC.sort()
-
-switch (kverfilter) {
-  case 'stable-head':
-    // Keep only the head of each stable branch
-    println('Filter kernel versions to keep only the latest point release of each stable branch.')
-
-    for (i = 0; i < kversions.size(); i++) {
-      def curr = kversions[i]
-      def next = i < kversions.size() - 1 ? kversions[i + 1] : null
-
-      if (next != null) {
-        if (curr.isSameStable(next)) {
-          kversions.remove(i)
-          i--
-        }
-      }
-    }
-    break
-
-  default:
-    // No filtering of kernel versions
-    println('No kernel versions filtering selected.')
-    break
-}
-
-// If the last RC version is newer than the last stable, add it to the build list
-if (kversionsRC.last() > kversions.last()) {
-  kversions.add(kversionsRC.last())
-}
-
-// Debug
-println "Building the following kernel versions:"
-for (k in kversions) {
-  println k
-}
-
-// Debug: Stop build here
-//throw new InterruptedException()
-
-def joburl = HyperlinkNote.encodeTo('/' + job.url, job.fullDisplayName)
-
-def allBuilds = []
-def ongoingBuild = []
-def failedRuns = []
-def isFailed = false
-def similarJobQueued = 0;
-
-// Loop while we have kernel versions remaining or jobs running
-while ( kversions.size() != 0 || ongoingBuild.size() != 0 ) {
-
-  if(ongoingBuild.size() < maxConcurrentBuild.toInteger() && kversions.size() != 0) {
-    def kversion = kversions.pop()
-    def job_params = [
-      new StringParameterValue('mversion', mversion),
-      new StringParameterValue('kversion', kversion.toString()),
-      new StringParameterValue('kgitrepo', kgitrepo),
-    ]
-
-    // Launch the parametrized build
-    def param_build = job.scheduleBuild2(0, new Cause.UpstreamCause(build), new ParametersAction(job_params))
-    println "triggering ${joburl} for the ${mversion} branch on kernel ${kversion}"
-
-    // Add it to the ongoing build queue
-    ongoingBuild.push(param_build)
-
-  } else {
-
-    println "Waiting... Queued: " + kversions.size() + " Running: " + ongoingBuild.size()
-    try {
-      Thread.sleep(5000)
-    } catch(e) {
-      if (e in InterruptedException) {
-        build.setResult(hudson.model.Result.ABORTED)
-        throw new InterruptedException()
-      } else {
-        throw(e)
-      }
-    }
-
-    // If a newer instance of this job is queued, abort to let it run
-    similarJobQueued = Hudson.instance.queue.items.count{it.task.getFullDisplayName() == currentJobName}
-    if ( similarJobQueued > 0 ) {
-	 // Abort since new build is queued
-        build.setResult(hudson.model.Result.ABORTED)
-        throw new InterruptedException()
-    }
-
-    def i = ongoingBuild.iterator()
-    while ( i.hasNext() ) {
-      currentBuild = i.next()
-      if ( currentBuild.isCancelled() || currentBuild.isDone() ) {
-        // Remove from queue
-        i.remove()
-
-        // Print results
-        def matrixParent = currentBuild.get()
-        allBuilds.add(matrixParent)
-        def kernelStr = matrixParent.buildVariableResolver.resolve("kversion")
-        println "${matrixParent.fullDisplayName} (${kernelStr}) completed with status ${matrixParent.result}"
-
-        // Process child runs of matrixBuild
-        def childRuns = matrixParent.getRuns()
-        for ( childRun in childRuns ) {
-          println "\t${childRun.fullDisplayName} (${kernelStr}) completed with status ${childRun.result}"
-          if (childRun.result != Result.SUCCESS) {
-            failedRuns.add(childRun)
-            isFailed = true
-          }
-        }
-      }
-    }
-  }
-}
-
-// Get log of failed runs
-for (failedRun in failedRuns) {
-  println "---START---"
-  failedRun.writeWholeLogTo(out)
-  println "---END---"
-}
-
-println "---Build report---"
-for (b in allBuilds) {
-  def kernelStr = b.buildVariableResolver.resolve("kversion")
-  println "${b.fullDisplayName} (${kernelStr}) completed with status ${b.result}"
-  // Cleanup builds
-  try {
-    b.delete()
-  } catch (all) {}
-}
-
-// Mark this build failed if any child build has failed
-if (isFailed) {
-  build.getExecutor().interrupt(Result.FAILURE)
-}
-
-// EOF
diff --git a/scripts/lttng-modules/master.groovy b/scripts/lttng-modules/master.groovy
new file mode 100644
index 0000000..0dbf61b
--- /dev/null
+++ b/scripts/lttng-modules/master.groovy
@@ -0,0 +1,534 @@
+/**
+ * Copyright (C) 2016-2017 - Michael Jeanson <mjeanson@efficios.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import hudson.model.*
+import hudson.AbortException
+import hudson.console.HyperlinkNote
+import java.util.concurrent.CancellationException
+import org.eclipse.jgit.api.Git
+import org.eclipse.jgit.lib.Ref
+
+
+class InvalidKVersionException extends Exception {
+  public InvalidKVersionException(String message) {
+    super(message)
+  }
+}
+
+class EmptyKVersionException extends Exception {
+  public EmptyKVersionException(String message) {
+    super(message)
+  }
+}
+
+class VanillaKVersion implements Comparable<VanillaKVersion> {
+
+  Integer major = 0
+  Integer majorB = 0
+  Integer minor = 0
+  Integer patch = 0
+  Integer rc = Integer.MAX_VALUE
+
+  VanillaKVersion() {}
+
+  VanillaKVersion(version) {
+    this.parse(version)
+  }
+
+  static VanillaKVersion minKVersion() {
+    return new VanillaKVersion("v0.0.0")
+  }
+
+  static VanillaKVersion maxKVersion() {
+    return new VanillaKVersion("v" + Integer.MAX_VALUE + ".0.0")
+  }
+
+  static VanillaKVersion factory(version) {
+    return new VanillaKVersion(version)
+  }
+
+  def parse(version) {
+    this.major = 0
+    this.majorB = 0
+    this.minor = 0
+    this.patch = 0
+    this.rc = Integer.MAX_VALUE
+
+    if (!version) {
+      throw new EmptyKVersionException("Empty kernel version")
+    }
+
+    def match = version =~ /^v(\d+)\.(\d+)(\.(\d+))?(\.(\d+))?(-rc(\d+))?$/
+    if (!match) {
+      throw new InvalidKVersionException("Invalid kernel version: ${version}")
+    }
+
+    Integer offset = 0;
+
+    // Major
+    this.major = Integer.parseInt(match.group(1))
+    if (this.major <= 2) {
+      offset = 2
+      this.majorB = Integer.parseInt(match.group(2))
+    }
+
+    // Minor
+    if (match.group(2 + offset) != null) {
+      this.minor = Integer.parseInt(match.group(2 + offset))
+    }
+
+    // Patch level
+    if (match.group(4 + offset) != null) {
+      this.patch = Integer.parseInt(match.group(4 + offset))
+    }
+
+    // RC
+    if (match.group(8) != null) {
+      this.rc = Integer.parseInt(match.group(8))
+    }
+  }
+
+  // Return true if this version is a release candidate
+  Boolean isRC() {
+    return this.rc != Integer.MAX_VALUE
+  }
+
+  // Return true if both version are of the same stable branch
+  Boolean isSameStable(VanillaKVersion o) {
+    if (this.major != o.major) {
+      return false
+    }
+    if (this.majorB != o.majorB) {
+      return false
+    }
+    if (this.minor != o.minor) {
+      return false
+    }
+
+    return true
+  }
+
+  @Override int compareTo(VanillaKVersion o) {
+    if (this.major != o.major) {
+      return Integer.compare(this.major, o.major)
+    }
+    if (this.majorB != o.majorB) {
+      return Integer.compare(this.majorB, o.majorB)
+    }
+    if (this.minor != o.minor) {
+      return Integer.compare(this.minor, o.minor)
+    }
+    if (this.patch != o.patch) {
+      return Integer.compare(this.patch, o.patch)
+    }
+    if (this.rc != o.rc) {
+      return Integer.compare(this.rc, o.rc)
+    }
+
+    // Same version
+    return 0;
+  }
+
+  String toString() {
+    String vString = "v${this.major}"
+
+    if (this.majorB > 0) {
+      vString = vString.concat(".${this.majorB}")
+    }
+
+    vString = vString.concat(".${this.minor}")
+
+    if (this.patch > 0) {
+      vString = vString.concat(".${this.patch}")
+    }
+
+    if (this.rc > 0 && this.rc < Integer.MAX_VALUE) {
+      vString = vString.concat("-rc${this.rc}")
+    }
+    return vString
+  }
+}
+
+class UbuntuKVersion implements Comparable<UbuntuKVersion> {
+
+  Integer major = 0
+  Integer minor = 0
+  Integer patch = 0
+  Integer umajor = 0
+  Integer uminor = 0
+  String suffix = ""
+  Boolean isLTS = false
+
+  UbuntuKVersion() {}
+
+  UbuntuKVersion(version) {
+    this.parse(version)
+  }
+
+  static UbuntuKVersion minKVersion() {
+    return new UbuntuKVersion("Ubuntu-lts-0.0.0-0.0")
+  }
+
+  static UbuntuKVersion maxKVersion() {
+    return new UbuntuKVersion("Ubuntu-" + Integer.MAX_VALUE + ".0.0-0.0")
+  }
+
+  static UbuntuKVersion factory(version) {
+    return new UbuntuKVersion(version)
+  }
+
+  def parse(version) {
+    this.major = 0
+    this.minor = 0
+    this.patch = 0
+    this.umajor = 0
+    this.uminor = 0
+    this.suffix = "";
+    this.isLTS = false
+
+    if (!version) {
+      throw new EmptyKVersionException("Empty kernel version")
+    }
+
+    //'Ubuntu-lts-4.8.0-27.29_16.04.1',
+    //'Ubuntu-4.4.0-70.91',
+    def match = version =~ /^Ubuntu-(lts-)??(\d+)\.(\d+)\.(\d+)-(\d+)\.(\d+)(.*)??$/
+    if (!match) {
+      throw new InvalidKVersionException("Invalid kernel version: ${version}")
+    }
+
+    this.isLTS = match.group(1) != null
+
+    // Major
+    this.major = Integer.parseInt(match.group(2))
+
+    // Minor
+    this.minor = Integer.parseInt(match.group(3))
+
+    // Patch level
+    this.patch = Integer.parseInt(match.group(4))
+
+    // Ubuntu major
+    this.umajor = Integer.parseInt(match.group(5))
+
+    // Ubuntu minor
+    this.uminor = Integer.parseInt(match.group(6))
+
+    if (match.group(7) != null) {
+      this.suffix = match.group(7)
+    }
+  }
+
+  // Return true if this version is a release candidate
+  Boolean isRC() {
+    return false
+  }
+
+  // Return true if both version are of the same stable branch
+  Boolean isSameStable(UbuntuKVersion o) {
+    if (this.isLTS != o.isLTS) {
+      return false
+    }
+    if (this.major != o.major) {
+      return false
+    }
+    if (this.minor != o.minor) {
+      return false
+    }
+    if (this.patch != o.patch) {
+      return false
+    }
+
+    return true
+  }
+
+  @Override int compareTo(UbuntuKVersion o) {
+    if (this.major != o.major) {
+      return Integer.compare(this.major, o.major)
+    }
+    if (this.minor != o.minor) {
+      return Integer.compare(this.minor, o.minor)
+    }
+    if (this.patch != o.patch) {
+      return Integer.compare(this.patch, o.patch)
+    }
+    if (this.umajor != o.umajor) {
+      return Integer.compare(this.umajor, o.umajor)
+    }
+    if (this.uminor != o.uminor) {
+      return Integer.compare(this.uminor, o.uminor)
+    }
+    if (this.isLTS != o.isLTS) {
+      if (o.isLTS) {
+        return 1
+      } else {
+        return -1
+      }
+    }
+
+    // Same version
+    return 0;
+  }
+
+  String toString() {
+    String vString = "Ubuntu-"
+
+    if (this.isLTS) {
+      vString = vString.concat("lts-")
+    }
+
+    vString = vString.concat("${this.major}.${this.minor}.${this.patch}-${this.umajor}.${this.uminor}${this.suffix}")
+
+    return vString
+  }
+}
+
+
+// Retrieve parameters of the current build
+def mversion = build.buildVariableResolver.resolve('mversion')
+def maxConcurrentBuild = build.buildVariableResolver.resolve('maxConcurrentBuild')
+def kgitrepo = build.buildVariableResolver.resolve('kgitrepo')
+def kverfloor_raw = build.buildVariableResolver.resolve('kverfloor')
+def kverceil_raw = build.buildVariableResolver.resolve('kverceil')
+def kverfilter = build.buildVariableResolver.resolve('kverfilter')
+def uversion = build.buildVariableResolver.resolve('uversion')
+def job = Hudson.instance.getJob(build.buildVariableResolver.resolve('kbuildjob'))
+def currentJobName = build.project.getFullDisplayName()
+
+
+// Get the out variable
+def config = new HashMap()
+def bindings = getBinding()
+config.putAll(bindings.getVariables())
+def out = config['out']
+
+
+// Get tags from git repository
+def refs = Git.lsRemoteRepository().setTags(true).setRemote(kgitrepo).call();
+
+// Get kernel versions to build
+def kversions = []
+def kversionsRC = []
+def matchStrs = []
+def blacklist = []
+def kversionFactory = ""
+
+if (uversion != null) {
+  kversionFactory = new UbuntuKVersion()
+  switch (uversion) {
+    case 'xenial':
+      matchStrs = [
+        ~/^refs\/tags\/(Ubuntu-4\.4\.0-\d{1,3}?\.[\d]+)$/,
+        ~/^refs\/tags\/(Ubuntu-lts-4\.8\.0-.*_16\.04\.\d+)$/,
+        ~/^refs\/tags\/(Ubuntu-lts-4\.10\.0-.*_16\.04\.\d+)$/,
+      ]
+
+      blacklist = [
+        'Ubuntu-lts-4.10.0-7.9_16.04.1',
+      ]
+      break
+
+    case 'trusty':
+      matchStrs = [
+        ~/^refs\/tags\/(Ubuntu-3\.13\.0-[\d\.]+)$/,
+        ~/^refs\/tags\/(Ubuntu-lts-.*_14\.04\.\d+)$/,
+      ]
+      break
+
+    default:
+      println 'Unsupported Ubuntu version: ${uversion}'
+      throw new InterruptedException()
+      break
+  }
+} else {
+  // Vanilla
+  kversionFactory = new VanillaKVersion()
+  matchStrs = [
+    ~/^refs\/tags\/(v[\d\.]+(-rc(\d+))?)$/,
+  ]
+}
+
+// Parse kernel versions
+def kverfloor = ""
+try {
+    kverfloor = kversionFactory.factory(kverfloor_raw)
+} catch (EmptyKVersionException e) {
+    kverfloor = kversionFactory.minKVersion()
+}
+
+def kverceil = ""
+try {
+    kverceil = kversionFactory.factory(kverceil_raw)
+} catch (EmptyKVersionException e) {
+    kverceil = kversionFactory.maxKVersion()
+}
+
+// Build a sorted list of versions to build
+for (ref in refs) {
+  for (matchStr in matchStrs) {
+    def match = ref.getName() =~ matchStr
+    if (match && !blacklist.contains(match.group(1))) {
+      def v = kversionFactory.factory(match.group(1))
+
+      if ((v >= kverfloor) && (v < kverceil)) {
+        if (v.isRC()) {
+          kversionsRC.add(v)
+        } else {
+          kversions.add(v)
+        }
+      }
+    }
+  }
+}
+
+kversions.sort()
+kversionsRC.sort()
+
+switch (kverfilter) {
+  case 'stable-head':
+    // Keep only the head of each stable branch
+    println('Filter kernel versions to keep only the latest point release of each stable branch.')
+
+    for (i = 0; i < kversions.size(); i++) {
+      def curr = kversions[i]
+      def next = i < kversions.size() - 1 ? kversions[i + 1] : null
+
+      if (next != null) {
+        if (curr.isSameStable(next)) {
+          kversions.remove(i)
+          i--
+        }
+      }
+    }
+    break
+
+  default:
+    // No filtering of kernel versions
+    println('No kernel versions filtering selected.')
+    break
+}
+
+// If the last RC version is newer than the last stable, add it to the build list
+if (kversionsRC.size() > 0 && kversionsRC.last() > kversions.last()) {
+  kversions.add(kversionsRC.last())
+}
+
+println "Building the following kernel versions:"
+for (k in kversions) {
+  println k
+}
+
+// Debug: Stop build here
+//throw new InterruptedException()
+
+def joburl = HyperlinkNote.encodeTo('/' + job.url, job.fullDisplayName)
+
+def allBuilds = []
+def ongoingBuild = []
+def failedRuns = []
+def isFailed = false
+def similarJobQueued = 0;
+
+// Loop while we have kernel versions remaining or jobs running
+while ( kversions.size() != 0 || ongoingBuild.size() != 0 ) {
+
+  if(ongoingBuild.size() < maxConcurrentBuild.toInteger() && kversions.size() != 0) {
+    def kversion = kversions.pop()
+    def job_params = [
+      new StringParameterValue('mversion', mversion),
+      new StringParameterValue('kversion', kversion.toString()),
+      new StringParameterValue('kgitrepo', kgitrepo),
+    ]
+
+    // Launch the parametrized build
+    def param_build = job.scheduleBuild2(0, new Cause.UpstreamCause(build), new ParametersAction(job_params))
+    println "triggering ${joburl} for the ${mversion} branch on kernel ${kversion}"
+
+    // Add it to the ongoing build queue
+    ongoingBuild.push(param_build)
+
+  } else {
+
+    println "Waiting... Queued: " + kversions.size() + " Running: " + ongoingBuild.size()
+    try {
+      Thread.sleep(10000)
+    } catch(e) {
+      if (e in InterruptedException) {
+        build.setResult(hudson.model.Result.ABORTED)
+        throw new InterruptedException()
+      } else {
+        throw(e)
+      }
+    }
+
+    // Abort job if a newer instance is queued
+    similarJobQueued = Hudson.instance.queue.items.count{it.task.getFullDisplayName() == currentJobName}
+    if ( similarJobQueued > 0 ) {
+        build.setResult(hudson.model.Result.ABORTED)
+        throw new InterruptedException()
+    }
+
+    def i = ongoingBuild.iterator()
+    while ( i.hasNext() ) {
+      currentBuild = i.next()
+      if ( currentBuild.isCancelled() || currentBuild.isDone() ) {
+        // Remove from queue
+        i.remove()
+
+        // Print results
+        def matrixParent = currentBuild.get()
+        allBuilds.add(matrixParent)
+        def kernelStr = matrixParent.buildVariableResolver.resolve("kversion")
+        println "${matrixParent.fullDisplayName} (${kernelStr}) completed with status ${matrixParent.result}"
+
+        // Process child runs of matrixBuild
+        def childRuns = matrixParent.getRuns()
+        for ( childRun in childRuns ) {
+          println "\t${childRun.fullDisplayName} (${kernelStr}) completed with status ${childRun.result}"
+          if (childRun.result != Result.SUCCESS) {
+            failedRuns.add(childRun)
+            isFailed = true
+          }
+        }
+      }
+    }
+  }
+}
+
+// Get log of failed runs
+for (failedRun in failedRuns) {
+  println "---START---"
+  failedRun.writeWholeLogTo(out)
+  println "---END---"
+}
+
+println "---Build report---"
+for (b in allBuilds) {
+  def kernelStr = b.buildVariableResolver.resolve("kversion")
+  println "${b.fullDisplayName} (${kernelStr}) completed with status ${b.result}"
+  // Cleanup builds
+  try {
+    b.delete()
+  } catch (all) {}
+}
+
+// Mark this build failed if any child build has failed
+if (isFailed) {
+  build.getExecutor().interrupt(Result.FAILURE)
+}
+
+// EOF