From 81fa42ce1c6a9cdd5b7674f8b93e477117f04073 Mon Sep 17 00:00:00 2001
From: Michael Jeanson <mjeanson@efficios.com>
Date: Mon, 29 Aug 2016 14:01:59 -0400
Subject: [PATCH] jjb: Add latency-tracker

Signed-off-by: Michael Jeanson <mjeanson@efficios.com>
---
 jobs/latency-tracker.yaml                     | 363 ++++++++++++++++++
 scripts/latency-tracker/master-rt.groovy      | 259 +++++++++++++
 scripts/latency-tracker/master-ubuntu.groovy  | 190 +++++++++
 scripts/latency-tracker/master-vanilla.groovy | 278 ++++++++++++++
 scripts/latency-tracker/param-build.sh        | 361 +++++++++++++++++
 5 files changed, 1451 insertions(+)
 create mode 100644 jobs/latency-tracker.yaml
 create mode 100644 scripts/latency-tracker/master-rt.groovy
 create mode 100644 scripts/latency-tracker/master-ubuntu.groovy
 create mode 100644 scripts/latency-tracker/master-vanilla.groovy
 create mode 100644 scripts/latency-tracker/param-build.sh

diff --git a/jobs/latency-tracker.yaml b/jobs/latency-tracker.yaml
new file mode 100644
index 0000000..292c027
--- /dev/null
+++ b/jobs/latency-tracker.yaml
@@ -0,0 +1,363 @@
+---
+- defaults:
+    name: latency-tracker
+    description: |
+
+      <p>Job is managed by Jenkins Job Builder.</p>
+
+    project-type: freestyle
+
+    wrappers:
+      - workspace-cleanup
+      - timestamps
+      - ansicolor
+
+    scm:
+      - git:
+          url: git://github.com/{github_user}/{github_name}.git
+          browser: githubweb
+          browser-url: https://github.com/{github_user}/{github_name}
+          branches:
+            - "{mversion}"
+          shallow-clone: true
+          skip-tag: true
+          fastpoll: true
+          basedir: src/latency-tracker
+
+    triggers:
+      - pollscm:
+          cron: "@hourly"
+
+    properties:
+      - build-discarder:
+          num-to-keep: 2
+      - github:
+          url: https://github.com/{github_user}/{github_name}
+
+
+## Templates
+- job-template:
+    name: latency-tracker_{mversion}_{kversion}_{buildtype}
+    defaults: latency-tracker
+
+    project-type: matrix
+    node: 'master' # Applies only to matrix flyweight task
+    axes:
+      - axis:
+         type: slave
+         name: arch
+         values: '{obj:arch}'
+
+    builders:
+      - copyartifact:
+          project: kernel_{kversion}_{buildtype}/arch=$arch
+          which-build: last-successful
+          stable: true
+          filter: 'build/**'
+          target: 'deps/linux'
+          do-not-fingerprint: true
+      - shell: |
+          git clone --depth=1 -b "v{kversion}" --reference $HOME/gitcache/linux-stable.git/ git://git-mirror.internal.efficios.com/kernel/stable/linux-stable.git src/linux
+      - shell:
+         !include-raw-escape: scripts/latency-tracker/build.sh
+
+    publishers:
+      - archive:
+          artifacts: 'build/**'
+          allow-empty: false
+      - workspace-cleanup
+
+- job-template:
+    name: latency-tracker_{mversion}_build-vanilla
+    defaults: latency-tracker
+    description: |
+      The LTTng modules provide Linux kernel tracing capability to the LTTng
+      2.0 tracer toolset.
+
+      This job will build the {mversion} branch against all stable vanilla
+      kernel tags.
+
+      <p>Job is managed by Jenkins Job Builder.</p>
+
+    node: 'master'
+
+    parameters:
+      - string:
+          name: 'mversion'
+          default: '{mversion}'
+          description: 'The latency-tracker branch to build.'
+      - string:
+          name: 'maxConcurrentBuild'
+          default: '20'
+          description: 'The maximum number of concurrent child build to run.'
+      - string:
+          name: 'kverfloor'
+          default: 'v2.6.36'
+          description: 'The lowest kernel version to build.'
+      - string:
+          name: 'kgitrepo'
+          default: 'git://git-mirror.internal.efficios.com/kernel/stable/linux-stable.git'
+          description: 'The linux kernel git repository url.'
+      - string:
+          name: 'kbuildjob'
+          default: 'latency-tracker_VERSION_param-build'
+          description: 'The parametrized job to use for child builds.'
+
+    builders:
+      - system-groovy:
+         command:
+           !include-raw-escape: scripts/latency-tracker/master-vanilla.groovy
+
+    publishers:
+      - workspace-cleanup
+
+- job-template:
+    name: latency-tracker_{mversion}_build-{uversion}
+    defaults: latency-tracker
+    description: |
+      The LTTng modules provide Linux kernel tracing capability to the LTTng
+      2.0 tracer toolset.
+
+      This job will build the {mversion} branch against all Ubuntu {uversion}
+      released kernels, including the LTS backport kernels.
+
+      <p>Job is managed by Jenkins Job Builder.</p>
+
+    node: 'master'
+
+    parameters:
+      - string:
+          name: 'mversion'
+          default: '{mversion}'
+          description: 'The latency-tracker branch to build.'
+      - string:
+          name: 'maxConcurrentBuild'
+          default: '20'
+          description: 'The maximum number of concurrent child build to run.'
+      - string:
+          name: 'uversion'
+          default: '{uversion}'
+          description: 'The lowest kernel version to build.'
+      - string:
+          name: 'kgitrepo'
+          default: 'git://git-mirror.internal.efficios.com/git/ubuntu-{uversion}.git'
+          description: 'The linux kernel git repository url.'
+      - string:
+          name: 'kbuildjob'
+          default: 'latency-tracker_VERSION_param-build'
+          description: 'The parametrized job to use for child builds.'
+
+    builders:
+      - system-groovy:
+         command:
+           !include-raw-escape: scripts/latency-tracker/master-ubuntu.groovy
+
+    publishers:
+      - workspace-cleanup
+
+- job-template:
+    name: latency-tracker_{mversion}_build-rt
+    defaults: latency-tracker
+    description: |
+      The LTTng modules provide Linux kernel tracing capability to the LTTng
+      2.0 tracer toolset.
+
+      This job will build the {mversion} branch against all Linutronix RT
+      kernels.
+
+      <p>Job is managed by Jenkins Job Builder.</p>
+
+    node: 'master'
+
+    parameters:
+      - string:
+          name: 'mversion'
+          default: '{mversion}'
+          description: 'The latency-tracker branch to build.'
+      - string:
+          name: 'maxConcurrentBuild'
+          default: '20'
+          description: 'The maximum number of concurrent child build to run.'
+      - string:
+          name: 'kverfloor'
+          default: 'v2.6.36-rt0-rebase'
+          description: 'The lowest kernel version to build.'
+      - string:
+          name: 'kgitrepo'
+          default: 'git://git-mirror.internal.efficios.com/kernel/rt/linux-rt-devel.git'
+          description: 'The linux kernel git repository url.'
+      - string:
+          name: 'kbuildjob'
+          default: 'latency-tracker_VERSION_param-build'
+          description: 'The parametrized job to use for child builds.'
+
+    builders:
+      - system-groovy:
+         command:
+           !include-raw-escape: scripts/latency-tracker/master-rt.groovy
+
+    publishers:
+      - workspace-cleanup
+
+- job-template:
+    name: latency-tracker_VERSION_param-build
+    defaults: latency-tracker
+    description: |
+      This is a parametrized job used by 'master' jobs to build any combinations
+      of latency-tracker and linux kernel versions.
+
+      <p>Job is managed by Jenkins Job Builder.</p>
+
+    project-type: matrix
+    node: 'master' # Applies only to matrix flyweight task
+    axes:
+      - axis:
+          type: slave
+          name: arch
+          values: '{obj:arch}'
+
+    properties:
+      - build-discarder:
+          days-to-keep: 2
+
+    parameters:
+      - string:
+          name: 'mversion'
+          default: 'master'
+          description: 'The latency-tracker branch to build.'
+      - string:
+          name: 'kversion'
+          default: ''
+          description: 'The linux kernel git tag to build against.'
+      - string:
+          name: 'kgitrepo'
+          default: 'git://git-mirror.internal.efficios.com/kernel/stable/linux-stable.git'
+          description: 'The linux kernel git repository url.'
+
+    concurrent: true
+
+    scm:
+      - git:
+          url: git://github.com/efficios/latency-tracker.git
+          browser: githubweb
+          browser-url: https://github.com/efficios/latency-tracker
+          branches:
+            - "${{mversion}}"
+          skip-tag: true
+          basedir: src/latency-tracker
+
+    triggers:
+
+    builders:
+      - shell: |
+          git clone --depth=1 -b "$kversion" --reference $HOME/gitcache/linux-stable.git/ "$kgitrepo" src/linux
+      - shell:
+          !include-raw-escape: scripts/latency-tracker/param-build.sh
+
+    publishers:
+      - workspace-cleanup
+
+- job-template:
+    name: latency-tracker_{mversion}_coverity
+    defaults: latency-tracker
+    node: 'x86-64'
+
+    triggers:
+      - pollscm:
+          cron: "@daily"
+
+    wrappers:
+      - workspace-cleanup
+      - timestamps
+      - ansicolor:
+          colormap: xterm
+      - credentials-binding:
+          - username-password-separated:
+              credential-id: latency-tracker_coverity_token
+              username: COVERITY_SCAN_PROJECT_NAME
+              password: COVERITY_SCAN_TOKEN
+
+    builders:
+      - shell: |
+         git clone --depth=1 -b v4.4 --reference $HOME/gitcache/linux-stable.git/ git://git-mirror.internal.efficios.com/kernel/stable/linux-stable.git src/linux
+         cd src/linux
+         make defconfig
+         sed -i "s/# CONFIG_KALLSYMS_ALL is not set/CONFIG_KALLSYMS_ALL=y/g" .config
+         make modules_prepare
+      - shell:
+         !include-raw-escape: scripts/common/coverity.sh
+
+    publishers:
+      - workspace-cleanup
+
+- job-template:
+    name: latency-tracker_{mversion}_cppcheck
+    defaults: latency-tracker
+
+    triggers:
+      - pollscm:
+          cron: "@daily"
+
+    builders:
+      - shell: |
+          rm -f cppcheck.xml
+          cppcheck --enable=all --xml --xml-version=2 $WORKSPACE/src/latency-tracker 2> cppcheck.xml
+
+    publishers:
+      - archive:
+          artifacts: 'cppcheck.xml'
+          allow-empty: false
+      - cppcheck:
+          pattern: 'cppcheck.xml'
+      - email:
+          recipients: 'ci-notification@lists.lttng.org'
+          notify-every-unstable-build: true
+          send-to-individuals: false
+
+- job-template:
+    name: latency-tracker_{mversion}_sloccount
+    defaults: latency-tracker
+    description: |
+      The LTTng modules provide Linux kernel tracing capability to the LTTng
+      2.0 tracer toolset.
+
+      This job runs the sloccount utility and generates a trend report.
+
+      <p>Job is managed by Jenkins Job Builder.</p>
+
+    triggers:
+      - pollscm:
+          cron: "@daily"
+
+    builders:
+      - shell: |
+          cloc --by-file --xml --out=cloc.xml src/latency-tracker/
+
+    publishers:
+      - archive:
+          artifacts: 'cloc.xml'
+          allow-empty: false
+      - sloccount:
+          report-files: 'cloc.xml'
+
+
+## Project
+- project:
+    name: latency-tracker
+    github_user: efficios
+    github_name: latency-tracker
+    mversion:
+      - master
+    jobs:
+      - 'latency-tracker_{mversion}_build-vanilla'
+      - 'latency-tracker_{mversion}_build-rt':
+#      - 'latency-tracker_{mversion}_build-{uversion}':
+#          uversion:
+#            - xenial
+      - 'latency-tracker_VERSION_param-build':
+          arch: !!python/tuple [x86-32, x86-64]
+      - 'latency-tracker_{mversion}_cppcheck'
+      - 'latency-tracker_{mversion}_sloccount':
+          mversion: master
+#      - 'latency-tracker_{mversion}_coverity':
+#          mversion: master
diff --git a/scripts/latency-tracker/master-rt.groovy b/scripts/latency-tracker/master-rt.groovy
new file mode 100644
index 0000000..177041d
--- /dev/null
+++ b/scripts/latency-tracker/master-rt.groovy
@@ -0,0 +1,259 @@
+/**
+ * Copyright (C) 2016 - Michael Jeanson <mjeanson@efficios.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import hudson.model.*
+import hudson.AbortException
+import hudson.console.HyperlinkNote
+import java.util.concurrent.CancellationException
+import org.eclipse.jgit.api.Git
+import org.eclipse.jgit.lib.Ref
+
+
+class kVersion implements Comparable<kVersion> {
+
+  Integer major = 0;
+  Integer majorB = 0;
+  Integer minor = 0;
+  Integer patch = 0;
+  Integer rt = 0;
+
+  kVersion() {}
+
+  kVersion(version) {
+    this.parse(version)
+  }
+
+  def parse(version) {
+    this.major = 0
+    this.majorB = 0
+    this.minor = 0
+    this.patch = 0
+    this.rt = 0
+
+    def match = version =~ /^v(\d+)\.(\d+)(\.(\d+))?(\.(\d+))?(-rt(\d+)-rebase)$/
+    if (!match) {
+      throw new Exception("Invalid kernel version: ${version}")
+    }
+
+    Integer offset = 0;
+
+    // Major
+    this.major = Integer.parseInt(match.group(1))
+    if (this.major <= 2) {
+      offset = 2
+      this.majorB = Integer.parseInt(match.group(2))
+    }
+
+    // Minor
+    if (match.group(2 + offset) != null) {
+      this.minor = Integer.parseInt(match.group(2 + offset))
+    }
+
+    // Patch level
+    if (match.group(4 + offset) != null) {
+      this.patch = Integer.parseInt(match.group(4 + offset))
+    }
+
+    // RT
+    this.rt = Integer.parseInt(match.group(8))
+  }
+
+  @Override int compareTo(kVersion o) {
+    if (this.major != o.major) {
+      return Integer.compare(this.major, o.major);
+    }
+    if (this.majorB != o.majorB) {
+      return Integer.compare(this.majorB, o.majorB);
+    }
+    if (this.minor != o.minor) {
+      return Integer.compare(this.minor, o.minor);
+    }
+    if (this.patch != o.patch) {
+      return Integer.compare(this.patch, o.patch);
+    }
+    if (this.rt != o.rc) {
+      return Integer.compare(this.rt, o.rt);
+    }
+
+    // Same version
+    return 0;
+  }
+
+  String toString() {
+    String vString = "v${this.major}"
+
+    if (this.majorB > 0) {
+      vString = vString.concat(".${this.majorB}")
+    }
+
+    vString = vString.concat(".${this.minor}")
+
+    if (this.patch > 0) {
+      vString = vString.concat(".${this.patch}")
+    }
+
+    if (this.rt > 0) {
+      vString = vString.concat("-rt${this.rt}-rebase")
+    }
+    return vString
+  }
+}
+
+
+// Retrieve parameters of the current build
+def mversion = build.buildVariableResolver.resolve('mversion')
+def maxConcurrentBuild = build.buildVariableResolver.resolve('maxConcurrentBuild')
+def kgitrepo = build.buildVariableResolver.resolve('kgitrepo')
+def kverfloor = new kVersion(build.buildVariableResolver.resolve('kverfloor'))
+def job = Hudson.instance.getJob(build.buildVariableResolver.resolve('kbuildjob'))
+def currentJobName = build.project.getFullDisplayName()
+
+// Get the out variable
+def config = new HashMap()
+def bindings = getBinding()
+config.putAll(bindings.getVariables())
+def out = config['out']
+
+def jlc = new jenkins.model.JenkinsLocationConfiguration()
+def jenkinsUrl = jlc.url
+
+// Get tags from git repository
+def refs = Git.lsRemoteRepository().setTags(true).setRemote(kgitrepo).call();
+
+// Get kernel versions to build
+def kversions = []
+for (ref in refs) {
+  def match = ref.getName() =~ /^refs\/tags\/(v[\d\.]+(-rt(\d+)-rebase))$/
+
+  if (match) {
+    def v = new kVersion(match.group(1))
+
+    if (v >= kverfloor) {
+      kversions.add(v)
+    }
+  }
+}
+
+kversions.sort()
+
+// Debug
+println "Building the following kernel versions:"
+for (k in kversions) {
+  println k
+}
+
+// Debug: Stop build here
+//throw new InterruptedException()
+
+def joburl = HyperlinkNote.encodeTo('/' + job.url, job.fullDisplayName)
+
+def allBuilds = []
+def ongoingBuild = []
+def failedRuns = []
+def isFailed = false
+
+// Loop while we have kernel versions remaining or jobs running
+while ( kversions.size() != 0 || ongoingBuild.size() != 0 ) {
+
+  if(ongoingBuild.size() < maxConcurrentBuild.toInteger() && kversions.size() != 0) {
+    def kversion = kversions.pop()
+    def job_params = [
+      new StringParameterValue('mversion', mversion),
+      new StringParameterValue('kversion', kversion.toString()),
+      new StringParameterValue('kgitrepo', kgitrepo),
+    ]
+
+    // Launch the parametrized build
+    def param_build = job.scheduleBuild2(0, new Cause.UpstreamCause(build), new ParametersAction(job_params))
+    println "triggering ${joburl} for the ${mversion} branch on kernel ${kversion}"
+
+    // Add it to the ongoing build queue
+    ongoingBuild.push(param_build)
+
+  } else {
+
+    println "Waiting... Queued: " + kversions.size() + " Running: " + ongoingBuild.size()
+    try {
+      Thread.sleep(5000)
+    } catch(e) {
+      if (e in InterruptedException) {
+        build.setResult(hudson.model.Result.ABORTED)
+        throw new InterruptedException()
+      } else {
+        throw(e)
+      }
+    }
+
+    // Check for queued similar job since we only want to run latest
+    // as Mathieu Desnoyers requirement
+    similarJobQueued = Hudson.instance.queue.items.count{it.task.getFullDisplayName() == currentJobName}
+    if ( similarJobQueued > 0 ) {
+	 // Abort since new build is queued
+        build.setResult(hudson.model.Result.ABORTED)
+        throw new InterruptedException()
+    }
+
+    def i = ongoingBuild.iterator()
+    while ( i.hasNext() ) {
+      currentBuild = i.next()
+      if ( currentBuild.isCancelled() || currentBuild.isDone() ) {
+        // Remove from queue
+        i.remove()
+
+        // Print results
+        def matrixParent = currentBuild.get()
+        allBuilds.add(matrixParent)
+        def kernelStr = matrixParent.buildVariableResolver.resolve("kversion")
+        println "${matrixParent.fullDisplayName} (${kernelStr}) completed with status ${matrixParent.result}"
+
+        // Process child runs of matrixBuild
+        def childRuns = matrixParent.getRuns()
+        for ( childRun in childRuns ) {
+          println "\t${childRun.fullDisplayName} (${kernelStr}) completed with status ${childRun.result}"
+          if (childRun.result != Result.SUCCESS) {
+            failedRuns.add(childRun)
+            isFailed = true
+          }
+        }
+      }
+    }
+  }
+}
+
+// Get log of failed runs
+for (failedRun in failedRuns) {
+  println "---START---"
+  failedRun.writeWholeLogTo(out)
+  println "---END---"
+}
+
+println "---Build report---"
+for (b in allBuilds) {
+  def kernelStr = b.buildVariableResolver.resolve("kversion")
+  println "${b.fullDisplayName} (${kernelStr}) completed with status ${b.result}"
+  // Cleanup builds
+  try {
+    b.delete()
+  } catch (all) {}
+}
+
+// Mark this build failed if any child build has failed
+if (isFailed) {
+  build.getExecutor().interrupt(Result.FAILURE)
+}
+
+// EOF
diff --git a/scripts/latency-tracker/master-ubuntu.groovy b/scripts/latency-tracker/master-ubuntu.groovy
new file mode 100644
index 0000000..da5f1fa
--- /dev/null
+++ b/scripts/latency-tracker/master-ubuntu.groovy
@@ -0,0 +1,190 @@
+/**
+ * Copyright (C) 2016 - Michael Jeanson <mjeanson@efficios.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import hudson.model.*
+import hudson.AbortException
+import hudson.console.HyperlinkNote
+import java.util.concurrent.CancellationException
+import org.eclipse.jgit.api.Git
+import org.eclipse.jgit.lib.Ref
+
+
+// Retrieve parameters of the current build
+def mversion = build.buildVariableResolver.resolve('mversion')
+def maxConcurrentBuild = build.buildVariableResolver.resolve('maxConcurrentBuild')
+def kgitrepo = build.buildVariableResolver.resolve('kgitrepo')
+def uversion = build.buildVariableResolver.resolve('uversion')
+def job = Hudson.instance.getJob(build.buildVariableResolver.resolve('kbuildjob'))
+def currentJobName = build.project.getFullDisplayName()
+
+// Get the out variable
+def config = new HashMap()
+def bindings = getBinding()
+config.putAll(bindings.getVariables())
+def out = config['out']
+
+def jlc = new jenkins.model.JenkinsLocationConfiguration()
+def jenkinsUrl = jlc.url
+
+// Get tags from git repository
+def refs = Git.lsRemoteRepository().setTags(true).setRemote(kgitrepo).call();
+
+// Get kernel versions to build
+def kversions = []
+
+def matchStrs = []
+
+switch (uversion) {
+  case 'xenial':
+    matchStrs = [
+      ~/^refs\/tags\/(Ubuntu-4\.4\.0-\d{1,3}\.[\d\.]+)$/,
+      ~/^refs\/tags\/(Ubuntu-lts-.*_16\.04\.\d+)$/,
+    ]
+    break
+
+  case 'trusty':
+    matchStrs = [
+      ~/^refs\/tags\/(Ubuntu-3\.13\.0-[\d\.]+)$/,
+      ~/^refs\/tags\/(Ubuntu-lts-.*_14\.04\.\d+)$/,
+    ]
+    break
+
+  default:
+    println 'Unsupported Ubuntu version: ${uversion}'
+    throw new InterruptedException()
+    break
+}
+
+for (ref in refs) {
+  for (matchStr in matchStrs) {
+    def match = ref.getName() =~ matchStr
+
+    if (match) {
+      kversions.add(match.group(1))
+    }
+  }
+}
+
+kversions.sort()
+
+// Debug
+println "Building the following kernel versions:"
+for (k in kversions) {
+  println k
+}
+
+// Debug: Stop build here
+//throw new InterruptedException()
+
+def joburl = HyperlinkNote.encodeTo('/' + job.url, job.fullDisplayName)
+
+def allBuilds = []
+def ongoingBuild = []
+def failedRuns = []
+def isFailed = false
+
+// Loop while we have kernel versions remaining or jobs running
+while ( kversions.size() != 0 || ongoingBuild.size() != 0 ) {
+
+  if(ongoingBuild.size() < maxConcurrentBuild.toInteger() && kversions.size() != 0) {
+    def kversion = kversions.pop()
+    def job_params = [
+      new StringParameterValue('mversion', mversion),
+      new StringParameterValue('kversion', kversion),
+      new StringParameterValue('kgitrepo', kgitrepo),
+    ]
+
+    // Launch the parametrized build
+    def param_build = job.scheduleBuild2(0, new Cause.UpstreamCause(build), new ParametersAction(job_params))
+    println "triggering ${joburl} for the ${mversion} branch on kernel ${kversion}"
+
+    // Add it to the ongoing build queue
+    ongoingBuild.push(param_build)
+
+  } else {
+
+    println "Waiting... Queued: " + kversions.size() + " Running: " + ongoingBuild.size()
+    try {
+      Thread.sleep(5000)
+    } catch(e) {
+      if (e in InterruptedException) {
+        build.setResult(hudson.model.Result.ABORTED)
+        throw new InterruptedException()
+      } else {
+        throw(e)
+      }
+    }
+
+    // Check for queued similar job since we only want to run latest
+    // as Mathieu Desnoyers requirement
+    similarJobQueued = Hudson.instance.queue.items.count{it.task.getFullDisplayName() == currentJobName}
+    if ( similarJobQueued > 0 ) {
+	 // Abort since new build is queued
+        build.setResult(hudson.model.Result.ABORTED)
+        throw new InterruptedException()
+    }
+
+    def i = ongoingBuild.iterator()
+    while ( i.hasNext() ) {
+      currentBuild = i.next()
+      if ( currentBuild.isCancelled() || currentBuild.isDone() ) {
+        // Remove from queue
+        i.remove()
+
+        // Print results
+        def matrixParent = currentBuild.get()
+        allBuilds.add(matrixParent)
+        def kernelStr = matrixParent.buildVariableResolver.resolve("kversion")
+        println "${matrixParent.fullDisplayName} (${kernelStr}) completed with status ${matrixParent.result}"
+
+        // Process child runs of matrixBuild
+        def childRuns = matrixParent.getRuns()
+        for ( childRun in childRuns ) {
+          println "\t${childRun.fullDisplayName} (${kernelStr}) completed with status ${childRun.result}"
+          if (childRun.result != Result.SUCCESS) {
+            failedRuns.add(childRun)
+            isFailed = true
+          }
+        }
+      }
+    }
+  }
+}
+
+// Get log of failed runs
+for (failedRun in failedRuns) {
+  println "---START---"
+  failedRun.writeWholeLogTo(out)
+  println "---END---"
+}
+
+println "---Build report---"
+for (b in allBuilds) {
+  def kernelStr = b.buildVariableResolver.resolve("kversion")
+  println "${b.fullDisplayName} (${kernelStr}) completed with status ${b.result}"
+  // Cleanup builds
+  try {
+    b.delete()
+  } catch (all) {}
+}
+
+// Mark this build failed if any child build has failed
+if (isFailed) {
+  build.getExecutor().interrupt(Result.FAILURE)
+}
+
+// EOF
diff --git a/scripts/latency-tracker/master-vanilla.groovy b/scripts/latency-tracker/master-vanilla.groovy
new file mode 100644
index 0000000..95b240e
--- /dev/null
+++ b/scripts/latency-tracker/master-vanilla.groovy
@@ -0,0 +1,278 @@
+/**
+ * Copyright (C) 2016 - Michael Jeanson <mjeanson@efficios.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import hudson.model.*
+import hudson.AbortException
+import hudson.console.HyperlinkNote
+import java.util.concurrent.CancellationException
+import org.eclipse.jgit.api.Git
+import org.eclipse.jgit.lib.Ref
+
+
+class kVersion implements Comparable<kVersion> {
+
+  Integer major = 0;
+  Integer majorB = 0;
+  Integer minor = 0;
+  Integer patch = 0;
+  Integer rc = Integer.MAX_VALUE;
+
+  kVersion() {}
+
+  kVersion(version) {
+    this.parse(version)
+  }
+
+  def parse(version) {
+    this.major = 0
+    this.majorB = 0
+    this.minor = 0
+    this.patch = 0
+    this.rc = Integer.MAX_VALUE
+
+    def match = version =~ /^v(\d+)\.(\d+)(\.(\d+))?(\.(\d+))?(-rc(\d+))?$/
+    if (!match) {
+      throw new Exception("Invalid kernel version: ${version}")
+    }
+
+    Integer offset = 0;
+
+    // Major
+    this.major = Integer.parseInt(match.group(1))
+    if (this.major <= 2) {
+      offset = 2
+      this.majorB = Integer.parseInt(match.group(2))
+    }
+
+    // Minor
+    if (match.group(2 + offset) != null) {
+      this.minor = Integer.parseInt(match.group(2 + offset))
+    }
+
+    // Patch level
+    if (match.group(4 + offset) != null) {
+      this.patch = Integer.parseInt(match.group(4 + offset))
+    }
+
+    // RC
+    if (match.group(8) != null) {
+      this.rc = Integer.parseInt(match.group(8))
+    }
+  }
+
+  // Return true if this version is a release candidate
+  Boolean isRC() {
+    return this.rc != Integer.MAX_VALUE
+  }
+
+  @Override int compareTo(kVersion o) {
+    if (this.major != o.major) {
+      return Integer.compare(this.major, o.major);
+    }
+    if (this.majorB != o.majorB) {
+      return Integer.compare(this.majorB, o.majorB);
+    }
+    if (this.minor != o.minor) {
+      return Integer.compare(this.minor, o.minor);
+    }
+    if (this.patch != o.patch) {
+      return Integer.compare(this.patch, o.patch);
+    }
+    if (this.rc != o.rc) {
+      return Integer.compare(this.rc, o.rc);
+    }
+
+    // Same version
+    return 0;
+  }
+
+  String toString() {
+    String vString = "v${this.major}"
+
+    if (this.majorB > 0) {
+      vString = vString.concat(".${this.majorB}")
+    }
+
+    vString = vString.concat(".${this.minor}")
+
+    if (this.patch > 0) {
+      vString = vString.concat(".${this.patch}")
+    }
+
+    if (this.rc > 0 && this.rc < Integer.MAX_VALUE) {
+      vString = vString.concat("-rc${this.rc}")
+    }
+    return vString
+  }
+}
+
+
+// Retrieve parameters of the current build
+def mversion = build.buildVariableResolver.resolve('mversion')
+def maxConcurrentBuild = build.buildVariableResolver.resolve('maxConcurrentBuild')
+def kgitrepo = build.buildVariableResolver.resolve('kgitrepo')
+def kverfloor = new kVersion(build.buildVariableResolver.resolve('kverfloor'))
+def job = Hudson.instance.getJob(build.buildVariableResolver.resolve('kbuildjob'))
+def currentJobName = build.project.getFullDisplayName()
+
+// Get the out variable
+def config = new HashMap()
+def bindings = getBinding()
+config.putAll(bindings.getVariables())
+def out = config['out']
+
+def jlc = new jenkins.model.JenkinsLocationConfiguration()
+def jenkinsUrl = jlc.url
+
+// Get tags from git repository
+def refs = Git.lsRemoteRepository().setTags(true).setRemote(kgitrepo).call();
+
+// Get kernel versions to build
+def kversions = []
+def kversionsRC = []
+for (ref in refs) {
+  def match = ref.getName() =~ /^refs\/tags\/(v[\d\.]+(-rc(\d+))?)$/
+
+  if (match) {
+    def v = new kVersion(match.group(1))
+
+    if (v >= kverfloor) {
+      if (v.isRC()) {
+        kversionsRC.add(v)
+      } else {
+        kversions.add(v)
+      }
+    }
+  }
+}
+
+kversions.sort()
+kversionsRC.sort()
+
+// If the last RC version is newer than the last stable, add it to the build list
+if (kversionsRC.last() > kversions.last()) {
+  kversions.add(kversionsRC.last())
+}
+
+// Debug
+println "Building the following kernel versions:"
+for (k in kversions) {
+  println k
+}
+
+// Debug: Stop build here
+//throw new InterruptedException()
+
+def joburl = HyperlinkNote.encodeTo('/' + job.url, job.fullDisplayName)
+
+def allBuilds = []
+def ongoingBuild = []
+def failedRuns = []
+def isFailed = false
+def similarJobQueued = 0;
+
+// Loop while we have kernel versions remaining or jobs running
+while ( kversions.size() != 0 || ongoingBuild.size() != 0 ) {
+
+  if(ongoingBuild.size() < maxConcurrentBuild.toInteger() && kversions.size() != 0) {
+    def kversion = kversions.pop()
+    def job_params = [
+      new StringParameterValue('mversion', mversion),
+      new StringParameterValue('kversion', kversion.toString()),
+      new StringParameterValue('kgitrepo', kgitrepo),
+    ]
+
+    // Launch the parametrized build
+    def param_build = job.scheduleBuild2(0, new Cause.UpstreamCause(build), new ParametersAction(job_params))
+    println "triggering ${joburl} for the ${mversion} branch on kernel ${kversion}"
+
+    // Add it to the ongoing build queue
+    ongoingBuild.push(param_build)
+
+  } else {
+
+    println "Waiting... Queued: " + kversions.size() + " Running: " + ongoingBuild.size()
+    try {
+      Thread.sleep(5000)
+    } catch(e) {
+      if (e in InterruptedException) {
+        build.setResult(hudson.model.Result.ABORTED)
+        throw new InterruptedException()
+      } else {
+        throw(e)
+      }
+    }
+
+    // Check for queued similar job since we only want to run latest
+    // as Mathieu Desnoyers requirement
+    similarJobQueued = Hudson.instance.queue.items.count{it.task.getFullDisplayName() == currentJobName}
+    if ( similarJobQueued > 0 ) {
+	 // Abort since new build is queued
+        build.setResult(hudson.model.Result.ABORTED)
+        throw new InterruptedException()
+    }
+
+    def i = ongoingBuild.iterator()
+    while ( i.hasNext() ) {
+      currentBuild = i.next()
+      if ( currentBuild.isCancelled() || currentBuild.isDone() ) {
+        // Remove from queue
+        i.remove()
+
+        // Print results
+        def matrixParent = currentBuild.get()
+        allBuilds.add(matrixParent)
+        def kernelStr = matrixParent.buildVariableResolver.resolve("kversion")
+        println "${matrixParent.fullDisplayName} (${kernelStr}) completed with status ${matrixParent.result}"
+
+        // Process child runs of matrixBuild
+        def childRuns = matrixParent.getRuns()
+        for ( childRun in childRuns ) {
+          println "\t${childRun.fullDisplayName} (${kernelStr}) completed with status ${childRun.result}"
+          if (childRun.result != Result.SUCCESS) {
+            failedRuns.add(childRun)
+            isFailed = true
+          }
+        }
+      }
+    }
+  }
+}
+
+// Get log of failed runs
+for (failedRun in failedRuns) {
+  println "---START---"
+  failedRun.writeWholeLogTo(out)
+  println "---END---"
+}
+
+println "---Build report---"
+for (b in allBuilds) {
+  def kernelStr = b.buildVariableResolver.resolve("kversion")
+  println "${b.fullDisplayName} (${kernelStr}) completed with status ${b.result}"
+  // Cleanup builds
+  try {
+    b.delete()
+  } catch (all) {}
+}
+
+// Mark this build failed if any child build has failed
+if (isFailed) {
+  build.getExecutor().interrupt(Result.FAILURE)
+}
+
+// EOF
diff --git a/scripts/latency-tracker/param-build.sh b/scripts/latency-tracker/param-build.sh
new file mode 100644
index 0000000..80a23ca
--- /dev/null
+++ b/scripts/latency-tracker/param-build.sh
@@ -0,0 +1,361 @@
+#!/bin/sh -exu
+#
+# Copyright (C) 2016 - Michael Jeanson <mjeanson@efficios.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+## FUNCTIONS ##
+
+# Kernel version compare functions
+verlte() {
+    [  "$1" = "`printf '%s\n%s' $1 $2 | sort -V | head -n1`" ]
+}
+
+verlt() {
+    [ "$1" = "$2" ] && return 1 || verlte $1 $2
+}
+
+vergte() {
+    [  "$1" = "`printf '%s\n%s' $1 $2 | sort -V | tail -n1`" ]
+}
+
+vergt() {
+    [ "$1" = "$2" ] && return 1 || vergte $1 $2
+}
+
+
+prepare_lnx_sources() {
+
+    outdir=$1
+
+    if [ "$outdir" = "." ]; then
+      koutput=""
+    else
+      koutput="O=\"${outdir}\""
+    fi
+
+    # Generate kernel configuration
+    case "$kversion" in
+      Ubuntu*)
+        fakeroot debian/rules clean
+        fakeroot debian/rules genconfigs
+        cp CONFIGS/${ubuntu_config} "${outdir}"/.config
+        ;;
+      *)
+        # Que sera sera
+        make ${koutput} allyesconfig CC=$CC
+        ;;
+    esac
+
+    # GCC 4.8
+    sed -i "s/CONFIG_CC_STACKPROTECTOR_STRONG=y/# CONFIG_CC_STACKPROTECTOR_STRONG is not set/g" "${outdir}"/.config
+
+    # Don't try to sign modules
+    sed -i "s/CONFIG_MODULE_SIG=y/# CONFIG_MODULE_SIG is not set/g" "${outdir}"/.config
+
+    # Disable kernel stack frame correctness validation, introduced in 4.6.0 and currently fails
+    sed -i "s/CONFIG_STACK_VALIDATION=y/# CONFIG_STACK_VALIDATION is not set/g" "${outdir}"/.config
+
+    # Enable CONFIG_KALLSYMS_ALL
+    echo "CONFIG_KPROBES=y" >> "${outdir}"/.config
+    echo "CONFIG_FTRACE=y" >> "${outdir}"/.config
+    echo "CONFIG_BLK_DEV_IO_TRACE=y" >> "${outdir}"/.config
+    echo "CONFIG_TRACEPOINTS=y" >> "${outdir}"/.config
+    echo "CONFIG_KALLSYMS_ALL=y" >> "${outdir}"/.config
+
+
+    make ${koutput} silentoldconfig CC=$CC
+    make ${koutput} modules_prepare CC=$CC
+
+    # Version specific tasks
+    case "$kversion" in
+      Ubuntu*)
+        # Add Ubuntu ABI number to kernel headers, this is normally done by the packaging code
+        ABINUM=$(echo $kversion | grep -P -o 'Ubuntu-(lts-)?.*-\K\d+(?=\..*)')
+        echo "#define UTS_UBUNTU_RELEASE_ABI $ABINUM" >> ${outdir}/include/generated/utsrelease.h
+        ;;
+    esac
+
+    # On powerpc this object is required to link modules
+    if [ "${karch}" = "powerpc" ]; then
+        make ${koutput} arch/powerpc/lib/crtsavres.o CC=$CC
+    fi
+}
+
+
+
+build_modules() {
+
+    kdir="$1"
+    bdir="$2"
+
+    # Get kernel version from source tree
+    cd "${kdir}"
+    kversion=$(make kernelversion)
+
+    # Enter latency-tracker source dir
+    cd "${LTTSRCDIR}"
+
+    # kernels 3.10 to 3.10.13 and 3.11 to 3.11.2 introduce a deadlock in the
+    # timekeeping subsystem. We want those build to fail.
+    if { vergte "$kversion" "3.10" && verlte "$kversion" "3.10.13"; } || \
+       { vergte "$kversion" "3.11" && verlte "$kversion" "3.11.2"; }; then
+
+        set +e
+
+        # Build modules
+        KERNELDIR="${kdir}" make -j${NPROC} V=1 CC=$CC
+
+        # We expect this build to fail, if it doesn't, fail the job.
+        if [ "$?" -eq 0 ]; then
+            exit 1
+        fi
+
+        # We have to publish at least one file or the build will fail
+        echo "This kernel is broken, there is a deadlock in the timekeeping subsystem." > "${bdir}/BROKEN.txt.ko"
+
+        set -e
+
+        KERNELDIR="${kdir}" make clean CC=$CC
+
+    else # Regular build
+
+        # Build modules against full kernel sources
+        KERNELDIR="${kdir}" make -j${NPROC} V=1 CC=$CC
+
+        # Install modules to build dir
+        KERNELDIR="${kdir}" make INSTALL_MOD_PATH="${bdir}" modules_install CC=$CC
+
+        # Clean build dir
+        KERNELDIR="${kdir}" make clean CC=$CC
+    fi
+}
+
+
+## MAIN ##
+
+# Use gcc 4.9, older kernel don't build with gcc 5
+export CC=gcc-4.9
+
+# Use all CPU cores
+NPROC=$(nproc)
+
+LTTSRCDIR="${WORKSPACE}/src/latency-tracker"
+LNXSRCDIR="${WORKSPACE}/src/linux"
+
+LNXBUILDDIR="${WORKSPACE}/build/linux"
+LNXHDRDIR="${WORKSPACE}/build/linux-headers"
+
+LTTBUILDKSRCDIR="${WORKSPACE}/build/latency-tracker-ksrc"
+LTTBUILDKHDRDIR="${WORKSPACE}/build/latency-tracker-khdr"
+
+
+# Setup cross compile env if available
+if [ "x${cross_arch:-}" != "x" ]; then
+
+    case "$cross_arch" in
+        "armhf")
+            karch="arm"
+            cross_compile="arm-linux-gnueabihf-"
+            ubuntu_config="armhf-config.flavour.generic"
+            ;;
+
+        "arm64")
+            karch="arm64"
+            cross_compile="aarch64-linux-gnu-"
+            ubuntu_config="arm64-config.flavour.generic"
+            ;;
+
+        "powerpc")
+            karch="powerpc"
+            cross_compile="powerpc-linux-gnu-"
+            ubuntu_config="powerpc-config.flavour.powerpc-smp"
+            ;;
+
+        "ppc64el")
+            karch="powerpc"
+            cross_compile="powerpc64le-linux-gnu-"
+            ubuntu_config="ppc64el-config.flavour.generic"
+            ;;
+
+        *)
+            echo "Unsupported cross arch $arch"
+            exit 1
+            ;;
+    esac
+
+    # Export variables used by Kbuild for cross compilation
+    export ARCH="${karch}"
+    export CROSS_COMPILE="${cross_compile}"
+
+
+# Set arch specific values if we are not cross compiling
+elif [ "x${arch:-}" != "x" ]; then
+    case "$arch" in
+        "x86-32")
+            karch="x86"
+            ubuntu_config="i386-config.flavour.generic"
+            ;;
+
+        "x86-64")
+            karch="x86"
+            ubuntu_config="amd64-config.flavour.generic"
+            ;;
+
+        "armhf")
+            karch="arm"
+            ubuntu_config="armhf-config.flavour.generic"
+            ;;
+
+        "arm64")
+            karch="arm64"
+            ubuntu_config="arm64-config.flavour.generic"
+            ;;
+
+        "powerpc")
+            karch="powerpc"
+            ubuntu_config="powerpc-config.flavour.powerpc-smp"
+            ;;
+
+        "ppc64el")
+            karch="powerpc"
+            ubuntu_config="ppc64el-config.flavour.generic"
+            ;;
+
+        *)
+            echo "Unsupported arch $arch"
+            exit 1
+            ;;
+    esac
+else
+    echo "Not arch or cross_arch specified"
+    exit 1
+fi
+
+
+
+
+# Create build directories
+mkdir -p "${LNXBUILDDIR}" "${LNXHDRDIR}" "${LTTBUILDKSRCDIR}" "${LTTBUILDKHDRDIR}"
+
+
+
+## PREPARE DISTRO STYLE KERNEL HEADERS / DEVEL
+
+# Enter linux source dir
+cd "${LNXSRCDIR}"
+
+prepare_lnx_sources "."
+
+# For RT kernels, copy version file
+if [ -s localversion-rt ]; then
+    cp -a localversion-rt "${LNXHDRDIR}"
+fi
+
+# Copy all Makefile related stuff
+find . -path './include/*' -prune \
+    -o -path './scripts/*' -prune -o -type f \
+	\( -name 'Makefile*' -o -name 'Kconfig*' -o -name 'Kbuild*' -o \
+        -name '*.sh' -o -name '*.pl' -o -name '*.lds' \) \
+    -print | cpio -pd --preserve-modification-time "${LNXHDRDIR}"
+
+# Copy base scripts and include dirs
+cp -a scripts include "${LNXHDRDIR}"
+
+# Copy arch includes
+(find arch -name include -type d -print | \
+    xargs -n1 -i: find : -type f) | \
+	cpio -pd --preserve-modification-time "${LNXHDRDIR}"
+
+# Copy arch scripts
+(find arch -name scripts -type d -print | \
+    xargs -n1 -i: find : -type f) | \
+	cpio -pd --preserve-modification-time "${LNXHDRDIR}"
+
+# Cleanup scripts
+rm -f "${LNXHDRDIR}/scripts/*.o"
+rm -f "${LNXHDRDIR}/scripts/*/*.o"
+
+# On powerpc this object is required to link modules
+if [ "${karch}" = "powerpc" ]; then
+    cp -a --parents arch/powerpc/lib/crtsavres.[So] "${LNXHDRDIR}/"
+fi
+
+# Copy modules related stuff, if available
+if [ -s Module.symvers ]; then
+    cp Module.symvers "${LNXHDRDIR}"
+fi
+
+if [ -s System.map ]; then
+    cp System.map "${LNXHDRDIR}"
+fi
+
+if [ -s Module.markers ]; then
+    cp Module.markers "${LNXHDRDIR}"
+fi
+
+# Copy config file
+cp .config "${LNXHDRDIR}"
+
+# Make sure the Makefile and version.h have a matching timestamp so that
+# external modules can be built
+if [ -s "${LNXHDRDIR}/include/generated/uapi/linux/version.h" ]; then
+    touch -r "${LNXHDRDIR}/Makefile" "${LNXHDRDIR}/include/generated/uapi/linux/version.h"
+elif [ -s "${LNXHDRDIR}/include/linux/version.h" ]; then
+    touch -r "${LNXHDRDIR}/Makefile" "${LNXHDRDIR}/include/linux/version.h"
+else
+    echo "Missing version.h"
+    exit 1
+fi
+touch -r "${LNXHDRDIR}/.config" "${LNXHDRDIR}/include/generated/autoconf.h"
+
+# Copy .config to include/config/auto.conf so "make prepare" is unnecessary.
+cp "${LNXHDRDIR}/.config" "${LNXHDRDIR}/include/config/auto.conf"
+
+
+
+
+## PREPARE FULL LINUX SOURCE TREE
+
+# Enter linux source dir
+cd "${LNXSRCDIR}"
+
+# Make sure linux source dir is clean
+git clean -xdf
+
+prepare_lnx_sources "${LNXBUILDDIR}"
+
+
+## BUILD modules
+
+# Build modules against full kernel sources
+build_modules "${LNXBUILDDIR}" "${LTTBUILDKSRCDIR}"
+
+# Build modules against kernel headers
+build_modules "${LNXHDRDIR}" "${LTTBUILDKHDRDIR}"
+
+# Make sure modules were built
+tree "${LTTBUILDKSRCDIR}"
+if [ "x$(find "${LTTBUILDKSRCDIR}" -name '*.ko*' -printf yes -quit)" != "xyes" ]; then
+  echo "No modules built!"
+  exit 1
+fi
+
+tree "${LTTBUILDKHDRDIR}"
+if [ "x$(find "${LTTBUILDKHDRDIR}" -name '*.ko*' -printf yes -quit)" != "xyes" ]; then
+  echo "No modules built!"
+  exit 1
+fi
+
+# EOF
-- 
2.34.1