From 6f1411b1fcc0adf846d4b08f676b45f882b07238 Mon Sep 17 00:00:00 2001 From: Michael Jeanson Date: Thu, 9 Nov 2017 11:17:34 -0500 Subject: [PATCH] jjb: update latency-tracker jobs Signed-off-by: Michael Jeanson --- jobs/latency-tracker.yaml | 40 ++- scripts/latency-tracker/master-rt.groovy | 169 ++++++++--- scripts/latency-tracker/master-ubuntu.groovy | 190 ------------ .../{master-vanilla.groovy => master.groovy} | 279 ++++++++++++++++-- scripts/latency-tracker/param-build.sh | 130 +++++--- 5 files changed, 502 insertions(+), 306 deletions(-) delete mode 100644 scripts/latency-tracker/master-ubuntu.groovy rename scripts/latency-tracker/{master-vanilla.groovy => master.groovy} (55%) diff --git a/jobs/latency-tracker.yaml b/jobs/latency-tracker.yaml index 7f52af0..d00c134 100644 --- a/jobs/latency-tracker.yaml +++ b/jobs/latency-tracker.yaml @@ -94,6 +94,10 @@ name: 'kverfloor' default: 'v3.12' description: 'The lowest kernel version to build.' + - string: + name: 'kverceil' + default: '' + description: 'The highest kernel version to build. (excluded)' - string: name: 'kverfilter' default: 'stable-head' @@ -110,7 +114,7 @@ builders: - system-groovy: command: - !include-raw-escape: scripts/latency-tracker/master-vanilla.groovy + !include-raw-escape: scripts/latency-tracker/master.groovy publishers: - workspace-cleanup @@ -141,9 +145,17 @@ default: '20' description: 'The maximum number of concurrent child build to run.' - string: - name: 'uversion' - default: '{uversion}' + name: 'kverfloor' + default: '' description: 'The lowest kernel version to build.' + - string: + name: 'kverceil' + default: '' + description: 'The highest kernel version to build. (excluded)' + - string: + name: 'kverfilter' + default: 'stable-head' + description: 'Kernel versions list filtering strategy.' - string: name: 'kgitrepo' default: 'git://git-mirror.internal.efficios.com/git/ubuntu-{uversion}.git' @@ -152,11 +164,15 @@ name: 'kbuildjob' default: 'latency-tracker_VERSION_param-build' description: 'The parametrized job to use for child builds.' + - string: + name: 'uversion' + default: '{uversion}' + description: 'The lowest kernel version to build.' builders: - system-groovy: command: - !include-raw-escape: scripts/latency-tracker/master-ubuntu.groovy + !include-raw-escape: scripts/latency-tracker/master.groovy publishers: - workspace-cleanup @@ -190,6 +206,14 @@ name: 'kverfloor' default: 'v2.6.36-rt0-rebase' description: 'The lowest kernel version to build.' + - string: + name: 'kverceil' + default: '' + description: 'The highest kernel version to build. (excluded)' + - string: + name: 'kverfilter' + default: 'stable-head' + description: 'Kernel versions list filtering strategy.' - string: name: 'kgitrepo' default: 'git://git-mirror.internal.efficios.com/kernel/rt/linux-rt-devel.git' @@ -236,7 +260,7 @@ default: 'master' description: 'The latency-tracker branch to build.' - string: - name: 'kversion' + name: 'ktag' default: '' description: 'The linux kernel git tag to build against.' - string: @@ -260,7 +284,7 @@ builders: - shell: | - git clone --depth=1 -b "$kversion" --reference $HOME/gitcache/linux-stable.git/ "$kgitrepo" src/linux + git clone --depth=1 -b "$ktag" --reference $HOME/gitcache/linux-stable.git/ "$kgitrepo" src/linux - shell: !include-raw-escape: scripts/latency-tracker/param-build.sh @@ -294,7 +318,7 @@ default: 'master' description: 'The latency-tracker branch to build.' - string: - name: 'kversion' + name: 'ktag' default: '' description: 'The linux kernel git tag to build against.' - string: @@ -318,7 +342,7 @@ builders: - shell: | - git clone --depth=1 -b "$kversion" --reference $HOME/gitcache/linux-stable.git/ "$kgitrepo" src/linux + git clone --depth=1 -b "$ktag" --reference $HOME/gitcache/linux-stable.git/ "$kgitrepo" src/linux - shell: !include-raw-escape: scripts/latency-tracker/param-build.sh diff --git a/scripts/latency-tracker/master-rt.groovy b/scripts/latency-tracker/master-rt.groovy index 177041d..9d94be2 100644 --- a/scripts/latency-tracker/master-rt.groovy +++ b/scripts/latency-tracker/master-rt.groovy @@ -1,5 +1,5 @@ /** - * Copyright (C) 2016 - Michael Jeanson + * Copyright (C) 2016-2017 - Michael Jeanson * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -23,20 +23,44 @@ import org.eclipse.jgit.api.Git import org.eclipse.jgit.lib.Ref -class kVersion implements Comparable { +class InvalidKVersionException extends Exception { + public InvalidKVersionException(String message) { + super(message) + } +} + +class EmptyKVersionException extends Exception { + public EmptyKVersionException(String message) { + super(message) + } +} - Integer major = 0; - Integer majorB = 0; - Integer minor = 0; - Integer patch = 0; - Integer rt = 0; +class RTKVersion implements Comparable { - kVersion() {} + Integer major = 0 + Integer majorB = 0 + Integer minor = 0 + Integer patch = 0 + Integer rt = 0 - kVersion(version) { + RTKVersion() {} + + RTKVersion(version) { this.parse(version) } + static RTKVersion minKVersion() { + return new RTKVersion("v0.0.0-rt0-rebase") + } + + static RTKVersion maxKVersion() { + return new RTKVersion("v" + Integer.MAX_VALUE + ".0.0-rt0-rebase") + } + + static RTKVersion factory(version) { + return new RTKVersion(version) + } + def parse(version) { this.major = 0 this.majorB = 0 @@ -44,9 +68,13 @@ class kVersion implements Comparable { this.patch = 0 this.rt = 0 + if (!version) { + throw new EmptyKVersionException("Empty kernel version") + } + def match = version =~ /^v(\d+)\.(\d+)(\.(\d+))?(\.(\d+))?(-rt(\d+)-rebase)$/ if (!match) { - throw new Exception("Invalid kernel version: ${version}") + throw new InvalidKVersionException("Invalid kernel version: ${version}") } Integer offset = 0; @@ -72,21 +100,36 @@ class kVersion implements Comparable { this.rt = Integer.parseInt(match.group(8)) } - @Override int compareTo(kVersion o) { + // Return true if both version are of the same stable branch + Boolean isSameStable(RTKVersion o) { if (this.major != o.major) { - return Integer.compare(this.major, o.major); + return false } if (this.majorB != o.majorB) { - return Integer.compare(this.majorB, o.majorB); + return false } if (this.minor != o.minor) { - return Integer.compare(this.minor, o.minor); + return false + } + + return true + } + + @Override int compareTo(RTKVersion o) { + if (this.major != o.major) { + return Integer.compare(this.major, o.major) + } + if (this.majorB != o.majorB) { + return Integer.compare(this.majorB, o.majorB) + } + if (this.minor != o.minor) { + return Integer.compare(this.minor, o.minor) } if (this.patch != o.patch) { - return Integer.compare(this.patch, o.patch); + return Integer.compare(this.patch, o.patch) } - if (this.rt != o.rc) { - return Integer.compare(this.rt, o.rt); + if (this.rt != o.rt) { + return Integer.compare(this.rt, o.rt) } // Same version @@ -118,39 +161,98 @@ class kVersion implements Comparable { def mversion = build.buildVariableResolver.resolve('mversion') def maxConcurrentBuild = build.buildVariableResolver.resolve('maxConcurrentBuild') def kgitrepo = build.buildVariableResolver.resolve('kgitrepo') -def kverfloor = new kVersion(build.buildVariableResolver.resolve('kverfloor')) +def kverfloor_raw = build.buildVariableResolver.resolve('kverfloor') +def kverceil_raw = build.buildVariableResolver.resolve('kverceil') +def kverfilter = build.buildVariableResolver.resolve('kverfilter') def job = Hudson.instance.getJob(build.buildVariableResolver.resolve('kbuildjob')) def currentJobName = build.project.getFullDisplayName() + // Get the out variable def config = new HashMap() def bindings = getBinding() config.putAll(bindings.getVariables()) def out = config['out'] -def jlc = new jenkins.model.JenkinsLocationConfiguration() -def jenkinsUrl = jlc.url // Get tags from git repository def refs = Git.lsRemoteRepository().setTags(true).setRemote(kgitrepo).call(); // Get kernel versions to build def kversions = [] -for (ref in refs) { - def match = ref.getName() =~ /^refs\/tags\/(v[\d\.]+(-rt(\d+)-rebase))$/ +def matchStrs = [ + ~/^refs\/tags\/(v[\d\.]+(-rt(\d+)-rebase))$/, +] +def blacklist = [ + 'v4.11.8-rt5-rebase', + 'v4.11.9-rt6-rebase', + 'v4.11.9-rt7-rebase', + 'v4.11.12-rt8-rebase', + 'v4.11.12-rt9-rebase', + 'v4.11.12-rt10-rebase', + 'v4.11.12-rt11-rebase', + 'v4.11.12-rt12-rebase', + 'v4.11.12-rt13-rebase', +] + +def kversionFactory = new RTKVersion() + +// Parse kernel versions +def kverfloor = "" +try { + kverfloor = kversionFactory.factory(kverfloor_raw) +} catch (EmptyKVersionException e) { + kverfloor = kversionFactory.minKVersion() +} - if (match) { - def v = new kVersion(match.group(1)) +def kverceil = "" +try { + kverceil = kversionFactory.factory(kverceil_raw) +} catch (EmptyKVersionException e) { + kverceil = kversionFactory.maxKVersion() +} - if (v >= kverfloor) { - kversions.add(v) +// Build a sorted list of versions to build +for (ref in refs) { + for (matchStr in matchStrs) { + def match = ref.getName() =~ matchStr + if (match && !blacklist.contains(match.group(1))) { + def v = kversionFactory.factory(match.group(1)) + + if ((v >= kverfloor) && (v < kverceil)) { + kversions.add(v) + } } } } kversions.sort() -// Debug +switch (kverfilter) { + case 'stable-head': + // Keep only the head of each stable branch + println('Filter kernel versions to keep only the latest point release of each stable branch.') + + for (i = 0; i < kversions.size(); i++) { + def curr = kversions[i] + def next = i < kversions.size() - 1 ? kversions[i + 1] : null + + if (next != null) { + if (curr.isSameStable(next)) { + kversions.remove(i) + i-- + } + } + } + break + + default: + // No filtering of kernel versions + println('No kernel versions filtering selected.') + break +} + + println "Building the following kernel versions:" for (k in kversions) { println k @@ -165,6 +267,7 @@ def allBuilds = [] def ongoingBuild = [] def failedRuns = [] def isFailed = false +def similarJobQueued = 0; // Loop while we have kernel versions remaining or jobs running while ( kversions.size() != 0 || ongoingBuild.size() != 0 ) { @@ -173,7 +276,7 @@ while ( kversions.size() != 0 || ongoingBuild.size() != 0 ) { def kversion = kversions.pop() def job_params = [ new StringParameterValue('mversion', mversion), - new StringParameterValue('kversion', kversion.toString()), + new StringParameterValue('ktag', kversion.toString()), new StringParameterValue('kgitrepo', kgitrepo), ] @@ -188,7 +291,7 @@ while ( kversions.size() != 0 || ongoingBuild.size() != 0 ) { println "Waiting... Queued: " + kversions.size() + " Running: " + ongoingBuild.size() try { - Thread.sleep(5000) + Thread.sleep(10000) } catch(e) { if (e in InterruptedException) { build.setResult(hudson.model.Result.ABORTED) @@ -198,11 +301,9 @@ while ( kversions.size() != 0 || ongoingBuild.size() != 0 ) { } } - // Check for queued similar job since we only want to run latest - // as Mathieu Desnoyers requirement + // Abort job if a newer instance is queued similarJobQueued = Hudson.instance.queue.items.count{it.task.getFullDisplayName() == currentJobName} if ( similarJobQueued > 0 ) { - // Abort since new build is queued build.setResult(hudson.model.Result.ABORTED) throw new InterruptedException() } @@ -217,7 +318,7 @@ while ( kversions.size() != 0 || ongoingBuild.size() != 0 ) { // Print results def matrixParent = currentBuild.get() allBuilds.add(matrixParent) - def kernelStr = matrixParent.buildVariableResolver.resolve("kversion") + def kernelStr = matrixParent.buildVariableResolver.resolve("ktag") println "${matrixParent.fullDisplayName} (${kernelStr}) completed with status ${matrixParent.result}" // Process child runs of matrixBuild @@ -243,7 +344,7 @@ for (failedRun in failedRuns) { println "---Build report---" for (b in allBuilds) { - def kernelStr = b.buildVariableResolver.resolve("kversion") + def kernelStr = b.buildVariableResolver.resolve("ktag") println "${b.fullDisplayName} (${kernelStr}) completed with status ${b.result}" // Cleanup builds try { diff --git a/scripts/latency-tracker/master-ubuntu.groovy b/scripts/latency-tracker/master-ubuntu.groovy deleted file mode 100644 index da5f1fa..0000000 --- a/scripts/latency-tracker/master-ubuntu.groovy +++ /dev/null @@ -1,190 +0,0 @@ -/** - * Copyright (C) 2016 - Michael Jeanson - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -import hudson.model.* -import hudson.AbortException -import hudson.console.HyperlinkNote -import java.util.concurrent.CancellationException -import org.eclipse.jgit.api.Git -import org.eclipse.jgit.lib.Ref - - -// Retrieve parameters of the current build -def mversion = build.buildVariableResolver.resolve('mversion') -def maxConcurrentBuild = build.buildVariableResolver.resolve('maxConcurrentBuild') -def kgitrepo = build.buildVariableResolver.resolve('kgitrepo') -def uversion = build.buildVariableResolver.resolve('uversion') -def job = Hudson.instance.getJob(build.buildVariableResolver.resolve('kbuildjob')) -def currentJobName = build.project.getFullDisplayName() - -// Get the out variable -def config = new HashMap() -def bindings = getBinding() -config.putAll(bindings.getVariables()) -def out = config['out'] - -def jlc = new jenkins.model.JenkinsLocationConfiguration() -def jenkinsUrl = jlc.url - -// Get tags from git repository -def refs = Git.lsRemoteRepository().setTags(true).setRemote(kgitrepo).call(); - -// Get kernel versions to build -def kversions = [] - -def matchStrs = [] - -switch (uversion) { - case 'xenial': - matchStrs = [ - ~/^refs\/tags\/(Ubuntu-4\.4\.0-\d{1,3}\.[\d\.]+)$/, - ~/^refs\/tags\/(Ubuntu-lts-.*_16\.04\.\d+)$/, - ] - break - - case 'trusty': - matchStrs = [ - ~/^refs\/tags\/(Ubuntu-3\.13\.0-[\d\.]+)$/, - ~/^refs\/tags\/(Ubuntu-lts-.*_14\.04\.\d+)$/, - ] - break - - default: - println 'Unsupported Ubuntu version: ${uversion}' - throw new InterruptedException() - break -} - -for (ref in refs) { - for (matchStr in matchStrs) { - def match = ref.getName() =~ matchStr - - if (match) { - kversions.add(match.group(1)) - } - } -} - -kversions.sort() - -// Debug -println "Building the following kernel versions:" -for (k in kversions) { - println k -} - -// Debug: Stop build here -//throw new InterruptedException() - -def joburl = HyperlinkNote.encodeTo('/' + job.url, job.fullDisplayName) - -def allBuilds = [] -def ongoingBuild = [] -def failedRuns = [] -def isFailed = false - -// Loop while we have kernel versions remaining or jobs running -while ( kversions.size() != 0 || ongoingBuild.size() != 0 ) { - - if(ongoingBuild.size() < maxConcurrentBuild.toInteger() && kversions.size() != 0) { - def kversion = kversions.pop() - def job_params = [ - new StringParameterValue('mversion', mversion), - new StringParameterValue('kversion', kversion), - new StringParameterValue('kgitrepo', kgitrepo), - ] - - // Launch the parametrized build - def param_build = job.scheduleBuild2(0, new Cause.UpstreamCause(build), new ParametersAction(job_params)) - println "triggering ${joburl} for the ${mversion} branch on kernel ${kversion}" - - // Add it to the ongoing build queue - ongoingBuild.push(param_build) - - } else { - - println "Waiting... Queued: " + kversions.size() + " Running: " + ongoingBuild.size() - try { - Thread.sleep(5000) - } catch(e) { - if (e in InterruptedException) { - build.setResult(hudson.model.Result.ABORTED) - throw new InterruptedException() - } else { - throw(e) - } - } - - // Check for queued similar job since we only want to run latest - // as Mathieu Desnoyers requirement - similarJobQueued = Hudson.instance.queue.items.count{it.task.getFullDisplayName() == currentJobName} - if ( similarJobQueued > 0 ) { - // Abort since new build is queued - build.setResult(hudson.model.Result.ABORTED) - throw new InterruptedException() - } - - def i = ongoingBuild.iterator() - while ( i.hasNext() ) { - currentBuild = i.next() - if ( currentBuild.isCancelled() || currentBuild.isDone() ) { - // Remove from queue - i.remove() - - // Print results - def matrixParent = currentBuild.get() - allBuilds.add(matrixParent) - def kernelStr = matrixParent.buildVariableResolver.resolve("kversion") - println "${matrixParent.fullDisplayName} (${kernelStr}) completed with status ${matrixParent.result}" - - // Process child runs of matrixBuild - def childRuns = matrixParent.getRuns() - for ( childRun in childRuns ) { - println "\t${childRun.fullDisplayName} (${kernelStr}) completed with status ${childRun.result}" - if (childRun.result != Result.SUCCESS) { - failedRuns.add(childRun) - isFailed = true - } - } - } - } - } -} - -// Get log of failed runs -for (failedRun in failedRuns) { - println "---START---" - failedRun.writeWholeLogTo(out) - println "---END---" -} - -println "---Build report---" -for (b in allBuilds) { - def kernelStr = b.buildVariableResolver.resolve("kversion") - println "${b.fullDisplayName} (${kernelStr}) completed with status ${b.result}" - // Cleanup builds - try { - b.delete() - } catch (all) {} -} - -// Mark this build failed if any child build has failed -if (isFailed) { - build.getExecutor().interrupt(Result.FAILURE) -} - -// EOF diff --git a/scripts/latency-tracker/master-vanilla.groovy b/scripts/latency-tracker/master.groovy similarity index 55% rename from scripts/latency-tracker/master-vanilla.groovy rename to scripts/latency-tracker/master.groovy index 67da3be..a8bf945 100644 --- a/scripts/latency-tracker/master-vanilla.groovy +++ b/scripts/latency-tracker/master.groovy @@ -23,20 +23,44 @@ import org.eclipse.jgit.api.Git import org.eclipse.jgit.lib.Ref -class kVersion implements Comparable { +class InvalidKVersionException extends Exception { + public InvalidKVersionException(String message) { + super(message) + } +} - Integer major = 0; - Integer majorB = 0; - Integer minor = 0; - Integer patch = 0; - Integer rc = Integer.MAX_VALUE; +class EmptyKVersionException extends Exception { + public EmptyKVersionException(String message) { + super(message) + } +} - kVersion() {} +class VanillaKVersion implements Comparable { - kVersion(version) { + Integer major = 0 + Integer majorB = 0 + Integer minor = 0 + Integer patch = 0 + Integer rc = Integer.MAX_VALUE + + VanillaKVersion() {} + + VanillaKVersion(version) { this.parse(version) } + static VanillaKVersion minKVersion() { + return new VanillaKVersion("v0.0.0") + } + + static VanillaKVersion maxKVersion() { + return new VanillaKVersion("v" + Integer.MAX_VALUE + ".0.0") + } + + static VanillaKVersion factory(version) { + return new VanillaKVersion(version) + } + def parse(version) { this.major = 0 this.majorB = 0 @@ -44,9 +68,13 @@ class kVersion implements Comparable { this.patch = 0 this.rc = Integer.MAX_VALUE + if (!version) { + throw new EmptyKVersionException("Empty kernel version") + } + def match = version =~ /^v(\d+)\.(\d+)(\.(\d+))?(\.(\d+))?(-rc(\d+))?$/ if (!match) { - throw new Exception("Invalid kernel version: ${version}") + throw new InvalidKVersionException("Invalid kernel version: ${version}") } Integer offset = 0; @@ -80,7 +108,7 @@ class kVersion implements Comparable { } // Return true if both version are of the same stable branch - Boolean isSameStable(kVersion o) { + Boolean isSameStable(VanillaKVersion o) { if (this.major != o.major) { return false } @@ -94,7 +122,7 @@ class kVersion implements Comparable { return true } - @Override int compareTo(kVersion o) { + @Override int compareTo(VanillaKVersion o) { if (this.major != o.major) { return Integer.compare(this.major, o.major) } @@ -135,24 +163,159 @@ class kVersion implements Comparable { } } +class UbuntuKVersion implements Comparable { + + Integer major = 0 + Integer minor = 0 + Integer patch = 0 + Integer umajor = 0 + Integer uminor = 0 + String suffix = "" + Boolean isLTS = false + + UbuntuKVersion() {} + + UbuntuKVersion(version) { + this.parse(version) + } + + static UbuntuKVersion minKVersion() { + return new UbuntuKVersion("Ubuntu-lts-0.0.0-0.0") + } + + static UbuntuKVersion maxKVersion() { + return new UbuntuKVersion("Ubuntu-" + Integer.MAX_VALUE + ".0.0-0.0") + } + + static UbuntuKVersion factory(version) { + return new UbuntuKVersion(version) + } + + def parse(version) { + this.major = 0 + this.minor = 0 + this.patch = 0 + this.umajor = 0 + this.uminor = 0 + this.suffix = ""; + this.isLTS = false + + if (!version) { + throw new EmptyKVersionException("Empty kernel version") + } + + //'Ubuntu-lts-4.8.0-27.29_16.04.1', + //'Ubuntu-4.4.0-70.91', + def match = version =~ /^Ubuntu-(lts-)??(\d+)\.(\d+)\.(\d+)-(\d+)\.(\d+)(.*)??$/ + if (!match) { + throw new InvalidKVersionException("Invalid kernel version: ${version}") + } + + this.isLTS = match.group(1) != null + + // Major + this.major = Integer.parseInt(match.group(2)) + + // Minor + this.minor = Integer.parseInt(match.group(3)) + + // Patch level + this.patch = Integer.parseInt(match.group(4)) + + // Ubuntu major + this.umajor = Integer.parseInt(match.group(5)) + + // Ubuntu minor + this.uminor = Integer.parseInt(match.group(6)) + + if (match.group(7) != null) { + this.suffix = match.group(7) + } + } + + // Return true if this version is a release candidate + Boolean isRC() { + return false + } + + // Return true if both version are of the same stable branch + Boolean isSameStable(UbuntuKVersion o) { + if (this.isLTS != o.isLTS) { + return false + } + if (this.major != o.major) { + return false + } + if (this.minor != o.minor) { + return false + } + if (this.patch != o.patch) { + return false + } + + return true + } + + @Override int compareTo(UbuntuKVersion o) { + if (this.major != o.major) { + return Integer.compare(this.major, o.major) + } + if (this.minor != o.minor) { + return Integer.compare(this.minor, o.minor) + } + if (this.patch != o.patch) { + return Integer.compare(this.patch, o.patch) + } + if (this.umajor != o.umajor) { + return Integer.compare(this.umajor, o.umajor) + } + if (this.uminor != o.uminor) { + return Integer.compare(this.uminor, o.uminor) + } + if (this.isLTS != o.isLTS) { + if (o.isLTS) { + return 1 + } else { + return -1 + } + } + + // Same version + return 0; + } + + String toString() { + String vString = "Ubuntu-" + + if (this.isLTS) { + vString = vString.concat("lts-") + } + + vString = vString.concat("${this.major}.${this.minor}.${this.patch}-${this.umajor}.${this.uminor}${this.suffix}") + + return vString + } +} + // Retrieve parameters of the current build def mversion = build.buildVariableResolver.resolve('mversion') def maxConcurrentBuild = build.buildVariableResolver.resolve('maxConcurrentBuild') def kgitrepo = build.buildVariableResolver.resolve('kgitrepo') -def kverfloor = new kVersion(build.buildVariableResolver.resolve('kverfloor')) +def kverfloor_raw = build.buildVariableResolver.resolve('kverfloor') +def kverceil_raw = build.buildVariableResolver.resolve('kverceil') def kverfilter = build.buildVariableResolver.resolve('kverfilter') +def uversion = build.buildVariableResolver.resolve('uversion') def job = Hudson.instance.getJob(build.buildVariableResolver.resolve('kbuildjob')) def currentJobName = build.project.getFullDisplayName() + // Get the out variable def config = new HashMap() def bindings = getBinding() config.putAll(bindings.getVariables()) def out = config['out'] -def jlc = new jenkins.model.JenkinsLocationConfiguration() -def jenkinsUrl = jlc.url // Get tags from git repository def refs = Git.lsRemoteRepository().setTags(true).setRemote(kgitrepo).call(); @@ -160,17 +323,73 @@ def refs = Git.lsRemoteRepository().setTags(true).setRemote(kgitrepo).call(); // Get kernel versions to build def kversions = [] def kversionsRC = [] -for (ref in refs) { - def match = ref.getName() =~ /^refs\/tags\/(v[\d\.]+(-rc(\d+))?)$/ +def matchStrs = [] +def blacklist = [] +def kversionFactory = "" + +if (uversion != null) { + kversionFactory = new UbuntuKVersion() + switch (uversion) { + case 'xenial': + matchStrs = [ + ~/^refs\/tags\/(Ubuntu-4\.4\.0-\d{1,3}?\.[\d]+)$/, + ~/^refs\/tags\/(Ubuntu-lts-4\.8\.0-.*_16\.04\.\d+)$/, + ~/^refs\/tags\/(Ubuntu-lts-4\.10\.0-.*_16\.04\.\d+)$/, + ] + + blacklist = [ + 'Ubuntu-lts-4.10.0-7.9_16.04.1', + ] + break + + case 'trusty': + matchStrs = [ + ~/^refs\/tags\/(Ubuntu-3\.13\.0-[\d\.]+)$/, + ~/^refs\/tags\/(Ubuntu-lts-.*_14\.04\.\d+)$/, + ] + break + + default: + println 'Unsupported Ubuntu version: ${uversion}' + throw new InterruptedException() + break + } +} else { + // Vanilla + kversionFactory = new VanillaKVersion() + matchStrs = [ + ~/^refs\/tags\/(v[\d\.]+(-rc(\d+))?)$/, + ] +} - if (match) { - def v = new kVersion(match.group(1)) +// Parse kernel versions +def kverfloor = "" +try { + kverfloor = kversionFactory.factory(kverfloor_raw) +} catch (EmptyKVersionException e) { + kverfloor = kversionFactory.minKVersion() +} - if (v >= kverfloor) { - if (v.isRC()) { - kversionsRC.add(v) - } else { - kversions.add(v) +def kverceil = "" +try { + kverceil = kversionFactory.factory(kverceil_raw) +} catch (EmptyKVersionException e) { + kverceil = kversionFactory.maxKVersion() +} + +// Build a sorted list of versions to build +for (ref in refs) { + for (matchStr in matchStrs) { + def match = ref.getName() =~ matchStr + if (match && !blacklist.contains(match.group(1))) { + def v = kversionFactory.factory(match.group(1)) + + if ((v >= kverfloor) && (v < kverceil)) { + if (v.isRC()) { + kversionsRC.add(v) + } else { + kversions.add(v) + } } } } @@ -204,11 +423,10 @@ switch (kverfilter) { } // If the last RC version is newer than the last stable, add it to the build list -if (kversionsRC.last() > kversions.last()) { +if (kversionsRC.size() > 0 && kversionsRC.last() > kversions.last()) { kversions.add(kversionsRC.last()) } -// Debug println "Building the following kernel versions:" for (k in kversions) { println k @@ -232,7 +450,7 @@ while ( kversions.size() != 0 || ongoingBuild.size() != 0 ) { def kversion = kversions.pop() def job_params = [ new StringParameterValue('mversion', mversion), - new StringParameterValue('kversion', kversion.toString()), + new StringParameterValue('ktag', kversion.toString()), new StringParameterValue('kgitrepo', kgitrepo), ] @@ -247,7 +465,7 @@ while ( kversions.size() != 0 || ongoingBuild.size() != 0 ) { println "Waiting... Queued: " + kversions.size() + " Running: " + ongoingBuild.size() try { - Thread.sleep(5000) + Thread.sleep(10000) } catch(e) { if (e in InterruptedException) { build.setResult(hudson.model.Result.ABORTED) @@ -257,10 +475,9 @@ while ( kversions.size() != 0 || ongoingBuild.size() != 0 ) { } } - // If a newer instance of this job is queued, abort to let it run + // Abort job if a newer instance is queued similarJobQueued = Hudson.instance.queue.items.count{it.task.getFullDisplayName() == currentJobName} if ( similarJobQueued > 0 ) { - // Abort since new build is queued build.setResult(hudson.model.Result.ABORTED) throw new InterruptedException() } @@ -275,7 +492,7 @@ while ( kversions.size() != 0 || ongoingBuild.size() != 0 ) { // Print results def matrixParent = currentBuild.get() allBuilds.add(matrixParent) - def kernelStr = matrixParent.buildVariableResolver.resolve("kversion") + def kernelStr = matrixParent.buildVariableResolver.resolve("ktag") println "${matrixParent.fullDisplayName} (${kernelStr}) completed with status ${matrixParent.result}" // Process child runs of matrixBuild @@ -301,7 +518,7 @@ for (failedRun in failedRuns) { println "---Build report---" for (b in allBuilds) { - def kernelStr = b.buildVariableResolver.resolve("kversion") + def kernelStr = b.buildVariableResolver.resolve("ktag") println "${b.fullDisplayName} (${kernelStr}) completed with status ${b.result}" // Cleanup builds try { diff --git a/scripts/latency-tracker/param-build.sh b/scripts/latency-tracker/param-build.sh index 3a77780..22e198b 100644 --- a/scripts/latency-tracker/param-build.sh +++ b/scripts/latency-tracker/param-build.sh @@ -15,23 +15,29 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +# Parameters +arch=${arch:-} +cross_arch=${cross_arch:-} +ktag=${ktag:-} + + ## FUNCTIONS ## # Kernel version compare functions verlte() { - [ "$1" = "`printf '%s\n%s' $1 $2 | sort -V | head -n1`" ] + [ "$1" = "$(printf '%s\n%s' "$1" "$2" | sort -V | head -n1)" ] } verlt() { - [ "$1" = "$2" ] && return 1 || verlte $1 $2 + [ "$1" = "$2" ] && return 1 || verlte "$1" "$2" } vergte() { - [ "$1" = "`printf '%s\n%s' $1 $2 | sort -V | tail -n1`" ] + [ "$1" = "$(printf '%s\n%s' "$1" "$2" | sort -V | tail -n1)" ] } vergt() { - [ "$1" = "$2" ] && return 1 || vergte $1 $2 + [ "$1" = "$2" ] && return 1 || vergte "$1" "$2" } @@ -46,15 +52,21 @@ prepare_lnx_sources() { fi # Generate kernel configuration - case "$kversion" in + case "$ktag" in Ubuntu*) + if [ "${cross_arch}" = "powerpc" ]; then + if vergte "$KVERSION" "4.10"; then + echo "Ubuntu removed big endian powerpc configuration from kernel >= 4.10. Don't try to build it." + exit 0 + fi + fi fakeroot debian/rules clean fakeroot debian/rules genconfigs - cp CONFIGS/${ubuntu_config} "${outdir}"/.config + cp CONFIGS/"${ubuntu_config}" "${outdir}"/.config ;; *) # Que sera sera - make ${koutput} allyesconfig CC=$CC + make "${vanilla_config}" CC="$CC" ${koutput} ;; esac @@ -67,30 +79,42 @@ prepare_lnx_sources() { # Disable kernel stack frame correctness validation, introduced in 4.6.0 and currently fails sed -i "s/CONFIG_STACK_VALIDATION=y/# CONFIG_STACK_VALIDATION is not set/g" "${outdir}"/.config - # Enable CONFIG_KALLSYMS_ALL - echo "CONFIG_KPROBES=y" >> "${outdir}"/.config - echo "CONFIG_FTRACE=y" >> "${outdir}"/.config - echo "CONFIG_BLK_DEV_IO_TRACE=y" >> "${outdir}"/.config - echo "CONFIG_TRACEPOINTS=y" >> "${outdir}"/.config - echo "CONFIG_KALLSYMS_ALL=y" >> "${outdir}"/.config + # Set required options + { + echo "CONFIG_KPROBES=y"; + echo "CONFIG_FTRACE=y"; + echo "CONFIG_BLK_DEV_IO_TRACE=y"; + echo "CONFIG_TRACEPOINTS=y"; + echo "CONFIG_KALLSYMS_ALL=y"; + } >> "${outdir}"/.config + + + make "$oldconf_target" CC="$CC" ${koutput} + make modules_prepare CC="$CC" ${koutput} + + # Debug + #cat "${outdir}"/.config + # On powerpc this object is required to link modules + if [ "${karch}" = "powerpc" ]; then + make arch/powerpc/lib/crtsavres.o CC="$CC" ${koutput} + fi - make ${koutput} olddefconfig CC=$CC - make ${koutput} modules_prepare CC=$CC + # On arm64 this object is required to build with ftrace support + if [ "${karch}" = "arm64" ]; then + if vergte "$KVERSION" "4.13-rc1"; then + make arch/arm64/kernel/ftrace-mod.o CC="$CC" ${koutput} + fi + fi # Version specific tasks - case "$kversion" in + case "$ktag" in Ubuntu*) # Add Ubuntu ABI number to kernel headers, this is normally done by the packaging code - ABINUM=$(echo $kversion | grep -P -o 'Ubuntu-(lts-)?.*-\K\d+(?=\..*)') - echo "#define UTS_UBUNTU_RELEASE_ABI $ABINUM" >> ${outdir}/include/generated/utsrelease.h + ABINUM="$(echo "$ktag" | grep -P -o 'Ubuntu-(lts-)?.*-\K\d+(?=\..*)')" + echo "#define UTS_UBUNTU_RELEASE_ABI $ABINUM" >> "${outdir}"/include/generated/utsrelease.h ;; esac - - # On powerpc this object is required to link modules - if [ "${karch}" = "powerpc" ]; then - make ${koutput} arch/powerpc/lib/crtsavres.o CC=$CC - fi } @@ -100,22 +124,18 @@ build_modules() { kdir="$1" bdir="$2" - # Get kernel version from source tree - cd "${kdir}" - kversion=$(make kernelversion) - # Enter latency-tracker source dir cd "${LTTSRCDIR}" # kernels 3.10 to 3.10.13 and 3.11 to 3.11.2 introduce a deadlock in the # timekeeping subsystem. We want those build to fail. - if { vergte "$kversion" "3.10" && verlte "$kversion" "3.10.13"; } || \ - { vergte "$kversion" "3.11" && verlte "$kversion" "3.11.2"; }; then + if { vergte "$KVERSION" "3.10" && verlte "$KVERSION" "3.10.13"; } || \ + { vergte "$KVERSION" "3.11" && verlte "$KVERSION" "3.11.2"; }; then set +e # Build modules - KERNELDIR="${kdir}" make -j${NPROC} V=1 CC=$CC + KERNELDIR="${kdir}" make -j"${NPROC}" V=1 CC="$CC" # We expect this build to fail, if it doesn't, fail the job. if [ "$?" -eq 0 ]; then @@ -128,18 +148,18 @@ build_modules() { set -e - KERNELDIR="${kdir}" make clean CC=$CC + KERNELDIR="${kdir}" make clean CC="$CC" else # Regular build # Build modules against full kernel sources - KERNELDIR="${kdir}" make -j${NPROC} V=1 CC=$CC + KERNELDIR="${kdir}" make -j"${NPROC}" V=1 CC="$CC" # Install modules to build dir - KERNELDIR="${kdir}" make INSTALL_MOD_PATH="${bdir}" modules_install CC=$CC + KERNELDIR="${kdir}" make INSTALL_MOD_PATH="${bdir}" modules_install CC="$CC" # Clean build dir - KERNELDIR="${kdir}" make clean CC=$CC + KERNELDIR="${kdir}" make clean CC="$CC" fi } @@ -160,77 +180,89 @@ LTTBUILDKHDRDIR="${WORKSPACE}/build/latency-tracker-khdr" # Setup cross compile env if available -if [ "x${cross_arch:-}" != "x" ]; then +if [ "x${cross_arch}" != "x" ]; then case "$cross_arch" in "armhf") karch="arm" cross_compile="arm-linux-gnueabihf-" + vanilla_config="allyesconfig" ubuntu_config="armhf-config.flavour.generic" ;; "arm64") karch="arm64" cross_compile="aarch64-linux-gnu-" + vanilla_config="allyesconfig" ubuntu_config="arm64-config.flavour.generic" ;; "powerpc") karch="powerpc" cross_compile="powerpc-linux-gnu-" + vanilla_config="ppc44x_defconfig" ubuntu_config="powerpc-config.flavour.powerpc-smp" ;; "ppc64el") karch="powerpc" cross_compile="powerpc64le-linux-gnu-" + vanilla_config="pseries_le_defconfig" ubuntu_config="ppc64el-config.flavour.generic" ;; *) - echo "Unsupported cross arch $arch" + echo "Unsupported cross arch $cross_arch" exit 1 ;; esac - # Use default gcc when cross-compiling - CC="${cross_compile}gcc" + # Use gcc 4.9, older kernel don't build with gcc 5 + CC="${cross_compile}gcc-4.9" # Export variables used by Kbuild for cross compilation export ARCH="${karch}" export CROSS_COMPILE="${cross_compile}" + oldconf_target="olddefconfig" + # Set arch specific values if we are not cross compiling -elif [ "x${arch:-}" != "x" ]; then +elif [ "x${arch}" != "x" ]; then case "$arch" in "x86-32") karch="x86" + vanilla_config="allyesconfig" ubuntu_config="i386-config.flavour.generic" ;; "x86-64") karch="x86" + vanilla_config="allyesconfig" ubuntu_config="amd64-config.flavour.generic" ;; "armhf") karch="arm" + vanilla_config="allyesconfig" ubuntu_config="armhf-config.flavour.generic" ;; "arm64") karch="arm64" + vanilla_config="allyesconfig" ubuntu_config="arm64-config.flavour.generic" ;; "powerpc") karch="powerpc" + vanilla_config="allyesconfig" ubuntu_config="powerpc-config.flavour.powerpc-smp" ;; "ppc64el") karch="powerpc" + vanilla_config="allyesconfig" ubuntu_config="ppc64el-config.flavour.generic" ;; @@ -243,6 +275,8 @@ elif [ "x${arch:-}" != "x" ]; then # Use gcc 4.9, older kernel don't build with gcc 5 CC=gcc-4.9 + oldconf_target="silentoldconfig" + else echo "Not arch or cross_arch specified" exit 1 @@ -261,6 +295,9 @@ mkdir -p "${LNXBUILDDIR}" "${LNXHDRDIR}" "${LTTBUILDKSRCDIR}" "${LTTBUILDKHDRDIR # Enter linux source dir cd "${LNXSRCDIR}" +# Get kernel version from source tree +KVERSION=$(make kernelversion) + prepare_lnx_sources "." # For RT kernels, copy version file @@ -279,13 +316,13 @@ find . -path './include/*' -prune \ cp -a scripts include "${LNXHDRDIR}" # Copy arch includes -(find arch -name include -type d -print | \ - xargs -n1 -i: find : -type f) | \ +(find arch -name include -type d -print0 | \ + xargs -0 -n1 -i: find : -type f) | \ cpio -pd --preserve-modification-time "${LNXHDRDIR}" # Copy arch scripts -(find arch -name scripts -type d -print | \ - xargs -n1 -i: find : -type f) | \ +(find arch -name scripts -type d -print0 | \ + xargs -0 -n1 -i: find : -type f) | \ cpio -pd --preserve-modification-time "${LNXHDRDIR}" # Cleanup scripts @@ -297,6 +334,13 @@ if [ "${karch}" = "powerpc" ]; then cp -a --parents arch/powerpc/lib/crtsavres.[So] "${LNXHDRDIR}/" fi +# On arm64 this object is required to build with ftrace support +if [ "${karch}" = "arm64" ]; then + if vergte "$KVERSION" "4.13-rc1"; then + cp -a --parents arch/arm64/kernel/ftrace-mod.[So] "${LNXHDRDIR}/" + fi +fi + # Copy modules related stuff, if available if [ -s Module.symvers ]; then cp Module.symvers "${LNXHDRDIR}" -- 2.34.1