From 2c1d386a795ee93d8d8af91559e8b23ea036d82a Mon Sep 17 00:00:00 2001 From: Michael Jeanson Date: Mon, 8 Feb 2016 14:15:25 -0500 Subject: [PATCH] Added modules RT build and sloccount --- .../ansible/roles/common/vars/Debian.yml | 1 + jobs/lttng-modules.yaml | 96 ++++++- scripts/lttng-modules/build.sh | 6 +- scripts/lttng-modules/master-rt.groovy | 251 ++++++++++++++++++ .../{master.groovy => master-vanilla.groovy} | 0 scripts/lttng-modules/param-build.sh | 6 +- 6 files changed, 352 insertions(+), 8 deletions(-) create mode 100644 scripts/lttng-modules/master-rt.groovy rename scripts/lttng-modules/{master.groovy => master-vanilla.groovy} (100%) diff --git a/automation/ansible/roles/common/vars/Debian.yml b/automation/ansible/roles/common/vars/Debian.yml index 0a43214..776d608 100644 --- a/automation/ansible/roles/common/vars/Debian.yml +++ b/automation/ansible/roles/common/vars/Debian.yml @@ -16,6 +16,7 @@ __common_packages: - libtap-harness-archive-perl - jq - cppcheck + - cloc - python-virtualenv - curl - gettext diff --git a/jobs/lttng-modules.yaml b/jobs/lttng-modules.yaml index c2743a4..27436c6 100644 --- a/jobs/lttng-modules.yaml +++ b/jobs/lttng-modules.yaml @@ -73,6 +73,14 @@ - job-template: name: lttng-modules_{mversion}_build-vanilla defaults: lttng-modules + description: | + The LTTng modules provide Linux kernel tracing capability to the LTTng + 2.0 tracer toolset. + + This job will build the {mversion} branch against all stable vanilla + kernel tags. + +

Job is managed by Jenkins Job Builder.

node: 'master' @@ -101,7 +109,7 @@ builders: - system-groovy: command: - !include-raw-escape: scripts/lttng-modules/master.groovy + !include-raw-escape: scripts/lttng-modules/master-vanilla.groovy publishers: - workspace-cleanup @@ -109,6 +117,14 @@ - job-template: name: lttng-modules_{mversion}_build-{uversion} defaults: lttng-modules + description: | + The LTTng modules provide Linux kernel tracing capability to the LTTng + 2.0 tracer toolset. + + This job will build the {mversion} branch against all Ubuntu {uversion} + released kernels, including the LTS backport kernels. + +

Job is managed by Jenkins Job Builder.

node: 'master' @@ -142,6 +158,50 @@ publishers: - workspace-cleanup +- job-template: + name: lttng-modules_{mversion}_build-rt + defaults: lttng-modules + description: | + The LTTng modules provide Linux kernel tracing capability to the LTTng + 2.0 tracer toolset. + + This job will build the {mversion} branch against all Linutronix RT + kernels. + +

Job is managed by Jenkins Job Builder.

+ + node: 'master' + + parameters: + - string: + name: 'mversion' + default: '{mversion}' + description: 'The lttng-modules branch to build.' + - string: + name: 'maxConcurrentBuild' + default: '4' + description: 'The maximum number of concurrent child build to run.' + - string: + name: 'kverfloor' + default: 'v2.6.36-rt0-rebase' + description: 'The lowest kernel version to build.' + - string: + name: 'kgitrepo' + default: 'git://artifacts.internal.efficios.com/git/linux-rt-devel.git' + description: 'The linux kernel git repository url.' + - string: + name: 'kbuildjob' + default: 'lttng-modules_VERSION_param-build' + description: 'The parametrized job to use for child builds.' + + builders: + - system-groovy: + command: + !include-raw-escape: scripts/lttng-modules/master-rt.groovy + + publishers: + - workspace-cleanup + - job-template: name: lttng-modules_VERSION_param-build defaults: lttng-modules @@ -175,7 +235,7 @@ concurrent: true logrotate: - daysToKeep: -1 + daysToKeep: 2 scm: - git: @@ -187,6 +247,8 @@ skip-tag: true basedir: lttng-modules + triggers: + builders: - shell: | git clone --depth=1 -b "$kversion" --reference $HOME/gitcache/linux-stable.git/ "$kgitrepo" linux @@ -263,6 +325,32 @@ notify-every-unstable-build: true send-to-individuals: false +- job-template: + name: lttng-modules_{mversion}_sloccount + defaults: lttng-modules + description: | + The LTTng modules provide Linux kernel tracing capability to the LTTng + 2.0 tracer toolset. + + This job runs the sloccount utility and generates a trend report. + +

Job is managed by Jenkins Job Builder.

+ + triggers: + - pollscm: + cron: "@daily" + + builders: + - shell: | + cloc --by-file --xml --out=cloc.xml lttng-modules/ + + publishers: + - archive: + artifacts: 'cloc.xml' + allow-empty: false + - sloccount: + report-files: 'cloc.xml' + ## Project - project: @@ -271,6 +359,8 @@ !include: jobs/inc/lttng-modules-versions.yaml.inc jobs: - 'lttng-modules_{mversion}_build-vanilla' + - 'lttng-modules_{mversion}_build-rt': + mversion: master - 'lttng-modules_{mversion}_build-{uversion}': uversion: - trusty @@ -278,5 +368,7 @@ - 'lttng-modules_VERSION_param-build': arch: !!python/tuple [x86-32, x86-64] - 'lttng-modules_{mversion}_cppcheck' + - 'lttng-modules_{mversion}_sloccount': + mversion: master - 'lttng-modules_{mversion}_coverity': mversion: master diff --git a/scripts/lttng-modules/build.sh b/scripts/lttng-modules/build.sh index 0d83cf4..ad41890 100755 --- a/scripts/lttng-modules/build.sh +++ b/scripts/lttng-modules/build.sh @@ -65,7 +65,7 @@ if { vergte "$KVERSION" "3.10" && verlte "$KVERSION" "3.10.13"; } || \ set +e # Build modules - make -j${NPROC} -C "${LNXBINDIR}" M="$(pwd)" V=1 CONFIG_LTTNG=m + KERNELDIR="${LNXBINDIR}" make -j${NPROC} V=1 CONFIG_LTTNG=m # We expect this build to fail, if it doesn't, fail the job. if [ "$?" -eq 0 ]; then @@ -80,10 +80,10 @@ if { vergte "$KVERSION" "3.10" && verlte "$KVERSION" "3.10.13"; } || \ else # Regular build # Build modules - make -j${NPROC} -C "${LNXBINDIR}" M="$(pwd)" V=1 CONFIG_LTTNG=m + KERNELDIR="${LNXBINDIR}" make -j${NPROC} V=1 CONFIG_LTTNG=m # Install modules to build dir - make INSTALL_MOD_PATH="${BUILDDIR}" -C "${LNXBINDIR}" M="$(pwd)" modules_install + KERNELDIR="${LNXBINDIR}" make INSTALL_MOD_PATH="${BUILDDIR}" modules_install fi # EOF diff --git a/scripts/lttng-modules/master-rt.groovy b/scripts/lttng-modules/master-rt.groovy new file mode 100644 index 0000000..796680f --- /dev/null +++ b/scripts/lttng-modules/master-rt.groovy @@ -0,0 +1,251 @@ +/** + * Copyright (C) 2016 - Michael Jeanson + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +import hudson.model.* +import hudson.AbortException +import hudson.console.HyperlinkNote +import java.util.concurrent.CancellationException +import org.eclipse.jgit.api.Git +import org.eclipse.jgit.lib.Ref + + +class kVersion implements Comparable { + + Integer major = 0; + Integer majorB = 0; + Integer minor = 0; + Integer patch = 0; + Integer rt = 0; + + kVersion() {} + + kVersion(version) { + this.parse(version) + } + + def parse(version) { + this.major = 0 + this.majorB = 0 + this.minor = 0 + this.patch = 0 + this.rt = 0 + + def match = version =~ /^v(\d+)\.(\d+)(\.(\d+))?(\.(\d+))?(-rt(\d+)-rebase)$/ + if (!match) { + throw new Exception("Invalid kernel version: ${version}") + } + + Integer offset = 0; + + // Major + this.major = Integer.parseInt(match.group(1)) + if (this.major <= 2) { + offset = 2 + this.majorB = Integer.parseInt(match.group(2)) + } + + // Minor + if (match.group(2 + offset) != null) { + this.minor = Integer.parseInt(match.group(2 + offset)) + } + + // Patch level + if (match.group(4 + offset) != null) { + this.patch = Integer.parseInt(match.group(4 + offset)) + } + + // RT + this.rt = Integer.parseInt(match.group(8)) + } + + @Override int compareTo(kVersion o) { + if (this.major != o.major) { + return Integer.compare(this.major, o.major); + } + if (this.majorB != o.majorB) { + return Integer.compare(this.majorB, o.majorB); + } + if (this.minor != o.minor) { + return Integer.compare(this.minor, o.minor); + } + if (this.patch != o.patch) { + return Integer.compare(this.patch, o.patch); + } + if (this.rt != o.rc) { + return Integer.compare(this.rt, o.rt); + } + + // Same version + return 0; + } + + String toString() { + String vString = "v${this.major}" + + if (this.majorB > 0) { + vString = vString.concat(".${this.majorB}") + } + + vString = vString.concat(".${this.minor}") + + if (this.patch > 0) { + vString = vString.concat(".${this.patch}") + } + + if (this.rt > 0) { + vString = vString.concat("-rt${this.rt}-rebase") + } + return vString + } +} + + +// Retrieve parameters of the current build +def mversion = build.buildVariableResolver.resolve('mversion') +def maxConcurrentBuild = build.buildVariableResolver.resolve('maxConcurrentBuild') +def kgitrepo = build.buildVariableResolver.resolve('kgitrepo') +def kverfloor = new kVersion(build.buildVariableResolver.resolve('kverfloor')) +def job = Hudson.instance.getJob(build.buildVariableResolver.resolve('kbuildjob')) + +// Get the out variable +def config = new HashMap() +def bindings = getBinding() +config.putAll(bindings.getVariables()) +def out = config['out'] + +def jlc = new jenkins.model.JenkinsLocationConfiguration() +def jenkinsUrl = jlc.url + +// Get tags from git repository +def refs = Git.lsRemoteRepository().setTags(true).setRemote(kgitrepo).call(); + +// Get kernel versions to build +def kversions = [] +for (ref in refs) { + def match = ref.getName() =~ /^refs\/tags\/(v[\d\.]+(-rt(\d+)-rebase))$/ + + if (match) { + def v = new kVersion(match.group(1)) + + if (v >= kverfloor) { + kversions.add(v) + } + } +} + +kversions.sort() + +// Debug +println "Building the following kernel versions:" +for (k in kversions) { + println k +} + +// Debug: Stop build here +//throw new InterruptedException() + +def joburl = HyperlinkNote.encodeTo('/' + job.url, job.fullDisplayName) + +def allBuilds = [] +def ongoingBuild = [] +def failedRuns = [] +def isFailed = false + +// Loop while we have kernel versions remaining or jobs running +while ( kversions.size() != 0 || ongoingBuild.size() != 0 ) { + + if(ongoingBuild.size() < maxConcurrentBuild.toInteger() && kversions.size() != 0) { + def kversion = kversions.pop() + def job_params = [ + new StringParameterValue('mversion', mversion), + new StringParameterValue('kversion', kversion.toString()), + new StringParameterValue('kgitrepo', kgitrepo), + ] + + // Launch the parametrized build + def param_build = job.scheduleBuild2(0, new Cause.UpstreamCause(build), new ParametersAction(job_params)) + println "triggering ${joburl} for the ${mversion} branch on kernel ${kversion}" + + // Add it to the ongoing build queue + ongoingBuild.push(param_build) + + } else { + + println "Waiting... Queued: " + kversions.size() + " Running: " + ongoingBuild.size() + try { + Thread.sleep(5000) + } catch(e) { + if (e in InterruptedException) { + build.setResult(hudson.model.Result.ABORTED) + throw new InterruptedException() + } else { + throw(e) + } + } + + def i = ongoingBuild.iterator() + while ( i.hasNext() ) { + currentBuild = i.next() + if ( currentBuild.isCancelled() || currentBuild.isDone() ) { + // Remove from queue + i.remove() + + // Print results + def matrixParent = currentBuild.get() + allBuilds.add(matrixParent) + def kernelStr = matrixParent.buildVariableResolver.resolve("kversion") + println "${matrixParent.fullDisplayName} (${kernelStr}) completed with status ${matrixParent.result}" + + // Process child runs of matrixBuild + def childRuns = matrixParent.getRuns() + for ( childRun in childRuns ) { + println "\t${childRun.fullDisplayName} (${kernelStr}) completed with status ${childRun.result}" + if (childRun.result != Result.SUCCESS) { + failedRuns.add(childRun) + isFailed = true + } + } + } + } + } +} + +// Get log of failed runs +for (failedRun in failedRuns) { + println "---START---" + failedRun.writeWholeLogTo(out) + println "---END---" +} + +println "---Build report---" +for (b in allBuilds) { + def kernelStr = b.buildVariableResolver.resolve("kversion") + println "${b.fullDisplayName} (${kernelStr}) completed with status ${b.result}" +} + +// Clean all builds +// TODO: Delete only builds generated from this job run +for (b in job.getBuilds()) { + b.delete() +} + +// Mark this build failed if any child build has failed +if (isFailed) { + build.getExecutor().interrupt(Result.FAILURE) +} + +// EOF diff --git a/scripts/lttng-modules/master.groovy b/scripts/lttng-modules/master-vanilla.groovy similarity index 100% rename from scripts/lttng-modules/master.groovy rename to scripts/lttng-modules/master-vanilla.groovy diff --git a/scripts/lttng-modules/param-build.sh b/scripts/lttng-modules/param-build.sh index d14f49d..a905a38 100644 --- a/scripts/lttng-modules/param-build.sh +++ b/scripts/lttng-modules/param-build.sh @@ -82,7 +82,7 @@ if { vergte "$KVERSION" "3.10" && verlte "$KVERSION" "3.10.13"; } || \ set +e # Build modules - make -j${NPROC} -C "${LNXBUILDDIR}" M="$(pwd)" V=1 CONFIG_LTTNG=m + KERNELDIR="${LNXBUILDDIR}" make -j${NPROC} V=1 # We expect this build to fail, if it doesn't, fail the job. if [ "$?" -eq 0 ]; then @@ -97,10 +97,10 @@ if { vergte "$KVERSION" "3.10" && verlte "$KVERSION" "3.10.13"; } || \ else # Regular build # Build modules - make -j${NPROC} -C "${LNXBUILDDIR}" M="$(pwd)" V=1 CONFIG_LTTNG=m + KERNELDIR="${LNXBUILDDIR}" make -j${NPROC} V=1 # Install modules to build dir - make INSTALL_MOD_PATH="${BUILDDIR}" -C "${LNXBUILDDIR}" M="$(pwd)" modules_install + KERNELDIR="${LNXBUILDDIR}" make INSTALL_MOD_PATH="${BUILDDIR}" modules_install fi # EOF -- 2.34.1