2 * Copyright (C) 2017 - Francis Deslauriers <francis.deslauriers@efficios.com>
4 * This program is free software: you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation, either version 3 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 import hudson.console.HyperlinkNote
21 import org.eclipse.jgit.api.Git
22 import org.eclipse.jgit.lib.Ref
24 class InvalidKVersionException extends Exception {
25 public InvalidKVersionException(String message) {
30 class EmptyKVersionException extends Exception {
31 public EmptyKVersionException(String message) {
36 class VanillaKVersion implements Comparable<VanillaKVersion> {
42 Integer rc = Integer.MAX_VALUE
43 Boolean inStable = false;
47 VanillaKVersion(version) {
51 static VanillaKVersion minKVersion() {
52 return new VanillaKVersion("v0.0.0")
55 static VanillaKVersion maxKVersion() {
56 return new VanillaKVersion("v" + Integer.MAX_VALUE + ".0.0")
59 static VanillaKVersion factory(version) {
60 return new VanillaKVersion(version)
68 this.rc = Integer.MAX_VALUE
71 throw new EmptyKVersionException("Empty kernel version")
74 def match = version =~ /^v(\d+)\.(\d+)(\.(\d+))?(\.(\d+))?(-rc(\d+))?$/
76 throw new InvalidKVersionException("Invalid kernel version: ${version}")
82 this.major = Integer.parseInt(match.group(1))
83 if (this.major <= 2) {
85 this.majorB = Integer.parseInt(match.group(2))
89 if (match.group(2 + offset) != null) {
90 this.minor = Integer.parseInt(match.group(2 + offset))
94 if (match.group(4 + offset) != null) {
95 this.patch = Integer.parseInt(match.group(4 + offset))
100 if (match.group(8) != null) {
101 this.rc = Integer.parseInt(match.group(8))
105 Boolean isInStableBranch() {
109 // Return true if both version are of the same stable branch
110 Boolean isSameStable(VanillaKVersion o) {
111 if (this.major != o.major) {
114 if (this.majorB != o.majorB) {
117 if (this.minor != o.minor) {
124 @Override int compareTo(VanillaKVersion o) {
125 if (this.major != o.major) {
126 return Integer.compare(this.major, o.major)
128 if (this.majorB != o.majorB) {
129 return Integer.compare(this.majorB, o.majorB)
131 if (this.minor != o.minor) {
132 return Integer.compare(this.minor, o.minor)
134 if (this.patch != o.patch) {
135 return Integer.compare(this.patch, o.patch)
137 if (this.rc != o.rc) {
138 return Integer.compare(this.rc, o.rc)
146 String vString = "v${this.major}"
148 if (this.majorB > 0) {
149 vString = vString.concat(".${this.majorB}")
152 vString = vString.concat(".${this.minor}")
154 if (this.patch > 0) {
155 vString = vString.concat(".${this.patch}")
158 if (this.rc > 0 && this.rc < Integer.MAX_VALUE) {
159 vString = vString.concat("-rc${this.rc}")
165 // Save the hashmap containing all the jobs and their status to disk. We can do
166 // that because this job is configured to always run on the master node on
168 def SaveCurrentJobsToWorkspace = { currentJobs, ondiskpath->
170 File myFile = new File(ondiskpath);
171 myFile.createNewFile();
172 def out = new ObjectOutputStream(new FileOutputStream(ondiskpath))
173 out.writeObject(currentJobs)
176 println("Failed to save previous Git object IDs to disk." + e);
180 // Load the hashmap containing all the jobs and their last status from disk.
181 // It's possible because this job is configured to always run on the master
183 def LoadPreviousJobsFromWorkspace = { ondiskpath ->
184 def previousJobs = [:]
186 File myFile = new File(ondiskpath);
187 def input = new ObjectInputStream(new FileInputStream(ondiskpath))
188 previousJobs = input.readObject()
191 println("Failed to load previous runs from disk." + e);
197 def GetHeadCommits = { remoteRepo, branchesOfInterest ->
198 def remoteHeads = [:]
199 def remoteHeadRefs = Git.lsRemoteRepository()
202 .setRemote(remoteRepo).call()
204 remoteHeadRefs.each {
205 def branch = it.getName().replaceAll('refs/heads/', '')
206 if (branchesOfInterest.contains(branch))
207 remoteHeads[branch] = it.getObjectId().name()
213 def GetTagIds = { remoteRepo ->
215 def remoteTagRefs = Git.lsRemoteRepository()
218 .setRemote(remoteRepo).call()
221 // Exclude release candidate tags
222 if (!it.getName().contains('-rc')) {
223 remoteTags[it.getName().replaceAll('refs/tags/', '')] = it.getObjectId().name()
230 def GetLastTagOfBranch = { tagRefs, branch ->
231 def tagVersions = tagRefs.collect {new VanillaKVersion(it.key)}
232 def currMax = new VanillaKVersion('v0.0.0');
233 if (!branch.contains('master')){
234 def targetVersion = new VanillaKVersion(branch.replaceAll('linux-', 'v').replaceAll('.y', ''))
236 if (it.isSameStable(targetVersion)) {
244 if (!it.isInStableBranch() && currMax < it) {
249 return currMax.toString()
252 // Returns the latest tags of each of the branches passed in the argument
253 def GetLastTagIds = { remoteRepo, branchesOfInterest ->
254 def remoteHeads = GetHeadCommits(remoteRepo, branchesOfInterest)
255 def remoteTagRefs = GetTagIds(remoteRepo)
256 def remoteLastTagCommit = [:]
258 remoteTagRefs = remoteTagRefs.findAll { !it.key.contains("v2.") }
259 branchesOfInterest.each {
260 remoteLastTagCommit[it] = remoteTagRefs[GetLastTagOfBranch(remoteTagRefs, it)]
263 return remoteLastTagCommit
266 def CraftJobName = { jobType, linuxBranch, lttngBranch ->
267 return "${jobType}_k${linuxBranch}_l${lttngBranch}"
270 def LaunchJob = { jobName, jobInfo ->
271 def job = Hudson.instance.getJob(jobName)
273 for (paramdef in job.getProperty(ParametersDefinitionProperty.class).getParameterDefinitions()) {
274 // If there is a default value for this parameter, use it. Don't use empty
275 // default value parameters.
276 if (paramdef.getDefaultValue()) {
277 params += paramdef.getDefaultParameterValue();
281 params.add(new StringParameterValue('LTTNG_TOOLS_COMMIT_ID', jobInfo['config']['toolsCommit']))
282 params.add(new StringParameterValue('LTTNG_MODULES_COMMIT_ID', jobInfo['config']['modulesCommit']))
283 params.add(new StringParameterValue('LTTNG_UST_COMMIT_ID', jobInfo['config']['ustCommit']))
284 params.add(new StringParameterValue('KERNEL_TAG_ID', jobInfo['config']['linuxTagID']))
285 def currBuild = job.scheduleBuild2(0, new Cause.UpstreamCause(build), new ParametersAction(params))
287 if (currBuild != null ) {
288 println("Launching job: ${HyperlinkNote.encodeTo('/' + job.url, job.fullDisplayName)}");
290 println("Job ${jobName} not found or deactivated.");
296 final String toolsRepo = "https://github.com/lttng/lttng-tools.git"
297 final String modulesRepo = "https://github.com/lttng/lttng-modules.git"
298 final String ustRepo = "https://github.com/lttng/lttng-ust.git"
299 final String linuxRepo = "git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git"
301 final String pastJobsPath = build.getEnvironment(listener).get('WORKSPACE') + "/pastjobs";
303 def recentLttngBranchesOfInterest = ['master', 'stable-2.10', 'stable-2.9']
304 def recentLinuxBranchesOfInterest = ['master',
312 def legacyLttngBranchesOfInterest = []
313 def legacyLinuxBranchesOfInterest = []
315 def vmLinuxBranchesOfInterest = ['linux-3.18.y']
317 // Generate configurations of interest.
318 def configurationOfInterest = [] as Set
320 recentLttngBranchesOfInterest.each { lttngBranch ->
321 recentLinuxBranchesOfInterest.each { linuxBranch ->
322 configurationOfInterest.add([lttngBranch, linuxBranch])
326 legacyLttngBranchesOfInterest.each { lttngBranch ->
327 legacyLinuxBranchesOfInterest.each { linuxBranch ->
328 configurationOfInterest.add([lttngBranch, linuxBranch])
332 def lttngBranchesOfInterest = recentLttngBranchesOfInterest + legacyLttngBranchesOfInterest
333 def linuxBranchesOfInterest = recentLinuxBranchesOfInterest + legacyLinuxBranchesOfInterest + vmLinuxBranchesOfInterest
335 // For LTTng branches, we look for new commits.
336 def toolsHeadCommits = GetHeadCommits(toolsRepo, lttngBranchesOfInterest)
337 def modulesHeadCommits = GetHeadCommits(modulesRepo, lttngBranchesOfInterest)
338 def ustHeadCommits = GetHeadCommits(ustRepo, lttngBranchesOfInterest)
340 // For Linux branches, we look for new non-RC tags.
341 def linuxLastTagIds = GetLastTagIds(linuxRepo, linuxBranchesOfInterest)
343 def CraftConfig = { linuxBr, lttngBr ->
346 job['config']['linuxBranch'] = linuxBr;
347 job['config']['lttngBranch'] = lttngBr;
348 job['config']['linuxTagID'] = linuxLastTagIds[linuxBr];
349 job['config']['toolsCommit'] = toolsHeadCommits[lttngBr];
350 job['config']['modulesCommit'] = modulesHeadCommits[lttngBr];
351 job['config']['ustCommit'] = ustHeadCommits[lttngBr];
352 job['status'] = 'NOT_SET';
357 // Check what type of jobs should be triggered.
358 triggerJobName = build.project.getFullDisplayName();
359 if (triggerJobName.contains("vm_tests")) {
360 jobType = 'vm_tests';
361 recentLttngBranchesOfInterest.each { lttngBranch ->
362 vmLinuxBranchesOfInterest.each { linuxBranch ->
363 configurationOfInterest.add([lttngBranch, linuxBranch])
366 } else if (triggerJobName.contains("baremetal_tests")) {
367 jobType = 'baremetal_tests';
368 } else if (triggerJobName.contains("baremetal_benchmarks")) {
369 jobType = 'baremetal_benchmarks';
372 // Hashmap containing all the jobs, their configuration (commit id, etc. )and
373 // their status (SUCCEEDED, FAILED, etc.). This Hashmap is made of basic strings
374 // rather than objects and enums because strings are easily serializable.
375 def currentJobs = [:];
377 // Get an up to date view of all the branches of interest.
378 configurationOfInterest.each { lttngBr, linuxBr ->
379 def jobName = CraftJobName(jobType, linuxBr, lttngBr);
380 currentJobs[jobName] = CraftConfig(linuxBr, lttngBr);
382 // Add fuzzing job in vm_tests on master branches of lttng and linux.
383 //if (jobType == 'vm_tests' && lttngBr == 'master' && linuxBr == 'master') {
384 // def vmFuzzingJobName = CraftJobName(jobType + '_fuzzing', linuxBr, lttngBr);
385 // currentJobs[vmFuzzingJobName] = CraftConfig(linuxBr, lttngBr);
390 def jobNameCanary = jobType + "_canary";
391 currentJobs[jobNameCanary] = [:];
392 currentJobs[jobNameCanary]['config'] = [:];
393 currentJobs[jobNameCanary]['config']['linuxBranch'] = 'v4.4.194';
394 currentJobs[jobNameCanary]['config']['lttngBranch'] = 'v2.10.7';
395 currentJobs[jobNameCanary]['config']['linuxTagID'] ='a227f8436f2b21146fc024d84e6875907475ace2';
396 currentJobs[jobNameCanary]['config']['toolsCommit'] = '93fa2c9ff6b52c30173bee80445501ce8677fecc'
397 currentJobs[jobNameCanary]['config']['modulesCommit'] = 'fe3ca7a9045221ffbedeac40ba7e09b1fc500e21'
398 currentJobs[jobNameCanary]['config']['ustCommit'] = '0172ce8ece2102d46c7785e6bd96163225c59e49'
399 currentJobs[jobNameCanary]['status'] = 'NOT_SET';
400 currentJobs[jobNameCanary]['build'] = null;
402 def pastJobs = LoadPreviousJobsFromWorkspace(pastJobsPath);
407 def isAborted = false
410 currentJobs.each { jobName, jobInfo ->
411 // If the job ran in the past, we check if the IDs changed since.
412 // Fetch past results only if the job is not of type canary or fuzzing.
413 if (!jobName.contains('_canary') && !jobName.contains('_fuzzing') &&
414 pastJobs.containsKey(jobName) &&
415 build.getBuildVariables().get('FORCE_JOB_RUN') == 'false') {
416 pastJob = pastJobs[jobName];
418 // If the code has not changed report previous status.
419 if (pastJob['config'] == jobInfo['config']) {
420 // if the config has not changed, we keep it.
421 // if it's failed, we don't launch a new job and keep it failed.
422 jobInfo['status'] = pastJob['status'];
423 if (pastJob['status'] == 'FAILED' &&
424 build.getBuildVariables().get('FORCE_FAILED_JOB_RUN') == 'false') {
425 println("${jobName} as not changed since the last failed run. Don't run it again.");
426 // Marked the umbrella job for failure but still run the jobs that since the
430 } else if (pastJob['status'] == 'ABORTED') {
431 println("${jobName} as not changed since last aborted run. Run it again.");
432 } else if (pastJob['status'] == 'SUCCEEDED') {
433 println("${jobName} as not changed since the last successful run. Don't run it again.");
439 jobInfo['status'] = 'PENDING';
440 jobInfo['build'] = LaunchJob(jobName, jobInfo);
444 while (ongoingJobs > 0) {
445 currentJobs.each { jobName, jobInfo ->
447 if (jobInfo['status'] != 'PENDING') {
451 jobBuild = jobInfo['build']
453 // The isCancelled() method checks if the run was cancelled before
454 // execution. We consider such run as being aborted.
455 if (jobBuild.isCancelled()) {
456 println("${jobName} was cancelled before launch.")
458 abortedRuns.add(jobName);
460 jobInfo['status'] = 'ABORTED'
461 // Invalidate the build field, as it's not serializable and we don't need
463 jobInfo['build'] = null;
464 } else if (jobBuild.isDone()) {
466 jobExitStatus = jobBuild.get();
468 // Invalidate the build field, as it's not serializable and we don't need
470 jobInfo['build'] = null;
471 println("${jobExitStatus.fullDisplayName} completed with status ${jobExitStatus.result}.");
473 // If the job didn't succeed, add its name to the right list so it can
474 // be printed at the end of the execution.
476 switch (jobExitStatus.result) {
479 abortedRuns.add(jobName);
480 jobInfo['status'] = 'ABORTED'
484 failedRuns.add(jobName);
485 jobInfo['status'] = 'FAILED'
488 jobInfo['status'] = 'SUCCEEDED'
496 // Sleep before the next iteration.
500 if (e in InterruptedException) {
501 build.setResult(hudson.model.Result.ABORTED)
502 throw new InterruptedException()
509 //All jobs are done running. Save their exit status to disk.
510 SaveCurrentJobsToWorkspace(currentJobs, pastJobsPath);
512 // Get log of failed runs.
513 if (failedRuns.size() > 0) {
514 println("Failed job(s):");
515 for (failedRun in failedRuns) {
516 println("\t" + failedRun)
520 // Get log of aborted runs.
521 if (abortedRuns.size() > 0) {
522 println("Cancelled job(s):");
523 for (cancelledRun in abortedRuns) {
524 println("\t" + cancelledRun)
528 // Mark this build as Failed if atleast one child build has failed and mark as
529 // aborted if there was no failure but atleast one job aborted.
531 build.setResult(hudson.model.Result.FAILURE)
532 } else if (isAborted) {
533 build.setResult(hudson.model.Result.ABORTED)