scripts/system-tests/system-trigger.groovy

   1 /**
   2  * Copyright (C) 2017 - Francis Deslauriers <francis.deslauriers@efficios.com>
   3  *
   4  * This program is free software: you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation, either version 3 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16  */
  17
  18 import hudson.console.HyperlinkNote
  19 import hudson.model.*
  20 import java.io.File
  21 import org.eclipse.jgit.api.Git
  22 import org.eclipse.jgit.lib.Ref
  23 import groovy.transform.EqualsAndHashCode
  24
  25 class InvalidKVersionException extends Exception {
  26   public InvalidKVersionException(String message) {
  27     super(message)
  28   }
  29 }
  30
  31 class EmptyKVersionException extends Exception {
  32   public EmptyKVersionException(String message) {
  33     super(message)
  34   }
  35 }
  36
  37 class VanillaKVersion implements Comparable<VanillaKVersion> {
  38
  39   Integer major = 0
  40   Integer majorB = 0
  41   Integer minor = 0
  42   Integer patch = 0
  43   Integer rc = Integer.MAX_VALUE
  44   Boolean inStable = false;
  45
  46   VanillaKVersion() {}
  47
  48   VanillaKVersion(version) {
  49     this.parse(version)
  50   }
  51
  52   static VanillaKVersion minKVersion() {
  53     return new VanillaKVersion("v0.0.0")
  54   }
  55
  56   static VanillaKVersion maxKVersion() {
  57     return new VanillaKVersion("v" + Integer.MAX_VALUE + ".0.0")
  58   }
  59
  60   static VanillaKVersion factory(version) {
  61     return new VanillaKVersion(version)
  62   }
  63
  64   def parse(version) {
  65     this.major = 0
  66     this.majorB = 0
  67     this.minor = 0
  68     this.patch = 0
  69     this.rc = Integer.MAX_VALUE
  70
  71     if (!version) {
  72       throw new EmptyKVersionException("Empty kernel version")
  73     }
  74
  75     def match = version =~ /^v(\d+)\.(\d+)(\.(\d+))?(\.(\d+))?(-rc(\d+))?$/
  76     if (!match) {
  77       throw new InvalidKVersionException("Invalid kernel version: ${version}")
  78     }
  79
  80     Integer offset = 0;
  81
  82     // Major
  83     this.major = Integer.parseInt(match.group(1))
  84     if (this.major <= 2) {
  85       offset = 2
  86       this.majorB = Integer.parseInt(match.group(2))
  87     }
  88
  89     // Minor
  90     if (match.group(2 + offset) != null) {
  91       this.minor = Integer.parseInt(match.group(2 + offset))
  92     }
  93
  94     // Patch level
  95     if (match.group(4 + offset) != null) {
  96       this.patch = Integer.parseInt(match.group(4 + offset))
  97       this.inStable = true
  98     }
  99
 100     // RC
 101     if (match.group(8) != null) {
 102       this.rc = Integer.parseInt(match.group(8))
 103     }
 104   }
 105
 106   Boolean isInStableBranch() {
 107     return this.inStable
 108   }
 109
 110   // Return true if both version are of the same stable branch
 111   Boolean isSameStable(VanillaKVersion o) {
 112     if (this.major != o.major) {
 113       return false
 114     }
 115     if (this.majorB != o.majorB) {
 116       return false
 117     }
 118     if (this.minor != o.minor) {
 119       return false
 120     }
 121
 122     return true
 123   }
 124
 125   @Override int compareTo(VanillaKVersion o) {
 126     if (this.major != o.major) {
 127       return Integer.compare(this.major, o.major)
 128     }
 129     if (this.majorB != o.majorB) {
 130       return Integer.compare(this.majorB, o.majorB)
 131     }
 132     if (this.minor != o.minor) {
 133       return Integer.compare(this.minor, o.minor)
 134     }
 135     if (this.patch != o.patch) {
 136       return Integer.compare(this.patch, o.patch)
 137     }
 138     if (this.rc != o.rc) {
 139       return Integer.compare(this.rc, o.rc)
 140     }
 141
 142     // Same version
 143     return 0;
 144   }
 145
 146   String toString() {
 147     String vString = "v${this.major}"
 148
 149     if (this.majorB > 0) {
 150       vString = vString.concat(".${this.majorB}")
 151     }
 152
 153     vString = vString.concat(".${this.minor}")
 154
 155     if (this.patch > 0) {
 156       vString = vString.concat(".${this.patch}")
 157     }
 158
 159     if (this.rc > 0 && this.rc < Integer.MAX_VALUE) {
 160       vString = vString.concat("-rc${this.rc}")
 161     }
 162     return vString
 163   }
 164 }
 165
 166 @EqualsAndHashCode(includeFields=true)
 167 class RunConfiguration {
 168   def linuxBranch
 169   def linuxTagId
 170   def lttngBranch
 171   def lttngModulesCommitId
 172   def lttngToolsCommitId
 173   def lttngUstCommitId
 174   RunConfiguration(linuxBranch, linuxTagId, lttngBranch, lttngToolsCommitId,
 175                   lttngModulesCommitId, lttngUstCommitId) {
 176     this.linuxBranch = linuxBranch
 177     this.linuxTagId = linuxTagId
 178     this.lttngBranch = lttngBranch
 179     this.lttngModulesCommitId = lttngModulesCommitId
 180     this.lttngToolsCommitId = lttngToolsCommitId
 181     this.lttngUstCommitId = lttngUstCommitId
 182   }
 183
 184   String toString() {
 185     return "${this.linuxBranch}:{${this.linuxTagId}}, ${this.lttngBranch}" +
 186       ":{${this.lttngModulesCommitId}, ${this.lttngToolsCommitId}," +
 187       "${this.lttngUstCommitId}}"
 188   }
 189 }
 190
 191 def LoadPreviousIdsFromWorkspace = { ondiskpath ->
 192   def previousIds = []
 193   try {
 194     File myFile = new File(ondiskpath);
 195     def input = new ObjectInputStream(new FileInputStream(ondiskpath))
 196     previousIds = input.readObject()
 197     input.close()
 198   } catch (e) {
 199     println("Failed to load previous Git object IDs from disk." + e);
 200   }
 201   return previousIds
 202 }
 203
 204 def saveCurrentIdsToWorkspace = { currentIds, ondiskpath ->
 205   try {
 206     File myFile = new File(ondiskpath);
 207     myFile.createNewFile();
 208     def out = new ObjectOutputStream(new FileOutputStream(ondiskpath))
 209     out.writeObject(currentIds)
 210     out.close()
 211   } catch (e) {
 212     println("Failed to save previous Git object IDs from disk." + e);
 213   }
 214 }
 215
 216 def GetHeadCommits = { remoteRepo, branchesOfInterest ->
 217   def remoteHeads = [:]
 218   def remoteHeadRefs = Git.lsRemoteRepository()
 219                           .setTags(false)
 220                           .setHeads(true)
 221                           .setRemote(remoteRepo).call()
 222
 223   remoteHeadRefs.each {
 224     def branch = it.getName().replaceAll('refs/heads/', '')
 225     if (branchesOfInterest.contains(branch))
 226       remoteHeads[branch] = it.getObjectId().name()
 227   }
 228
 229   return remoteHeads
 230 }
 231
 232 def GetTagIds = { remoteRepo ->
 233   def remoteTags = [:]
 234   def remoteTagRefs = Git.lsRemoteRepository()
 235                          .setTags(true)
 236                          .setHeads(false)
 237                          .setRemote(remoteRepo).call()
 238
 239   remoteTagRefs.each {
 240     // Exclude release candidate tags
 241     if (!it.getName().contains('-rc')) {
 242       remoteTags[it.getName().replaceAll('refs/tags/', '')] = it.getObjectId().name()
 243     }
 244   }
 245
 246   return remoteTags
 247 }
 248
 249 def GetLastTagOfBranch = { tagRefs, branch ->
 250   def tagVersions = tagRefs.collect {new VanillaKVersion(it.key)}
 251   def currMax = new VanillaKVersion('v0.0.0');
 252   if (!branch.contains('master')){
 253     def targetVersion = new VanillaKVersion(branch.replaceAll('linux-', 'v').replaceAll('.y', ''))
 254     tagVersions.each {
 255       if (it.isSameStable(targetVersion)) {
 256         if (currMax < it) {
 257           currMax = it;
 258         }
 259       }
 260     }
 261   } else {
 262     tagVersions.each {
 263       if (!it.isInStableBranch() && currMax < it) {
 264         currMax = it;
 265       }
 266     }
 267   }
 268   return currMax.toString()
 269 }
 270
 271 // Returns the latest tags of each of the branches passed in the argument
 272 def GetLastTagIds = { remoteRepo, branchesOfInterest ->
 273   def remoteHeads = GetHeadCommits(remoteRepo, branchesOfInterest)
 274   def remoteTagRefs = GetTagIds(remoteRepo)
 275   def remoteLastTagCommit = [:]
 276
 277   remoteTagRefs = remoteTagRefs.findAll { !it.key.contains("v2.") }
 278   branchesOfInterest.each {
 279     remoteLastTagCommit[it] = remoteTagRefs[GetLastTagOfBranch(remoteTagRefs, it)]
 280   }
 281
 282   return remoteLastTagCommit
 283 }
 284
 285 def CraftJobName = { jobType, runConfig ->
 286   return "${jobType}_k${runConfig.linuxBranch}_l${runConfig.lttngBranch}"
 287 }
 288
 289 def LaunchJob = { jobName, runConfig ->
 290   def job = Hudson.instance.getJob(jobName)
 291   def params = []
 292   for (paramdef in job.getProperty(ParametersDefinitionProperty.class).getParameterDefinitions()) {
 293     // If there is a default value for this parameter, use it. Don't use empty
 294     // default value parameters.
 295     if (paramdef.getDefaultValue()) {
 296       params += paramdef.getDefaultParameterValue();
 297     }
 298   }
 299
 300   params.add(new StringParameterValue('LTTNG_TOOLS_COMMIT_ID', runConfig.lttngToolsCommitId))
 301   params.add(new StringParameterValue('LTTNG_MODULES_COMMIT_ID', runConfig.lttngModulesCommitId))
 302   params.add(new StringParameterValue('LTTNG_UST_COMMIT_ID', runConfig.lttngUstCommitId))
 303   params.add(new StringParameterValue('KERNEL_TAG_ID', runConfig.linuxTagId))
 304   def currBuild = job.scheduleBuild2(0, new Cause.UpstreamCause(build), new ParametersAction(params))
 305
 306   if (currBuild != null ) {
 307     println("Launching job: ${HyperlinkNote.encodeTo('/' + job.url, job.fullDisplayName)}");
 308   } else {
 309     println("Job ${jobName} not found or deactivated.");
 310   }
 311
 312   return currBuild
 313 }
 314
 315 final String toolsRepo = "https://github.com/lttng/lttng-tools.git"
 316 final String modulesRepo = "https://github.com/lttng/lttng-modules.git"
 317 final String ustRepo = "https://github.com/lttng/lttng-ust.git"
 318 final String linuxRepo = "git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git"
 319
 320 final String toolsOnDiskPath = build.getEnvironment(listener).get('WORKSPACE') + "/on-disk-tools-ref"
 321 final String modulesOnDiskPath = build.getEnvironment(listener).get('WORKSPACE') + "/on-disk-modules-ref"
 322 final String ustOnDiskPath = build.getEnvironment(listener).get('WORKSPACE') + "/on-disk-ust-ref"
 323 final String linuxOnDiskPath = build.getEnvironment(listener).get('WORKSPACE') + "/on-disk-linux-ref"
 324
 325 def recentLttngBranchesOfInterest = ['master', 'stable-2.10', 'stable-2.9']
 326 def recentLinuxBranchesOfInterest = ['master', 'linux-4.9.y', 'linux-4.4.y']
 327
 328 def legacyLttngBranchesOfInterest = ['stable-2.7']
 329 def legacyLinuxBranchesOfInterest = ['linux-3.18.y']
 330
 331 // Generate configurations of interest.
 332 def configurationOfInterest = [] as Set
 333
 334 recentLttngBranchesOfInterest.each { lttngBranch ->
 335   recentLinuxBranchesOfInterest.each { linuxBranch ->
 336     configurationOfInterest.add([lttngBranch, linuxBranch])
 337   }
 338 }
 339
 340 legacyLttngBranchesOfInterest.each { lttngBranch ->
 341   legacyLinuxBranchesOfInterest.each { linuxBranch ->
 342     configurationOfInterest.add([lttngBranch, linuxBranch])
 343   }
 344 }
 345
 346 def lttngBranchesOfInterest = recentLttngBranchesOfInterest + legacyLttngBranchesOfInterest
 347 def linuxBranchesOfInterest = recentLinuxBranchesOfInterest + legacyLinuxBranchesOfInterest
 348
 349 // For LTTng branches, we look for new commits.
 350 def toolsHeadCommits = GetHeadCommits(toolsRepo, lttngBranchesOfInterest)
 351 def modulesHeadCommits = GetHeadCommits(modulesRepo, lttngBranchesOfInterest)
 352 def ustHeadCommits = GetHeadCommits(ustRepo, lttngBranchesOfInterest)
 353
 354 // For Linux branches, we look for new non-RC tags.
 355 def linuxLastTagIds = GetLastTagIds(linuxRepo, linuxBranchesOfInterest)
 356
 357 // Load previously built Linux tag ids.
 358 println("Loading Git object IDs of previously built projects from the workspace.");
 359 def oldLinuxTags = LoadPreviousIdsFromWorkspace(linuxOnDiskPath) as Set
 360
 361 // Load previously built LTTng commit ids.
 362 def oldToolsHeadCommits = LoadPreviousIdsFromWorkspace(toolsOnDiskPath) as Set
 363 def oldModulesHeadCommits = LoadPreviousIdsFromWorkspace(modulesOnDiskPath) as Set
 364 def oldUstHeadCommits = LoadPreviousIdsFromWorkspace(ustOnDiskPath) as Set
 365
 366 def newOldLinuxTags = oldLinuxTags
 367 def newOldToolsHeadCommits = oldToolsHeadCommits
 368 def newOldModulesHeadCommits = oldModulesHeadCommits
 369 def newOldUstHeadCommits = oldUstHeadCommits
 370
 371 // Canary jobs are run daily to make sure the lava pipeline is working properly.
 372 def canaryRunConfigs = [] as Set
 373 canaryRunConfigs.add(
 374     ['v4.4.9', '1a1a512b983108015ced1e7a7c7775cfeec42d8c', 'v2.8.1','d11e0db', '7fd9215', '514a87f'] as RunConfiguration)
 375
 376 def runConfigs = [] as Set
 377
 378 // For each top of branch kernel tags that were not seen before, schedule one
 379 // job for each lttng/linux tracked configurations.
 380 linuxLastTagIds.each { linuxTag ->
 381   if (!oldLinuxTags.contains(linuxTag.value)) {
 382     lttngBranchesOfInterest.each { lttngBranch ->
 383       if (configurationOfInterest.contains([lttngBranch, linuxTag.key])) {
 384         runConfigs.add([linuxTag.key, linuxTag.value,
 385                     lttngBranch, toolsHeadCommits[lttngBranch],
 386                     modulesHeadCommits[lttngBranch], ustHeadCommits[lttngBranch]]
 387                     as RunConfiguration)
 388
 389         newOldLinuxTags.add(linuxTag.value)
 390       }
 391     }
 392   }
 393 }
 394
 395 // For each top of branch commits of LTTng-Tools that were not seen before,
 396 // schedule one job for each lttng/linux tracked configurations
 397 toolsHeadCommits.each { toolsHead ->
 398   if (!oldToolsHeadCommits.contains(toolsHead.value)) {
 399     linuxLastTagIds.each { linuxTag ->
 400       def lttngBranch = toolsHead.key
 401       if (configurationOfInterest.contains([lttngBranch, linuxTag.key])) {
 402         runConfigs.add([linuxTag.key, linuxTag.value,
 403                     lttngBranch, toolsHeadCommits[lttngBranch],
 404                     modulesHeadCommits[lttngBranch], ustHeadCommits[lttngBranch]]
 405                     as RunConfiguration)
 406
 407         newOldToolsHeadCommits.add(toolsHead.value)
 408       }
 409     }
 410   }
 411 }
 412
 413 // For each top of branch commits of LTTng-Modules that were not seen before,
 414 // schedule one job for each lttng/linux tracked configurations
 415 modulesHeadCommits.each { modulesHead ->
 416   if (!oldModulesHeadCommits.contains(modulesHead.value)) {
 417     linuxLastTagIds.each { linuxTag ->
 418       def lttngBranch = modulesHead.key
 419       if (configurationOfInterest.contains([lttngBranch, linuxTag.key])) {
 420         runConfigs.add([linuxTag.key, linuxTag.value,
 421                     lttngBranch, toolsHeadCommits[lttngBranch],
 422                     modulesHeadCommits[lttngBranch], ustHeadCommits[lttngBranch]]
 423                     as RunConfiguration)
 424
 425         newOldModulesHeadCommits.add(modulesHead.value)
 426       }
 427     }
 428   }
 429 }
 430
 431 // For each top of branch commits of LTTng-UST that were not seen before,
 432 // schedule one job for each lttng/linux tracked configurations
 433 ustHeadCommits.each { ustHead ->
 434   if (!oldUstHeadCommits.contains(ustHead.value)) {
 435     linuxLastTagIds.each { linuxTag ->
 436       def lttngBranch = ustHead.key
 437       if (configurationOfInterest.contains([lttngBranch, linuxTag.key])) {
 438         runConfigs.add([linuxTag.key, linuxTag.value,
 439                     lttngBranch, toolsHeadCommits[lttngBranch],
 440                     modulesHeadCommits[lttngBranch], ustHeadCommits[lttngBranch]]
 441                     as RunConfiguration)
 442
 443         newOldUstHeadCommits.add(ustHead.value)
 444       }
 445     }
 446   }
 447 }
 448
 449 def ongoingBuild = [:]
 450 def failedRuns = []
 451 def abortedRuns = []
 452 def isFailed = false
 453 def isAborted = false
 454
 455 // Check what type of jobs should be triggered.
 456 triggerJobName = build.project.getFullDisplayName();
 457 if (triggerJobName.contains("vm_tests")) {
 458   jobType = 'vm_tests';
 459 } else if (triggerJobName.contains("baremetal_tests")) {
 460   jobType = 'baremetal_tests';
 461 } else if (triggerJobName.contains("baremetal_benchmarks")) {
 462   jobType = 'baremetal_benchmarks';
 463 }
 464
 465 // Launch regular jobs.
 466 if (runConfigs.size() > 0) {
 467   println("\nSchedule jobs triggered by code changes:");
 468   runConfigs.each { config ->
 469     def jobName = CraftJobName(jobType, config);
 470     def currBuild = LaunchJob(jobName, config);
 471
 472     // LaunchJob will return null if the job doesn't exist or is disabled.
 473     if (currBuild != null) {
 474       ongoingBuild[jobName] = currBuild;
 475     }
 476
 477     // Jobs to run only on master branchs of both Linux and LTTng.
 478     if (config.linuxBranch.contains('master') &&
 479         config.lttngBranch.contains('master')) {
 480       // vm_tests specific.
 481       if (jobType.contains("vm_tests")) {
 482         jobName = CraftJobName('vm_tests_fuzzing', config);
 483         currBuild = LaunchJob(jobName, config);
 484
 485         // LaunchJob will return null if the job doesn't exist or is disabled.
 486         if (currBuild != null) {
 487           ongoingBuild[jobName] = currBuild;
 488         }
 489       }
 490     }
 491   }
 492 } else {
 493   println("No new commit or tags, nothing more to do.")
 494 }
 495
 496 // Launch canary jobs.
 497 println("\nSchedule canary jobs once a day:")
 498 canaryRunConfigs.each { config ->
 499   def jobName = jobType + '_canary';
 500   def currBuild = LaunchJob(jobName, config);
 501
 502   // LaunchJob will return null if the job doesn't exist or is disabled.
 503   if (currBuild != null) {
 504     ongoingBuild[jobName] = currBuild;
 505   }
 506 }
 507
 508 // Save the tag and commit IDs scheduled in the past and during this run to the
 509 // workspace. We save it at the end to be sure all jobs were launched. We save
 510 // the object IDs even in case of failure. There is no point of re-running the
 511 // same job is there are no code changes even in case of failure.
 512 println("Saving Git object IDs of previously built projects to the workspace.");
 513 saveCurrentIdsToWorkspace(newOldLinuxTags, linuxOnDiskPath);
 514 saveCurrentIdsToWorkspace(newOldToolsHeadCommits, toolsOnDiskPath);
 515 saveCurrentIdsToWorkspace(newOldModulesHeadCommits, modulesOnDiskPath);
 516 saveCurrentIdsToWorkspace(newOldUstHeadCommits, ustOnDiskPath);
 517
 518 // Iterate over all the running jobs. Record the status of completed jobs.
 519 while (ongoingBuild.size() > 0) {
 520   def ongoingIterator = ongoingBuild.iterator();
 521   while (ongoingIterator.hasNext()) {
 522     currentBuild = ongoingIterator.next();
 523
 524     jobName = currentBuild.getKey();
 525     job_run = currentBuild.getValue();
 526
 527     // The isCancelled() method checks if the run was cancelled before
 528     // execution. We consider such run as being aborted.
 529     if (job_run.isCancelled()) {
 530       println("${jobName} was cancelled before launch.")
 531       abortedRuns.add(jobName);
 532       isAborted = true;
 533       ongoingIterator.remove();
 534     } else if (job_run.isDone()) {
 535
 536       job_status = job_run.get();
 537       println("${job_status.fullDisplayName} completed with status ${job_status.result}.");
 538
 539       // If the job didn't succeed, add its name to the right list so it can
 540       // be printed at the end of the execution.
 541       switch (job_status.result) {
 542       case Result.ABORTED:
 543         isAborted = true;
 544         abortedRuns.add(jobName);
 545         break;
 546       case Result.FAILURE:
 547         isFailed = true;
 548         failedRuns.add(jobName);
 549         break;
 550       case Result.SUCCESS:
 551       default:
 552         break;
 553       }
 554
 555       ongoingIterator.remove();
 556     }
 557   }
 558
 559   // Sleep before the next iteration.
 560   try {
 561     Thread.sleep(30000)
 562   } catch(e) {
 563     if (e in InterruptedException) {
 564       build.setResult(hudson.model.Result.ABORTED)
 565       throw new InterruptedException()
 566     } else {
 567       throw(e)
 568     }
 569   }
 570 }
 571
 572 // Get log of failed runs.
 573 if (failedRuns.size() > 0) {
 574   println("Failed job(s):");
 575   for (failedRun in failedRuns) {
 576     println("\t" + failedRun)
 577   }
 578 }
 579
 580 // Get log of aborted runs.
 581 if (abortedRuns.size() > 0) {
 582   println("Cancelled job(s):");
 583   for (cancelledRun in abortedRuns) {
 584     println("\t" + cancelledRun)
 585   }
 586 }
 587
 588 // Mark this build as Failed if atleast one child build has failed and mark as
 589 // aborted if there was no failure but atleast one job aborted.
 590 if (isFailed) {
 591   build.setResult(hudson.model.Result.FAILURE)
 592 } else if (isAborted) {
 593   build.setResult(hudson.model.Result.ABORTED)
 594 }