| 1 | /** |
| 2 | * Copyright (C) 2017 - Francis Deslauriers <francis.deslauriers@efficios.com> |
| 3 | * |
| 4 | * This program is free software: you can redistribute it and/or modify |
| 5 | * it under the terms of the GNU General Public License as published by |
| 6 | * the Free Software Foundation, either version 3 of the License, or |
| 7 | * (at your option) any later version. |
| 8 | * |
| 9 | * This program is distributed in the hope that it will be useful, |
| 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 12 | * GNU General Public License for more details. |
| 13 | * |
| 14 | * You should have received a copy of the GNU General Public License |
| 15 | * along with this program. If not, see <http://www.gnu.org/licenses/>. |
| 16 | */ |
| 17 | |
| 18 | import hudson.console.HyperlinkNote |
| 19 | import hudson.model.* |
| 20 | import java.io.File |
| 21 | import org.eclipse.jgit.api.Git |
| 22 | import org.eclipse.jgit.lib.Ref |
| 23 | |
| 24 | class InvalidKVersionException extends Exception { |
| 25 | public InvalidKVersionException(String message) { |
| 26 | super(message) |
| 27 | } |
| 28 | } |
| 29 | |
| 30 | class EmptyKVersionException extends Exception { |
| 31 | public EmptyKVersionException(String message) { |
| 32 | super(message) |
| 33 | } |
| 34 | } |
| 35 | |
| 36 | class VanillaKVersion implements Comparable<VanillaKVersion> { |
| 37 | |
| 38 | Integer major = 0 |
| 39 | Integer majorB = 0 |
| 40 | Integer minor = 0 |
| 41 | Integer patch = 0 |
| 42 | Integer rc = Integer.MAX_VALUE |
| 43 | Boolean inStable = false; |
| 44 | |
| 45 | VanillaKVersion() {} |
| 46 | |
| 47 | VanillaKVersion(version) { |
| 48 | this.parse(version) |
| 49 | } |
| 50 | |
| 51 | static VanillaKVersion minKVersion() { |
| 52 | return new VanillaKVersion("v0.0.0") |
| 53 | } |
| 54 | |
| 55 | static VanillaKVersion maxKVersion() { |
| 56 | return new VanillaKVersion("v" + Integer.MAX_VALUE + ".0.0") |
| 57 | } |
| 58 | |
| 59 | static VanillaKVersion factory(version) { |
| 60 | return new VanillaKVersion(version) |
| 61 | } |
| 62 | |
| 63 | def parse(version) { |
| 64 | this.major = 0 |
| 65 | this.majorB = 0 |
| 66 | this.minor = 0 |
| 67 | this.patch = 0 |
| 68 | this.rc = Integer.MAX_VALUE |
| 69 | |
| 70 | if (!version) { |
| 71 | throw new EmptyKVersionException("Empty kernel version") |
| 72 | } |
| 73 | |
| 74 | def match = version =~ /^v(\d+)\.(\d+)(\.(\d+))?(\.(\d+))?(-rc(\d+))?$/ |
| 75 | if (!match) { |
| 76 | throw new InvalidKVersionException("Invalid kernel version: ${version}") |
| 77 | } |
| 78 | |
| 79 | Integer offset = 0; |
| 80 | |
| 81 | // Major |
| 82 | this.major = Integer.parseInt(match.group(1)) |
| 83 | if (this.major <= 2) { |
| 84 | offset = 2 |
| 85 | this.majorB = Integer.parseInt(match.group(2)) |
| 86 | } |
| 87 | |
| 88 | // Minor |
| 89 | if (match.group(2 + offset) != null) { |
| 90 | this.minor = Integer.parseInt(match.group(2 + offset)) |
| 91 | } |
| 92 | |
| 93 | // Patch level |
| 94 | if (match.group(4 + offset) != null) { |
| 95 | this.patch = Integer.parseInt(match.group(4 + offset)) |
| 96 | this.inStable = true |
| 97 | } |
| 98 | |
| 99 | // RC |
| 100 | if (match.group(8) != null) { |
| 101 | this.rc = Integer.parseInt(match.group(8)) |
| 102 | } |
| 103 | } |
| 104 | |
| 105 | Boolean isInStableBranch() { |
| 106 | return this.inStable |
| 107 | } |
| 108 | |
| 109 | // Return true if both version are of the same stable branch |
| 110 | Boolean isSameStable(VanillaKVersion o) { |
| 111 | if (this.major != o.major) { |
| 112 | return false |
| 113 | } |
| 114 | if (this.majorB != o.majorB) { |
| 115 | return false |
| 116 | } |
| 117 | if (this.minor != o.minor) { |
| 118 | return false |
| 119 | } |
| 120 | |
| 121 | return true |
| 122 | } |
| 123 | |
| 124 | @Override int compareTo(VanillaKVersion o) { |
| 125 | if (this.major != o.major) { |
| 126 | return Integer.compare(this.major, o.major) |
| 127 | } |
| 128 | if (this.majorB != o.majorB) { |
| 129 | return Integer.compare(this.majorB, o.majorB) |
| 130 | } |
| 131 | if (this.minor != o.minor) { |
| 132 | return Integer.compare(this.minor, o.minor) |
| 133 | } |
| 134 | if (this.patch != o.patch) { |
| 135 | return Integer.compare(this.patch, o.patch) |
| 136 | } |
| 137 | if (this.rc != o.rc) { |
| 138 | return Integer.compare(this.rc, o.rc) |
| 139 | } |
| 140 | |
| 141 | // Same version |
| 142 | return 0; |
| 143 | } |
| 144 | |
| 145 | String toString() { |
| 146 | String vString = "v${this.major}" |
| 147 | |
| 148 | if (this.majorB > 0) { |
| 149 | vString = vString.concat(".${this.majorB}") |
| 150 | } |
| 151 | |
| 152 | vString = vString.concat(".${this.minor}") |
| 153 | |
| 154 | if (this.patch > 0) { |
| 155 | vString = vString.concat(".${this.patch}") |
| 156 | } |
| 157 | |
| 158 | if (this.rc > 0 && this.rc < Integer.MAX_VALUE) { |
| 159 | vString = vString.concat("-rc${this.rc}") |
| 160 | } |
| 161 | return vString |
| 162 | } |
| 163 | } |
| 164 | |
| 165 | class RunConfiguration { |
| 166 | def linuxBranch |
| 167 | def linuxTagId |
| 168 | def lttngBranch |
| 169 | def lttngModulesCommitId |
| 170 | def lttngToolsCommitId |
| 171 | def lttngUstCommitId |
| 172 | RunConfiguration(linuxBranch, linuxTagId, lttngBranch, lttngToolsCommitId, |
| 173 | lttngModulesCommitId, lttngUstCommitId) { |
| 174 | this.linuxBranch = linuxBranch |
| 175 | this.linuxTagId = linuxTagId |
| 176 | this.lttngBranch = lttngBranch |
| 177 | this.lttngModulesCommitId = lttngModulesCommitId |
| 178 | this.lttngToolsCommitId = lttngToolsCommitId |
| 179 | this.lttngUstCommitId = lttngUstCommitId |
| 180 | } |
| 181 | |
| 182 | String toString() { |
| 183 | return "${this.linuxBranch}:{${this.linuxTagId}}, ${this.lttngBranch}" + |
| 184 | ":{${this.lttngModulesCommitId}, ${this.lttngToolsCommitId}," + |
| 185 | "${this.lttngUstCommitId}}" |
| 186 | } |
| 187 | } |
| 188 | |
| 189 | def LoadPreviousIdsFromWorkspace = { ondiskpath -> |
| 190 | def previousIds = [] |
| 191 | try { |
| 192 | File myFile = new File(ondiskpath); |
| 193 | def input = new ObjectInputStream(new FileInputStream(ondiskpath)) |
| 194 | previousIds = input.readObject() |
| 195 | input.close() |
| 196 | } catch (all) { |
| 197 | println("Failed to load previous ids from disk.") |
| 198 | } |
| 199 | return previousIds |
| 200 | } |
| 201 | |
| 202 | def saveCurrentIdsToWorkspace = { currentIds, ondiskpath -> |
| 203 | try { |
| 204 | File myFile = new File(ondiskpath); |
| 205 | myFile.createNewFile(); |
| 206 | def out = new ObjectOutputStream(new FileOutputStream(ondiskpath)) |
| 207 | out.writeObject(currentIds) |
| 208 | out.close() |
| 209 | } catch (all) { |
| 210 | println("Failed to save previous ids from disk.") |
| 211 | } |
| 212 | } |
| 213 | |
| 214 | def GetHeadCommits = { remoteRepo, branchesOfInterest -> |
| 215 | def remoteHeads = [:] |
| 216 | def remoteHeadRefs = Git.lsRemoteRepository() |
| 217 | .setTags(false) |
| 218 | .setHeads(true) |
| 219 | .setRemote(remoteRepo).call() |
| 220 | |
| 221 | remoteHeadRefs.each { |
| 222 | def branch = it.getName().replaceAll('refs/heads/', '') |
| 223 | if (branchesOfInterest.contains(branch)) |
| 224 | remoteHeads[branch] = it.getObjectId().name() |
| 225 | } |
| 226 | |
| 227 | return remoteHeads |
| 228 | } |
| 229 | |
| 230 | def GetTagIds = { remoteRepo -> |
| 231 | def remoteTags = [:] |
| 232 | def remoteTagRefs = Git.lsRemoteRepository() |
| 233 | .setTags(true) |
| 234 | .setHeads(false) |
| 235 | .setRemote(remoteRepo).call() |
| 236 | |
| 237 | remoteTagRefs.each { |
| 238 | // Exclude release candidate tags |
| 239 | if (!it.getName().contains('-rc')) { |
| 240 | remoteTags[it.getName().replaceAll('refs/tags/', '')] = it.getObjectId().name() |
| 241 | } |
| 242 | } |
| 243 | |
| 244 | return remoteTags |
| 245 | } |
| 246 | |
| 247 | def GetLastTagOfBranch = { tagRefs, branch -> |
| 248 | def tagVersions = tagRefs.collect {new VanillaKVersion(it.key)} |
| 249 | def currMax = new VanillaKVersion('v0.0.0'); |
| 250 | if (!branch.contains('master')){ |
| 251 | def targetVersion = new VanillaKVersion(branch.replaceAll('linux-', 'v').replaceAll('.y', '')) |
| 252 | tagVersions.each { |
| 253 | if (it.isSameStable(targetVersion)) { |
| 254 | if (currMax < it) { |
| 255 | currMax = it; |
| 256 | } |
| 257 | } |
| 258 | } |
| 259 | } else { |
| 260 | tagVersions.each { |
| 261 | if (!it.isInStableBranch() && currMax < it) { |
| 262 | currMax = it; |
| 263 | } |
| 264 | } |
| 265 | } |
| 266 | return currMax.toString() |
| 267 | } |
| 268 | |
| 269 | // Returns the latest tags of each of the branches passed in the argument |
| 270 | def GetLastTagIds = { remoteRepo, branchesOfInterest -> |
| 271 | def remoteHeads = GetHeadCommits(remoteRepo, branchesOfInterest) |
| 272 | def remoteTagRefs = GetTagIds(remoteRepo) |
| 273 | def remoteLastTagCommit = [:] |
| 274 | |
| 275 | remoteTagRefs = remoteTagRefs.findAll { !it.key.contains("v2.") } |
| 276 | branchesOfInterest.each { |
| 277 | remoteLastTagCommit[it] = remoteTagRefs[GetLastTagOfBranch(remoteTagRefs, it)] |
| 278 | } |
| 279 | |
| 280 | return remoteLastTagCommit |
| 281 | } |
| 282 | |
| 283 | def CraftJobName = { jobType, runConfig -> |
| 284 | return "${jobType}_k${runConfig.linuxBranch}_l${runConfig.lttngBranch}" |
| 285 | } |
| 286 | |
| 287 | def LaunchJob = { jobName, runConfig -> |
| 288 | def job = Hudson.instance.getJob(jobName) |
| 289 | def params = [] |
| 290 | for (paramdef in job.getProperty(ParametersDefinitionProperty.class).getParameterDefinitions()) { |
| 291 | params += paramdef.getDefaultParameterValue(); |
| 292 | } |
| 293 | |
| 294 | params.add(new StringParameterValue('tools_commit_id', runConfig.lttngToolsCommitId)) |
| 295 | params.add(new StringParameterValue('modules_commit_id', runConfig.lttngModulesCommitId)) |
| 296 | params.add(new StringParameterValue('ust_commit_id', runConfig.lttngUstCommitId)) |
| 297 | params.add(new StringParameterValue('kernel_tag_id', runConfig.linuxTagId)) |
| 298 | job.scheduleBuild2(0, new Cause.UpstreamCause(build), new ParametersAction(params)) |
| 299 | println "Launching job: ${HyperlinkNote.encodeTo('/' + job.url, job.fullDisplayName)}" |
| 300 | } |
| 301 | |
| 302 | def jobTypes = ['baremetal_tests', 'vm_tests', 'baremetal_benchmarks'] |
| 303 | final String toolsRepo = "https://github.com/lttng/lttng-tools.git" |
| 304 | final String modulesRepo = "https://github.com/lttng/lttng-modules.git" |
| 305 | final String ustRepo = "https://github.com/lttng/lttng-ust.git" |
| 306 | final String linuxRepo = "git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git" |
| 307 | |
| 308 | final String toolsOnDiskPath = build.getEnvironment(listener).get('WORKSPACE') + "/on-disk-tools-ref" |
| 309 | final String modulesOnDiskPath = build.getEnvironment(listener).get('WORKSPACE') + "/on-disk-modules-ref" |
| 310 | final String ustOnDiskPath = build.getEnvironment(listener).get('WORKSPACE') + "/on-disk-ust-ref" |
| 311 | final String linuxOnDiskPath = build.getEnvironment(listener).get('WORKSPACE') + "/on-disk-linux-ref" |
| 312 | |
| 313 | def recentLttngBranchesOfInterest = ['master', 'stable-2.10', 'stable-2.9'] |
| 314 | def recentLinuxBranchesOfInterest = ['master', 'linux-4.9.y', 'linux-4.4.y'] |
| 315 | |
| 316 | def legacyLttngBranchesOfInterest = ['stable-2.7'] |
| 317 | def legacyLinuxBranchesOfInterest = ['linux-3.18.y', 'linux-4.4.y'] |
| 318 | |
| 319 | // Generate configurations of interest |
| 320 | def configurationOfInterest = [] as Set |
| 321 | |
| 322 | recentLttngBranchesOfInterest.each { lttngBranch -> |
| 323 | recentLinuxBranchesOfInterest.each { linuxBranch -> |
| 324 | configurationOfInterest.add([lttngBranch, linuxBranch]) |
| 325 | } |
| 326 | } |
| 327 | |
| 328 | legacyLttngBranchesOfInterest.each { lttngBranch -> |
| 329 | legacyLinuxBranchesOfInterest.each { linuxBranch -> |
| 330 | configurationOfInterest.add([lttngBranch, linuxBranch]) |
| 331 | } |
| 332 | } |
| 333 | |
| 334 | def lttngBranchesOfInterest = recentLttngBranchesOfInterest + legacyLttngBranchesOfInterest |
| 335 | def linuxBranchesOfInterest = recentLinuxBranchesOfInterest + legacyLinuxBranchesOfInterest |
| 336 | |
| 337 | // For Linux branches, we look for new non-RC tags |
| 338 | def toolsHeadCommits = GetHeadCommits(toolsRepo, lttngBranchesOfInterest) |
| 339 | def modulesHeadCommits = GetHeadCommits(modulesRepo, lttngBranchesOfInterest) |
| 340 | def ustHeadCommits = GetHeadCommits(ustRepo, lttngBranchesOfInterest) |
| 341 | |
| 342 | // For LTTng branches, we look for new commits |
| 343 | def linuxLastTagIds = GetLastTagIds(linuxRepo, linuxBranchesOfInterest) |
| 344 | |
| 345 | // Load previously build Linux tag ids |
| 346 | def oldLinuxTags = LoadPreviousIdsFromWorkspace(linuxOnDiskPath) as Set |
| 347 | |
| 348 | // Load previously built LTTng commit ids |
| 349 | def oldToolsHeadCommits = LoadPreviousIdsFromWorkspace(toolsOnDiskPath) as Set |
| 350 | def oldModulesHeadCommits = LoadPreviousIdsFromWorkspace(modulesOnDiskPath) as Set |
| 351 | def oldUstHeadCommits = LoadPreviousIdsFromWorkspace(ustOnDiskPath) as Set |
| 352 | |
| 353 | def newOldLinuxTags = oldLinuxTags |
| 354 | def newOldToolsHeadCommits = oldToolsHeadCommits |
| 355 | def newOldModulesHeadCommits = oldModulesHeadCommits |
| 356 | def newOldUstHeadCommits = oldUstHeadCommits |
| 357 | |
| 358 | def canaryRunConfigs = [] as Set |
| 359 | canaryRunConfigs.add( |
| 360 | ['v4.4.9', '1a1a512b983108015ced1e7a7c7775cfeec42d8c', 'v2.8.1','d11e0db', '7fd9215', '514a87f'] as RunConfiguration) |
| 361 | |
| 362 | def runConfigs = [] as Set |
| 363 | |
| 364 | // For each top of branch kernel tags that were not seen before, schedule one |
| 365 | // job for each lttng/linux tracked configurations |
| 366 | linuxLastTagIds.each { linuxTag -> |
| 367 | if (!oldLinuxTags.contains(linuxTag.value)) { |
| 368 | lttngBranchesOfInterest.each { lttngBranch -> |
| 369 | if (configurationOfInterest.contains([lttngBranch, linuxTag.key])) { |
| 370 | runConfigs.add([linuxTag.key, linuxTag.value, |
| 371 | lttngBranch, toolsHeadCommits[lttngBranch], |
| 372 | modulesHeadCommits[lttngBranch], ustHeadCommits[lttngBranch]] |
| 373 | as RunConfiguration) |
| 374 | |
| 375 | newOldLinuxTags.add(linuxTag.value) |
| 376 | } |
| 377 | } |
| 378 | } |
| 379 | } |
| 380 | |
| 381 | // For each top of branch commits of LTTng-Tools that were not seen before, |
| 382 | // schedule one job for each lttng/linux tracked configurations |
| 383 | toolsHeadCommits.each { toolsHead -> |
| 384 | if (!oldToolsHeadCommits.contains(toolsHead.value)) { |
| 385 | linuxLastTagIds.each { linuxTag -> |
| 386 | def lttngBranch = toolsHead.key |
| 387 | if (configurationOfInterest.contains([lttngBranch, linuxTag.key])) { |
| 388 | runConfigs.add([linuxTag.key, linuxTag.value, |
| 389 | lttngBranch, toolsHeadCommits[lttngBranch], |
| 390 | modulesHeadCommits[lttngBranch], ustHeadCommits[lttngBranch]] |
| 391 | as RunConfiguration) |
| 392 | |
| 393 | newOldToolsHeadCommits.add(toolsHead.value) |
| 394 | } |
| 395 | } |
| 396 | } |
| 397 | } |
| 398 | |
| 399 | // For each top of branch commits of LTTng-Modules that were not seen before, |
| 400 | // schedule one job for each lttng/linux tracked configurations |
| 401 | modulesHeadCommits.each { modulesHead -> |
| 402 | if (!oldModulesHeadCommits.contains(modulesHead.value)) { |
| 403 | linuxLastTagIds.each { linuxTag -> |
| 404 | def lttngBranch = modulesHead.key |
| 405 | if (configurationOfInterest.contains([lttngBranch, linuxTag.key])) { |
| 406 | runConfigs.add([linuxTag.key, linuxTag.value, |
| 407 | lttngBranch, toolsHeadCommits[lttngBranch], |
| 408 | modulesHeadCommits[lttngBranch], ustHeadCommits[lttngBranch]] |
| 409 | as RunConfiguration) |
| 410 | |
| 411 | newOldModulesHeadCommits.add(modulesHead.value) |
| 412 | } |
| 413 | } |
| 414 | } |
| 415 | } |
| 416 | |
| 417 | // For each top of branch commits of LTTng-UST that were not seen before, |
| 418 | // schedule one job for each lttng/linux tracked configurations |
| 419 | ustHeadCommits.each { ustHead -> |
| 420 | if (!oldUstHeadCommits.contains(ustHead.value)) { |
| 421 | linuxLastTagIds.each { linuxTag -> |
| 422 | def lttngBranch = ustHead.key |
| 423 | if (configurationOfInterest.contains([lttngBranch, linuxTag.key])) { |
| 424 | runConfigs.add([linuxTag.key, linuxTag.value, |
| 425 | lttngBranch, toolsHeadCommits[lttngBranch], |
| 426 | modulesHeadCommits[lttngBranch], ustHeadCommits[lttngBranch]] |
| 427 | as RunConfiguration) |
| 428 | |
| 429 | newOldUstHeadCommits.add(ustHead.value) |
| 430 | } |
| 431 | } |
| 432 | } |
| 433 | } |
| 434 | |
| 435 | // Save the tag and commit IDs scheduled in the past and during this run to the |
| 436 | // workspace |
| 437 | saveCurrentIdsToWorkspace(newOldLinuxTags, linuxOnDiskPath) |
| 438 | saveCurrentIdsToWorkspace(newOldToolsHeadCommits, toolsOnDiskPath) |
| 439 | saveCurrentIdsToWorkspace(newOldModulesHeadCommits, modulesOnDiskPath) |
| 440 | saveCurrentIdsToWorkspace(newOldUstHeadCommits, ustOnDiskPath) |
| 441 | |
| 442 | // Launch jobs |
| 443 | println("Schedule canary jobs once a day") |
| 444 | canaryRunConfigs.each { config -> |
| 445 | jobTypes.each { type -> |
| 446 | LaunchJob(type + '_canary', config) |
| 447 | } |
| 448 | } |
| 449 | |
| 450 | if (runConfigs.size() > 0) { |
| 451 | println("Schedule jobs because of code changes.") |
| 452 | runConfigs.each { config -> |
| 453 | jobTypes.each { type -> |
| 454 | LaunchJob(CraftJobName(type, config), config); |
| 455 | } |
| 456 | |
| 457 | // Jobs to run only on master branchs of both linux and lttng |
| 458 | if (config.linuxBranch.contains('master') && |
| 459 | config.lttngBranch.contains('master')) { |
| 460 | LaunchJob(CraftJobName('vm_tests_fuzzing', config), config) |
| 461 | } |
| 462 | } |
| 463 | } else { |
| 464 | println("No new commit or tags, nothing more to do.") |
| 465 | } |