| 1 | enum KernelVersioning { |
| 2 | MAJOR,MINOR,REVISION,BUILD |
| 3 | } |
| 4 | |
| 5 | class BasicVersion implements Comparable<BasicVersion> { |
| 6 | int major = -1 |
| 7 | int minor = -1 |
| 8 | int revision = -1 |
| 9 | int build = -1 |
| 10 | int rc = -1 |
| 11 | String gitRefs |
| 12 | |
| 13 | // Default Constructor |
| 14 | BasicVersion() {} |
| 15 | |
| 16 | // Parse a version string of format X.Y.Z.W-A |
| 17 | BasicVersion(String version, String ref) { |
| 18 | gitRefs = ref |
| 19 | def tokenVersion |
| 20 | def token |
| 21 | if (version.contains('-')) { |
| 22 | // Release canditate |
| 23 | token = version.tokenize('-') |
| 24 | tokenVersion = token[0] |
| 25 | if (token[1]?.isInteger()) { |
| 26 | rc = token[1].toInteger() |
| 27 | } |
| 28 | } else { |
| 29 | tokenVersion = version |
| 30 | } |
| 31 | |
| 32 | tokenVersion = tokenVersion.tokenize('.') |
| 33 | |
| 34 | def tagEnum = KernelVersioning.MAJOR |
| 35 | tokenVersion.each { |
| 36 | if (it?.isInteger()) { |
| 37 | switch (tagEnum) { |
| 38 | case KernelVersioning.MAJOR: |
| 39 | major = it.toInteger() |
| 40 | tagEnum = KernelVersioning.MINOR |
| 41 | break |
| 42 | case KernelVersioning.MINOR: |
| 43 | minor = it.toInteger() |
| 44 | tagEnum = KernelVersioning.REVISION |
| 45 | break |
| 46 | case KernelVersioning.REVISION: |
| 47 | revision = it.toInteger() |
| 48 | tagEnum = KernelVersioning.BUILD |
| 49 | break |
| 50 | case KernelVersioning.BUILD: |
| 51 | build = it.toInteger() |
| 52 | tagEnum = -1 |
| 53 | break |
| 54 | default: |
| 55 | println("Unsupported version extension") |
| 56 | println("Trying to parse: ${version}") |
| 57 | println("Invalid sub version value: ${it}") |
| 58 | //TODO: throw exception for jenkins |
| 59 | } |
| 60 | } |
| 61 | } |
| 62 | } |
| 63 | |
| 64 | String print() { |
| 65 | String ret = "" |
| 66 | if (major != -1) { |
| 67 | ret += major |
| 68 | if (minor != -1) { |
| 69 | ret += "." + minor |
| 70 | if (revision != -1) { |
| 71 | ret += "." + revision |
| 72 | if (build != -1) { |
| 73 | ret += "." + build |
| 74 | } |
| 75 | } |
| 76 | } |
| 77 | if (rc != -1) { |
| 78 | ret += "-rc" + rc |
| 79 | } |
| 80 | } |
| 81 | return ret |
| 82 | } |
| 83 | |
| 84 | @Override |
| 85 | int compareTo(BasicVersion kernelVersion) { |
| 86 | return major <=> kernelVersion.major ?: minor <=> kernelVersion.minor ?: revision <=> kernelVersion.revision ?: build <=> kernelVersion.build ?: rc <=> kernelVersion.rc |
| 87 | } |
| 88 | } |
| 89 | |
| 90 | def kernelTagCutOff = new BasicVersion("2.6.36", "") |
| 91 | def modulesBranches = ["master", "stable-2.5", "stable-2.6"] |
| 92 | |
| 93 | //def modulesBranches = ["master","stable-2.5","stable-2.6", "stable-2.4"] |
| 94 | |
| 95 | def linuxURL = "git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git" |
| 96 | def modulesURL = "https://github.com/lttng/lttng-modules.git" |
| 97 | |
| 98 | // Linux specific variable |
| 99 | String linuxCheckoutTo = "linux-source" |
| 100 | String recipeCheckoutTo = "recipe" |
| 101 | String modulesCheckoutTo = "lttng-modules" |
| 102 | |
| 103 | def linuxGitReference = "/home/jenkins/gitcache/linux-stable.git" |
| 104 | |
| 105 | // Check if we are on jenkins |
| 106 | // Useful for outside jenkins devellopment related to groovy only scripting |
| 107 | def isJenkinsInstance = binding.variables.containsKey('JENKINS_HOME') |
| 108 | |
| 109 | // Fetch tags and format |
| 110 | // Split the string into sections based on | |
| 111 | // And pipe the results together |
| 112 | String process = "git ls-remote -t $linuxURL | cut -c42- | sort -V" |
| 113 | def out = new StringBuilder() |
| 114 | def err = new StringBuilder() |
| 115 | Process result = process.tokenize( '|' ).inject( null ) { p, c -> |
| 116 | if( p ) |
| 117 | p | c.execute() |
| 118 | else |
| 119 | c.execute() |
| 120 | } |
| 121 | |
| 122 | result.waitForProcessOutput(out,err) |
| 123 | |
| 124 | if ( result.exitValue() == 0 ) { |
| 125 | def branches = out.readLines().collect { |
| 126 | // Scrap special string tag |
| 127 | it.replaceAll("\\^\\{\\}", '') |
| 128 | } |
| 129 | |
| 130 | branches = branches.unique() |
| 131 | |
| 132 | List versions = [] |
| 133 | branches.each { branch -> |
| 134 | def stripBranch = branch.replaceAll("rc", '').replaceAll(/refs\/tags\/v/,'') |
| 135 | BasicVersion kVersion = new BasicVersion(stripBranch, branch) |
| 136 | versions.add(kVersion) |
| 137 | } |
| 138 | |
| 139 | // Sort the version via Comparable implementation of KernelVersion |
| 140 | versions = versions.sort() |
| 141 | |
| 142 | // Find the version cutoff |
| 143 | def cutoffPos = versions.findIndexOf{(it.major >= kernelTagCutOff.major) && (it.minor >= kernelTagCutOff.minor) && (it.revision >= kernelTagCutOff.revision) && (it.build >= kernelTagCutOff.build) && (it.rc >= kernelTagCutOff.rc)} |
| 144 | |
| 145 | // If error set cutoff on last so no job are created |
| 146 | if (cutoffPos == -1) { |
| 147 | cutoffPos = versions.size() |
| 148 | } |
| 149 | // Get last version and include only last rc |
| 150 | def last |
| 151 | def lastNoRcPos |
| 152 | last = versions.last() |
| 153 | if (last.rc != -1) { |
| 154 | int i = versions.size()-1 |
| 155 | while (i > -1 && versions[i].rc != -1 ) { |
| 156 | i-- |
| 157 | } |
| 158 | lastNoRcPos = i + 1 |
| 159 | } else { |
| 160 | lastNoRcPos = versions.size() |
| 161 | } |
| 162 | |
| 163 | String modulesPrefix = "lttng-modules" |
| 164 | String kernelPrefix = "dsl-kernel" |
| 165 | String separator = "-" |
| 166 | |
| 167 | |
| 168 | println("CutOff index") |
| 169 | println(cutoffPos) |
| 170 | |
| 171 | |
| 172 | // Actual job creation |
| 173 | for (int i = cutoffPos; i < versions.size() ; i++) { |
| 174 | |
| 175 | // Only create for valid build |
| 176 | if ( (i < lastNoRcPos && versions[i].rc == -1) || (i >= lastNoRcPos)) { |
| 177 | println ("Preparing job for") |
| 178 | |
| 179 | String jobName = kernelPrefix + separator + versions[i].print() |
| 180 | |
| 181 | // Generate modules job based on supported modules jobs |
| 182 | def modulesJob = [:] |
| 183 | modulesBranches.each { branch -> |
| 184 | modulesJob[branch] = modulesPrefix + separator + branch + separator + jobName |
| 185 | } |
| 186 | |
| 187 | // Jenkins only dsl |
| 188 | println(jobName) |
| 189 | if (isJenkinsInstance) { |
| 190 | matrixJob("${jobName}") { |
| 191 | using("linux-master") |
| 192 | scm { |
| 193 | git { |
| 194 | remote { |
| 195 | url("${linuxURL}") |
| 196 | } |
| 197 | branch(versions[i].gitRefs) |
| 198 | shallowClone(true) |
| 199 | relativeTargetDir(linuxCheckoutTo) |
| 200 | reference(linuxGitReference) |
| 201 | } |
| 202 | } |
| 203 | publishers { |
| 204 | modulesJob.each { |
| 205 | downstream(it.value, 'SUCCESS') |
| 206 | } |
| 207 | } |
| 208 | } |
| 209 | } |
| 210 | // Corresponding Module job |
| 211 | modulesJob.each { job -> |
| 212 | println("\t" + job.key + " " + job.value) |
| 213 | if (isJenkinsInstance) { |
| 214 | matrixJob(job.value) { |
| 215 | using("modules") |
| 216 | multiscm { |
| 217 | git { |
| 218 | remote { |
| 219 | name(kernelPrefix) |
| 220 | url("${linuxURL}") |
| 221 | } |
| 222 | branch(versions[i].gitRefs) |
| 223 | shallowClone(true) |
| 224 | relativeTargetDir(linuxCheckoutTo) |
| 225 | reference(linuxGitReference) |
| 226 | } |
| 227 | git { |
| 228 | remote { |
| 229 | name(modulesPrefix) |
| 230 | url(modulesURL) |
| 231 | } |
| 232 | branch(job.key) |
| 233 | relativeTargetDir(modulesCheckoutTo) |
| 234 | } |
| 235 | } |
| 236 | steps { |
| 237 | copyArtifacts("${jobName}/arch=\$arch,label=kernel", "linux-artifact/**", '', false, false) { |
| 238 | latestSuccessful(true) // Latest successful build |
| 239 | } |
| 240 | shell(readFileFromWorkspace('lttng-modules/lttng-modules-dsl-master.sh')) |
| 241 | } |
| 242 | } |
| 243 | } |
| 244 | } |
| 245 | } |
| 246 | } |
| 247 | |
| 248 | // Trigger generations |
| 249 | def dslTriggerKernel = """\ |
| 250 | import hudson.model.* |
| 251 | import jenkins.model.* |
| 252 | import hudson.AbortException |
| 253 | import hudson.console.HyperlinkNote |
| 254 | import java.util.concurrent.CancellationException |
| 255 | import java.util.Random |
| 256 | |
| 257 | |
| 258 | Random random = new Random() |
| 259 | def jobs = hudson.model.Hudson.instance.items |
| 260 | def fail = false |
| 261 | def jobStartWithKernel = "KERNELPREFIX" |
| 262 | def jobStartWithModule = "MODULEPREFIX" |
| 263 | def toBuild = [] |
| 264 | def counter = 0 |
| 265 | def limitQueue = 4 |
| 266 | |
| 267 | def anotherBuild |
| 268 | jobs.each { job -> |
| 269 | def jobName = job.getName() |
| 270 | if (jobName.startsWith(jobStartWithKernel)) { |
| 271 | counter = counter + 1 |
| 272 | def lastBuild = job.getLastBuild() |
| 273 | if (lastBuild == null || lastBuild.result != Result.SUCCESS) { |
| 274 | toBuild.push(job) |
| 275 | } else { |
| 276 | println("\tAlready built") |
| 277 | } |
| 278 | } |
| 279 | } |
| 280 | |
| 281 | println "Kernel total "+ counter |
| 282 | println "Kernel to build "+ toBuild.size() |
| 283 | |
| 284 | |
| 285 | def kernelEnabledNode = 0 |
| 286 | hudson.model.Hudson.instance.nodes.each { node -> |
| 287 | if (node.getLabelString().contains("kernel")){ |
| 288 | kernelEnabledNode++ |
| 289 | } |
| 290 | } |
| 291 | println "Nb of live kernel enabled build node "+ kernelEnabledNode |
| 292 | |
| 293 | def ongoingBuild = [] |
| 294 | def q = jenkins.model.Jenkins.getInstance().getQueue() |
| 295 | |
| 296 | def queuedTaskKernel = 0 |
| 297 | def queuedTaskModule = 0 |
| 298 | |
| 299 | while (toBuild.size() != 0) { |
| 300 | // Throttle the build with both the number of current parent task and queued |
| 301 | // task.Look for both kernel and downstream module from previous kernel. |
| 302 | queuedTaskKernel = q.getItems().findAll { |
| 303 | it.task.getParent().name.startsWith(jobStartWithKernel) |
| 304 | }.size() |
| 305 | |
| 306 | queuedTaskModule = q.getItems().findAll { |
| 307 | it.task.getParent().name.startsWith(jobStartWithModule) |
| 308 | }.size() |
| 309 | |
| 310 | it.task.getParent().name.startsWith(jobStartWithModule) |
| 311 | if ((ongoingBuild.size() <= kernelEnabledNode.intdiv(2)) && (queuedTaskKernel + queuedTaskModule < limitQueue)) { |
| 312 | def job = toBuild.pop() |
| 313 | ongoingBuild.push(job.scheduleBuild2(0)) |
| 314 | println "\t trigering " + HyperlinkNote.encodeTo('/' + job.url, job.fullDisplayName) |
| 315 | } else { |
| 316 | println "Currently " + ongoingBuild.size() + " build currently on execution. Limit: " + kernelEnabledNode.intdiv(2) |
| 317 | println "Currently " + queuedTask.findAll{it.task.getParent().name.startsWith(jobStartWithModule)}.size() + " module jobs are queued. Limit: " + limitQueue |
| 318 | println "Currently " + queuedTask.findAll{it.task.getParent().name.startsWith(jobStartWithKernel)}.size() + " kernel jobs are queued. Limit: " + limitQueue |
| 319 | Thread.sleep(random.nextInt(60000)) |
| 320 | ongoingBuild.removeAll{ it.isCancelled() || it.isDone() } |
| 321 | } |
| 322 | } |
| 323 | |
| 324 | if (fail){ |
| 325 | throw new AbortException("Some job failed") |
| 326 | } |
| 327 | """ |
| 328 | def dslTriggerModule = """\ |
| 329 | import hudson.model.* |
| 330 | import hudson.AbortException |
| 331 | import hudson.console.HyperlinkNote |
| 332 | import java.util.concurrent.CancellationException |
| 333 | import java.util.Random |
| 334 | |
| 335 | |
| 336 | Random random = new Random() |
| 337 | def jobs = hudson.model.Hudson.instance.items |
| 338 | def fail = false |
| 339 | def modulePrefix = "MODULEPREFIX" |
| 340 | def branchName = "BRANCHNAME" |
| 341 | def kernelPrefix = "KERNELPREFIX" |
| 342 | def nodeLabels=["kernel"] |
| 343 | def validNodeDivider = 2 |
| 344 | |
| 345 | def fullModulePrefix = modulesPrefix + branchName |
| 346 | |
| 347 | def toBuild = [] |
| 348 | def counter = 0 |
| 349 | def limitQueue = 4 |
| 350 | |
| 351 | jobs.each { job -> |
| 352 | def jobName = job.getName() |
| 353 | if (jobName.startsWith(fullModulePrefix)) { |
| 354 | counter = counter + 1 |
| 355 | toBuild.push(job) |
| 356 | } |
| 357 | } |
| 358 | |
| 359 | // Get valid labeled node node |
| 360 | def validNodeCount = 0 |
| 361 | hudson.model.Hudson.instance.nodes.each { node -> |
| 362 | def valid = true |
| 363 | nodeLabels.each { label -> |
| 364 | if (!node.getLabelString().contains(nodeLabel)){ |
| 365 | valid = false |
| 366 | break; |
| 367 | } |
| 368 | } |
| 369 | if (valid){ |
| 370 | validNodeCount++ |
| 371 | } |
| 372 | } |
| 373 | |
| 374 | // Divide the valid node by validNodeDivider based on user defined label slave descriminant ex arck type |
| 375 | def finalValidNodeCount = validNodeCount.intdiv(validNodeDivider |
| 376 | |
| 377 | // Scheduling |
| 378 | |
| 379 | def ongoingBuild = [] |
| 380 | def q = jenkins.model.Jenkins.getInstance().getQueue() |
| 381 | def queuedTaskKernel = 0 |
| 382 | def queuedTaskModule = 0 |
| 383 | def sleep = 0 |
| 384 | |
| 385 | while (toBuild.size() != 0) { |
| 386 | // Throttle the build with both the number of current parent task and queued |
| 387 | // task.Look for both kernel and downstream module from previous kernel. |
| 388 | queuedTaskKernel = q.getItems().findAll {it.task.getParent().getDisplayName().startsWith(jobStartWithKernel)}.size() |
| 389 | queuedTaskModule = q.getItems().findAll {it.task.getParent().getDisplayName().startsWith(jobStartWithModule)}.size() |
| 390 | if ((ongoingBuild.size() <= finalValidNodeCount) && (queuedTaskKernel + queuedTaskModule < limitQueue)) { |
| 391 | def job = toBuild.pop() |
| 392 | ongoingBuild.push(job.scheduleBuild2(0)) |
| 393 | println "\t trigering " + HyperlinkNote.encodeTo('/' + job.url, job.fullDisplayName) |
| 394 | } else { |
| 395 | println "Holding trigger" |
| 396 | println "Currently " + ongoingBuild.size() + " build ongoing. Max = " + validNodeCount |
| 397 | println "Currently " + queuedTaskKernel + " Kernel build ongoing." |
| 398 | println "Currently " + queuedTaskModule + " LTTng-modules build ongoing." |
| 399 | println "Limit for combination of both:" + limitQueue |
| 400 | |
| 401 | sleep = random.nextInt(60000) |
| 402 | println "Sleeping for " + sleep.intdiv(1000) + " seconds" |
| 403 | Thread.sleep(sleep) |
| 404 | ongoingBuild.removeAll{ it.isCancelled() || it.isDone() } |
| 405 | } |
| 406 | } |
| 407 | if (fail){ |
| 408 | throw new AbortException("Some job failed") |
| 409 | } |
| 410 | """ |
| 411 | |
| 412 | dslTriggerKernel = dslTriggerKernel.replaceAll("KERNELPREFIX", kernelPrefix) |
| 413 | dslTriggerKernel = dslTriggerKernel.replaceAll("MODULEPREFIX", modulesPrefix) |
| 414 | if (isJenkinsInstance) { |
| 415 | freeStyleJob("dsl-trigger-kernel") { |
| 416 | steps { |
| 417 | systemGroovyCommand(dslTriggerKernel) |
| 418 | } |
| 419 | triggers { |
| 420 | cron("H 0 * * *") |
| 421 | } |
| 422 | } |
| 423 | |
| 424 | modulesBranches.each { branch -> |
| 425 | dslTriggerModule = dslTriggerModule.replaceAll("MODULEPREFIX",modulesPrefix + separator + branch + separator) |
| 426 | dslTriggerModule = dslTriggerModule.replaceAll("BRANCHNAME",separator + branch + separator) |
| 427 | freeStyleJob("dsl-trigger-module-${branch}") { |
| 428 | steps { |
| 429 | systemGroovyCommand(dslTriggerModule) |
| 430 | } |
| 431 | triggers { |
| 432 | scm('@daily') |
| 433 | } |
| 434 | } |
| 435 | } |
| 436 | } |
| 437 | } |