From: Francis Deslauriers Date: Thu, 21 Dec 2017 21:33:10 +0000 (-0500) Subject: jjb: lava: rename lttng-baremetal-tests to system-tests X-Git-Url: https://git.lttng.org./?a=commitdiff_plain;h=02126236dc58f0d88d92c2869868af3cac3af71c;p=lttng-ci.git jjb: lava: rename lttng-baremetal-tests to system-tests Signed-off-by: Francis Deslauriers --- diff --git a/jobs/lttng-baremetal-tests.yaml b/jobs/lttng-baremetal-tests.yaml deleted file mode 100644 index cf2692d..0000000 --- a/jobs/lttng-baremetal-tests.yaml +++ /dev/null @@ -1,451 +0,0 @@ ---- -## Anchors -- lttng_baremetal_tests_parameters_defaults: <tng_baremetal_tests_parameters_defaults - name: 'lttng_baremetal_tests_parameters_defaults' - parameters: - - string: - name: 'tools_commit_id' - default: '' - description: 'The lttng-tools commit id to build.' - - string: - name: 'modules_commit_id' - default: '' - description: 'The lttng-modules commit id to build.' - - string: - name: 'ust_commit_id' - default: '' - description: 'The lttng-ust commit id to build.' - - string: - name: 'kernel_tag_id' - default: '' - description: 'The linux kernel git tag to build against.' - - string: - name: 'kernel_repo' - default: 'git://git-mirror.internal.efficios.com/kernel/stable/linux-stable.git' - description: 'Linux kernel git repo to checkout the kernel id' - - string: - name: 'tools_repo' - default: 'https://github.com/lttng/lttng-tools.git' - description: 'LTTng-Tools git repo to checkout the tools id' - - string: - name: 'modules_repo' - default: 'https://github.com/lttng/lttng-modules.git' - description: 'LTTng-Modules git repo to checkout the Modules id' - - string: - name: 'ust_repo' - default: 'https://github.com/lttng/lttng-ust.git' - description: 'LTTng-UST git repo to checkout the UST id' - -- lttng_baremetal_tests_publisher_defaults: <tng_baremetal_tests_publisher_defaults - name: 'lttng_baremetal_tests_publisher_defaults' - publishers: - - email-ext: - recipients: '{obj:email_to}' - reply-to: ci-notification@lists.lttng.org - always: false - unstable: false - first-failure: true - first-unstable: true - not-built: false - aborted: false - regression: false - failure: false - second-failure: false - improvement: false - still-failing: false - success: false - fixed: false - fixed-unhealthy: true - still-unstable: false - pre-build: false - matrix-trigger: only-parent - send-to: - - recipients - - workspace-cleanup - -## Defaults -- defaults: - name: baremetal_benchmarks - description: | - Runs baremetal kernel benchmarks over different combination of kernel and lttng configurations. - project-type: freestyle - node: 'x86-64' - - <<: *lttng_baremetal_tests_parameters_defaults - - properties: - - build-discarder: - num-to-keep: 10 - - throttle: - max-total: 2 - option: 'category' - categories: - - 'baremetal-tests' - - wrappers: - - workspace-cleanup - - timestamps - - ansicolor - - credentials-binding: - - text: - credential-id: jenkins_lava_key - variable: LAVA_JENKINS_TOKEN - - file: - credential-id: system_tests_storage_key - variable: identity_file - - inject: - properties-content: | - BUILD_DEVICE=baremetal - - builders: - - shell: !include-raw-escape: scripts/lttng-baremetal-tests/generate-properties-master.sh - - shell: !include-raw-escape: scripts/lttng-baremetal-tests/inject-ssh-commands.sh - - trigger-builds: - - project: "build_kernel_PARAM" - property-file: 'properties.txt' - block: true - - inject: - properties-file: properties.txt - - shell: !include-raw-escape: scripts/lttng-baremetal-tests/run-baremetal-benchmarks.sh - - shell: !include-raw-escape: scripts/lttng-baremetal-tests/summarize-results.sh - - publishers: - - archive: - artifacts: '*.png,*.csv' - stable: true - do-not-fingerprint: true - - email-ext: - recipients: '{obj:email_to}' - reply-to: ci-notification@lists.lttng.org - always: false - unstable: false - first-failure: true - first-unstable: true - not-built: false - aborted: false - regression: false - failure: false - second-failure: false - improvement: false - still-failing: false - success: false - fixed: false - fixed-unhealthy: true - still-unstable: false - pre-build: false - matrix-trigger: only-parent - send-to: - - recipients - - image-gallery: - - gallery-type: archived-images-gallery - title: Results - includes: '*.png' - - workspace-cleanup - -- defaults: - name: vm_tests - description: | - Runs virtual machine kernel tests over different combination of kernel and lttng configurations. - project-type: freestyle - node: 'master' - <<: *lttng_baremetal_tests_parameters_defaults - - properties: - - build-discarder: - num-to-keep: 10 - - throttle: - max-total: 2 - option: 'category' - categories: - - 'kvm-tests' - - wrappers: - - workspace-cleanup - - timestamps - - ansicolor - - credentials-binding: - - text: - credential-id: jenkins_lava_key - variable: LAVA_JENKINS_TOKEN - - inject: - properties-content: | - BUILD_DEVICE=kvm - - builders: - - shell: !include-raw-escape: scripts/lttng-baremetal-tests/generate-properties-master.sh - - trigger-builds: - - project: "build_kernel_PARAM" - property-file: 'properties.txt' - block: true - - inject: - properties-file: properties.txt - - shell: !include-raw-escape: scripts/lttng-baremetal-tests/run-kvm-tests.sh - - <<: *lttng_baremetal_tests_publisher_defaults - -- defaults: - name: vm_tests_fuzzing - description: | - Runs virtual machine fuzzing tests over different combination of kernel and lttng configurations. - project-type: freestyle - node: 'master' - <<: *lttng_baremetal_tests_parameters_defaults - - properties: - - build-discarder: - num-to-keep: 10 - - throttle: - max-total: 2 - option: 'category' - categories: - - 'kvm-tests' - - wrappers: - - workspace-cleanup - - timestamps - - ansicolor - - credentials-binding: - - text: - credential-id: jenkins_lava_key - variable: LAVA_JENKINS_TOKEN - - inject: - properties-content: | - BUILD_DEVICE=kvm - - builders: - - shell: !include-raw-escape: scripts/lttng-baremetal-tests/generate-properties-master.sh - - trigger-builds: - - project: "build_kernel_PARAM" - property-file: 'properties.txt' - block: true - - inject: - properties-file: properties.txt - - shell: !include-raw-escape: scripts/lttng-baremetal-tests/run-kvm-fuzzing-tests.sh - - <<: *lttng_baremetal_tests_publisher_defaults - -- defaults: - name: baremetal_tests - description: | - Runs baremetal kernel tests over different combination of kernel and lttng configurations. - project-type: freestyle - node: 'master' - - <<: *lttng_baremetal_tests_parameters_defaults - - properties: - - build-discarder: - num-to-keep: 10 - - throttle: - max-total: 2 - option: 'category' - categories: - - 'baremetal-tests' - - wrappers: - - workspace-cleanup - - timestamps - - ansicolor - - credentials-binding: - - text: - credential-id: jenkins_lava_key - variable: LAVA_JENKINS_TOKEN - - inject: - properties-content: | - BUILD_DEVICE=baremetal - - builders: - - shell: !include-raw-escape: scripts/lttng-baremetal-tests/generate-properties-master.sh - - trigger-builds: - - project: "build_kernel_PARAM" - property-file: 'properties.txt' - block: true - - inject: - properties-file: properties.txt - - shell: !include-raw-escape: scripts/lttng-baremetal-tests/run-baremetal-tests.sh - - <<: *lttng_baremetal_tests_publisher_defaults - -## Templates -- job-template: - name: build_kernel_PARAM - description: | - Builds a Linux Kernel and LTTng Modules if necessary - concurrent: true - node: 'x86-64' - - properties: - - build-discarder: - num-to-keep: 100 - - wrappers: - - workspace-cleanup - - timestamps - - ansicolor - - credentials-binding: - - file: - credential-id: system_tests_storage_key - variable: identity_file - - builders: - - shell: !include-raw-escape: scripts/lttng-baremetal-tests/generate-properties-slave.sh - - inject: - properties-file: properties.txt - - shell: !include-raw-escape: scripts/lttng-baremetal-tests/check-build-needs.sh - - conditional-step: - condition-kind: not - condition-operand: - condition-kind: file-exists - condition-filename: kernel-built.txt - condition-basedir: workspace - steps: - - shell: !include-raw-escape: scripts/lttng-baremetal-tests/build-kernel.sh - - conditional-step: - condition-kind: not - condition-operand: - condition-kind: file-exists - condition-filename: modules-built.txt - condition-basedir: workspace - steps: - - shell: !include-raw-escape: scripts/lttng-baremetal-tests/build-modules.sh - - parameters: - - string: - name: 'LTTNG_MODULES_COMMIT_ID' - description: 'The lttng-modules commmit to build.' - - string: - name: 'LTTNG_MODULES_REPO' - description: 'The LTTng Modules git repo to fetch from' - default: 'git://git-mirror.internal.efficios.com/lttng/lttng-modules.git' - - string: - name: 'KERNEL_COMMIT_ID' - description: 'The kernel commit to build.' - - string: - name: 'KGITREPO' - description: 'The kernel git repo to fetch from' - - string: - name: 'STORAGE_KERNEL_FOLDER' - description: 'Path to store the Kernel image' - - string: - name: 'STORAGE_KERNEL_IMAGE' - description: 'Path to store the Kernel IMAGE' - - string: - name: 'STORAGE_LINUX_MODULES' - description: 'Path to store the Kernel Modules' - - string: - name: 'STORAGE_LTTNG_MODULES' - description: 'Path to store the LTTng Modules' - - string: - name: 'BUILD_DEVICE' - description: 'The target device. (kvm or baremetal)' - - publishers: - - workspace-cleanup - -- job-template: - name: baremetal_ALL_trigger - description: | - This job will trigger the build of jobs when a new tag is push specific - tracked Linux branches and new commits on LTTng tracked branches - - project-type: freestyle - node: 'master' - - properties: - - build-discarder: - num-to-keep: 10 - - triggers: - - timed: "@daily" - - wrappers: - - timestamps - - ansicolor - - builders: - - system-groovy: - command: - !include-raw-escape: scripts/lttng-baremetal-tests/baremetal-trigger.groovy - - <<: *lttng_baremetal_tests_publisher_defaults - -- job-template: - name: baremetal_benchmarks_k{kversion}_l{lttngversion} - defaults: baremetal_benchmarks -- job-template: - name: vm_tests_k{kversion}_l{lttngversion} - defaults: vm_tests -- job-template: - name: baremetal_tests_k{kversion}_l{lttngversion} - defaults: baremetal_tests - -- job-template: - name: vm_tests_fuzzing_k{kversion}_l{lttngversion} - defaults: vm_tests_fuzzing - -# Canary jobs are made to confirm that the whole Lava pipeline is working. -# They are scheduled once a day always on the same (LTTng, Kernel) code. If any -# of those jobs fails, it means that there is an issue with the configuration -- job-template: - name: baremetal_benchmarks_canary - defaults: baremetal_benchmarks - -- job-template: - name: baremetal_tests_canary - defaults: baremetal_tests - -- job-template: - name: vm_tests_canary - defaults: vm_tests - -## Project -- project: - name: lttng-baremetal-tests - email_to: 'ci-notification@lists.lttng.org, cc:francis.deslauriers@efficios.com' - kversion: - - master - - linux-4.4.y - - linux-4.9.y - lttngversion: - - master - - stable-2.9 - - stable-2.10 - jobs: - - 'vm_tests_k{kversion}_l{lttngversion}' - - 'baremetal_benchmarks_k{kversion}_l{lttngversion}' - - 'baremetal_tests_k{kversion}_l{lttngversion}' - -- project: - name: lttng-baremetal-tests-oldkernel - email_to: 'ci-notification@lists.lttng.org, cc:francis.deslauriers@efficios.com' - kversion: - - linux-3.18.y - - linux-4.4.y - lttngversion: - - stable-2.7 - jobs: - - 'vm_tests_k{kversion}_l{lttngversion}': - - 'baremetal_tests_k{kversion}_l{lttngversion}': - - 'baremetal_benchmarks_k{kversion}_l{lttngversion}': - -- project: - name: lttng-baremetal-tests-canary - email_to: 'ci-notification@lists.lttng.org, cc:francis.deslauriers@efficios.com' - jobs: - - 'vm_tests_canary' - - 'baremetal_tests_canary' - - 'baremetal_benchmarks_canary' - -- project: - name: lttng-baremetal-fuzzing-tests - email_to: 'ci-notification@lists.lttng.org, cc:francis.deslauriers@efficios.com' - kversion: master - lttngversion: master - jobs: - - 'vm_tests_fuzzing_k{kversion}_l{lttngversion}' -- project: - name: lttng-baremetal-general - email_to: 'ci-notification@lists.lttng.org, cc:francis.deslauriers@efficios.com' - jobs: - - 'build_kernel_PARAM' - - 'baremetal_ALL_trigger' diff --git a/jobs/system-tests.yaml b/jobs/system-tests.yaml new file mode 100644 index 0000000..87b92de --- /dev/null +++ b/jobs/system-tests.yaml @@ -0,0 +1,451 @@ +--- +## Anchors +- system_tests_parameters_defaults: &system_tests_parameters_defaults + name: 'system_tests_parameters_defaults' + parameters: + - string: + name: 'tools_commit_id' + default: '' + description: 'The lttng-tools commit id to build.' + - string: + name: 'modules_commit_id' + default: '' + description: 'The lttng-modules commit id to build.' + - string: + name: 'ust_commit_id' + default: '' + description: 'The lttng-ust commit id to build.' + - string: + name: 'kernel_tag_id' + default: '' + description: 'The linux kernel git tag to build against.' + - string: + name: 'kernel_repo' + default: 'git://git-mirror.internal.efficios.com/kernel/stable/linux-stable.git' + description: 'Linux kernel git repo to checkout the kernel id' + - string: + name: 'tools_repo' + default: 'https://github.com/lttng/lttng-tools.git' + description: 'LTTng-Tools git repo to checkout the tools id' + - string: + name: 'modules_repo' + default: 'https://github.com/lttng/lttng-modules.git' + description: 'LTTng-Modules git repo to checkout the Modules id' + - string: + name: 'ust_repo' + default: 'https://github.com/lttng/lttng-ust.git' + description: 'LTTng-UST git repo to checkout the UST id' + +- system_tests_publisher_defaults: &system_tests_publisher_defaults + name: 'system_tests_publisher_defaults' + publishers: + - email-ext: + recipients: '{obj:email_to}' + reply-to: ci-notification@lists.lttng.org + always: false + unstable: false + first-failure: true + first-unstable: true + not-built: false + aborted: false + regression: false + failure: false + second-failure: false + improvement: false + still-failing: false + success: false + fixed: false + fixed-unhealthy: true + still-unstable: false + pre-build: false + matrix-trigger: only-parent + send-to: + - recipients + - workspace-cleanup + +## Defaults +- defaults: + name: baremetal_benchmarks + description: | + Runs baremetal kernel benchmarks over different combination of kernel and lttng configurations. + project-type: freestyle + node: 'x86-64' + + <<: *system_tests_parameters_defaults + + properties: + - build-discarder: + num-to-keep: 10 + - throttle: + max-total: 2 + option: 'category' + categories: + - 'baremetal-tests' + + wrappers: + - workspace-cleanup + - timestamps + - ansicolor + - credentials-binding: + - text: + credential-id: jenkins_lava_key + variable: LAVA_JENKINS_TOKEN + - file: + credential-id: system_tests_storage_key + variable: identity_file + - inject: + properties-content: | + BUILD_DEVICE=baremetal + + builders: + - shell: !include-raw-escape: scripts/system-tests/generate-properties-master.sh + - shell: !include-raw-escape: scripts/system-tests/inject-ssh-commands.sh + - trigger-builds: + - project: "build_kernel_PARAM" + property-file: 'properties.txt' + block: true + - inject: + properties-file: properties.txt + - shell: !include-raw-escape: scripts/system-tests/run-baremetal-benchmarks.sh + - shell: !include-raw-escape: scripts/system-tests/summarize-results.sh + + publishers: + - archive: + artifacts: '*.png,*.csv' + stable: true + do-not-fingerprint: true + - email-ext: + recipients: '{obj:email_to}' + reply-to: ci-notification@lists.lttng.org + always: false + unstable: false + first-failure: true + first-unstable: true + not-built: false + aborted: false + regression: false + failure: false + second-failure: false + improvement: false + still-failing: false + success: false + fixed: false + fixed-unhealthy: true + still-unstable: false + pre-build: false + matrix-trigger: only-parent + send-to: + - recipients + - image-gallery: + - gallery-type: archived-images-gallery + title: Results + includes: '*.png' + - workspace-cleanup + +- defaults: + name: vm_tests + description: | + Runs virtual machine kernel tests over different combination of kernel and lttng configurations. + project-type: freestyle + node: 'master' + <<: *system_tests_parameters_defaults + + properties: + - build-discarder: + num-to-keep: 10 + - throttle: + max-total: 2 + option: 'category' + categories: + - 'kvm-tests' + + wrappers: + - workspace-cleanup + - timestamps + - ansicolor + - credentials-binding: + - text: + credential-id: jenkins_lava_key + variable: LAVA_JENKINS_TOKEN + - inject: + properties-content: | + BUILD_DEVICE=kvm + + builders: + - shell: !include-raw-escape: scripts/system-tests/generate-properties-master.sh + - trigger-builds: + - project: "build_kernel_PARAM" + property-file: 'properties.txt' + block: true + - inject: + properties-file: properties.txt + - shell: !include-raw-escape: scripts/system-tests/run-kvm-tests.sh + + <<: *system_tests_publisher_defaults + +- defaults: + name: vm_tests_fuzzing + description: | + Runs virtual machine fuzzing tests over different combination of kernel and lttng configurations. + project-type: freestyle + node: 'master' + <<: *system_tests_parameters_defaults + + properties: + - build-discarder: + num-to-keep: 10 + - throttle: + max-total: 2 + option: 'category' + categories: + - 'kvm-tests' + + wrappers: + - workspace-cleanup + - timestamps + - ansicolor + - credentials-binding: + - text: + credential-id: jenkins_lava_key + variable: LAVA_JENKINS_TOKEN + - inject: + properties-content: | + BUILD_DEVICE=kvm + + builders: + - shell: !include-raw-escape: scripts/system-tests/generate-properties-master.sh + - trigger-builds: + - project: "build_kernel_PARAM" + property-file: 'properties.txt' + block: true + - inject: + properties-file: properties.txt + - shell: !include-raw-escape: scripts/system-tests/run-kvm-fuzzing-tests.sh + + <<: *system_tests_publisher_defaults + +- defaults: + name: baremetal_tests + description: | + Runs baremetal kernel tests over different combination of kernel and lttng configurations. + project-type: freestyle + node: 'master' + + <<: *system_tests_parameters_defaults + + properties: + - build-discarder: + num-to-keep: 10 + - throttle: + max-total: 2 + option: 'category' + categories: + - 'baremetal-tests' + + wrappers: + - workspace-cleanup + - timestamps + - ansicolor + - credentials-binding: + - text: + credential-id: jenkins_lava_key + variable: LAVA_JENKINS_TOKEN + - inject: + properties-content: | + BUILD_DEVICE=baremetal + + builders: + - shell: !include-raw-escape: scripts/system-tests/generate-properties-master.sh + - trigger-builds: + - project: "build_kernel_PARAM" + property-file: 'properties.txt' + block: true + - inject: + properties-file: properties.txt + - shell: !include-raw-escape: scripts/system-tests/run-baremetal-tests.sh + + <<: *system_tests_publisher_defaults + +## Templates +- job-template: + name: build_kernel_PARAM + description: | + Builds a Linux Kernel and LTTng Modules if necessary + concurrent: true + node: 'x86-64' + + properties: + - build-discarder: + num-to-keep: 100 + + wrappers: + - workspace-cleanup + - timestamps + - ansicolor + - credentials-binding: + - file: + credential-id: system_tests_storage_key + variable: identity_file + + builders: + - shell: !include-raw-escape: scripts/system-tests/generate-properties-slave.sh + - inject: + properties-file: properties.txt + - shell: !include-raw-escape: scripts/system-tests/check-build-needs.sh + - conditional-step: + condition-kind: not + condition-operand: + condition-kind: file-exists + condition-filename: kernel-built.txt + condition-basedir: workspace + steps: + - shell: !include-raw-escape: scripts/system-tests/build-kernel.sh + - conditional-step: + condition-kind: not + condition-operand: + condition-kind: file-exists + condition-filename: modules-built.txt + condition-basedir: workspace + steps: + - shell: !include-raw-escape: scripts/system-tests/build-modules.sh + + parameters: + - string: + name: 'LTTNG_MODULES_COMMIT_ID' + description: 'The lttng-modules commmit to build.' + - string: + name: 'LTTNG_MODULES_REPO' + description: 'The LTTng Modules git repo to fetch from' + default: 'git://git-mirror.internal.efficios.com/lttng/lttng-modules.git' + - string: + name: 'KERNEL_COMMIT_ID' + description: 'The kernel commit to build.' + - string: + name: 'KGITREPO' + description: 'The kernel git repo to fetch from' + - string: + name: 'STORAGE_KERNEL_FOLDER' + description: 'Path to store the Kernel image' + - string: + name: 'STORAGE_KERNEL_IMAGE' + description: 'Path to store the Kernel IMAGE' + - string: + name: 'STORAGE_LINUX_MODULES' + description: 'Path to store the Kernel Modules' + - string: + name: 'STORAGE_LTTNG_MODULES' + description: 'Path to store the LTTng Modules' + - string: + name: 'BUILD_DEVICE' + description: 'The target device. (kvm or baremetal)' + + publishers: + - workspace-cleanup + +- job-template: + name: system_ALL_trigger + description: | + This job will trigger the build of jobs when a new tag is push specific + tracked Linux branches and new commits on LTTng tracked branches + + project-type: freestyle + node: 'master' + + properties: + - build-discarder: + num-to-keep: 10 + + triggers: + - timed: "@daily" + + wrappers: + - timestamps + - ansicolor + + builders: + - system-groovy: + command: + !include-raw-escape: scripts/system-tests/system-trigger.groovy + + <<: *system_tests_publisher_defaults + +- job-template: + name: baremetal_benchmarks_k{kversion}_l{lttngversion} + defaults: baremetal_benchmarks +- job-template: + name: vm_tests_k{kversion}_l{lttngversion} + defaults: vm_tests +- job-template: + name: baremetal_tests_k{kversion}_l{lttngversion} + defaults: baremetal_tests + +- job-template: + name: vm_tests_fuzzing_k{kversion}_l{lttngversion} + defaults: vm_tests_fuzzing + +# Canary jobs are made to confirm that the whole Lava pipeline is working. +# They are scheduled once a day always on the same (LTTng, Kernel) code. If any +# of those jobs fails, it means that there is an issue with the configuration +- job-template: + name: baremetal_benchmarks_canary + defaults: baremetal_benchmarks + +- job-template: + name: baremetal_tests_canary + defaults: baremetal_tests + +- job-template: + name: vm_tests_canary + defaults: vm_tests + +## Project +- project: + name: system-tests + email_to: 'ci-notification@lists.lttng.org, cc:francis.deslauriers@efficios.com' + kversion: + - master + - linux-4.4.y + - linux-4.9.y + lttngversion: + - master + - stable-2.9 + - stable-2.10 + jobs: + - 'vm_tests_k{kversion}_l{lttngversion}' + - 'baremetal_benchmarks_k{kversion}_l{lttngversion}' + - 'baremetal_tests_k{kversion}_l{lttngversion}' + +- project: + name: system-tests-oldkernel + email_to: 'ci-notification@lists.lttng.org, cc:francis.deslauriers@efficios.com' + kversion: + - linux-3.18.y + - linux-4.4.y + lttngversion: + - stable-2.7 + jobs: + - 'vm_tests_k{kversion}_l{lttngversion}': + - 'baremetal_tests_k{kversion}_l{lttngversion}': + - 'baremetal_benchmarks_k{kversion}_l{lttngversion}': + +- project: + name: system-tests-canary + email_to: 'ci-notification@lists.lttng.org, cc:francis.deslauriers@efficios.com' + jobs: + - 'vm_tests_canary' + - 'baremetal_tests_canary' + - 'baremetal_benchmarks_canary' + +- project: + name: system-fuzzing-tests + email_to: 'ci-notification@lists.lttng.org, cc:francis.deslauriers@efficios.com' + kversion: master + lttngversion: master + jobs: + - 'vm_tests_fuzzing_k{kversion}_l{lttngversion}' +- project: + name: system-general + email_to: 'ci-notification@lists.lttng.org, cc:francis.deslauriers@efficios.com' + jobs: + - 'build_kernel_PARAM' + - 'system_ALL_trigger' diff --git a/scripts/lttng-baremetal-tests/baremetal-trigger.groovy b/scripts/lttng-baremetal-tests/baremetal-trigger.groovy deleted file mode 100644 index 394227c..0000000 --- a/scripts/lttng-baremetal-tests/baremetal-trigger.groovy +++ /dev/null @@ -1,462 +0,0 @@ -/** - * Copyright (C) 2017 - Francis Deslauriers - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -import hudson.console.HyperlinkNote -import hudson.model.* -import java.io.File -import org.eclipse.jgit.api.Git -import org.eclipse.jgit.lib.Ref - -class InvalidKVersionException extends Exception { - public InvalidKVersionException(String message) { - super(message) - } -} - -class EmptyKVersionException extends Exception { - public EmptyKVersionException(String message) { - super(message) - } -} - -class VanillaKVersion implements Comparable { - - Integer major = 0 - Integer majorB = 0 - Integer minor = 0 - Integer patch = 0 - Integer rc = Integer.MAX_VALUE - Boolean inStable = false; - - VanillaKVersion() {} - - VanillaKVersion(version) { - this.parse(version) - } - - static VanillaKVersion minKVersion() { - return new VanillaKVersion("v0.0.0") - } - - static VanillaKVersion maxKVersion() { - return new VanillaKVersion("v" + Integer.MAX_VALUE + ".0.0") - } - - static VanillaKVersion factory(version) { - return new VanillaKVersion(version) - } - - def parse(version) { - this.major = 0 - this.majorB = 0 - this.minor = 0 - this.patch = 0 - this.rc = Integer.MAX_VALUE - - if (!version) { - throw new EmptyKVersionException("Empty kernel version") - } - - def match = version =~ /^v(\d+)\.(\d+)(\.(\d+))?(\.(\d+))?(-rc(\d+))?$/ - if (!match) { - throw new InvalidKVersionException("Invalid kernel version: ${version}") - } - - Integer offset = 0; - - // Major - this.major = Integer.parseInt(match.group(1)) - if (this.major <= 2) { - offset = 2 - this.majorB = Integer.parseInt(match.group(2)) - } - - // Minor - if (match.group(2 + offset) != null) { - this.minor = Integer.parseInt(match.group(2 + offset)) - } - - // Patch level - if (match.group(4 + offset) != null) { - this.patch = Integer.parseInt(match.group(4 + offset)) - this.inStable = true - } - - // RC - if (match.group(8) != null) { - this.rc = Integer.parseInt(match.group(8)) - } - } - - Boolean isInStableBranch() { - return this.inStable - } - - // Return true if both version are of the same stable branch - Boolean isSameStable(VanillaKVersion o) { - if (this.major != o.major) { - return false - } - if (this.majorB != o.majorB) { - return false - } - if (this.minor != o.minor) { - return false - } - - return true - } - - @Override int compareTo(VanillaKVersion o) { - if (this.major != o.major) { - return Integer.compare(this.major, o.major) - } - if (this.majorB != o.majorB) { - return Integer.compare(this.majorB, o.majorB) - } - if (this.minor != o.minor) { - return Integer.compare(this.minor, o.minor) - } - if (this.patch != o.patch) { - return Integer.compare(this.patch, o.patch) - } - if (this.rc != o.rc) { - return Integer.compare(this.rc, o.rc) - } - - // Same version - return 0; - } - - String toString() { - String vString = "v${this.major}" - - if (this.majorB > 0) { - vString = vString.concat(".${this.majorB}") - } - - vString = vString.concat(".${this.minor}") - - if (this.patch > 0) { - vString = vString.concat(".${this.patch}") - } - - if (this.rc > 0 && this.rc < Integer.MAX_VALUE) { - vString = vString.concat("-rc${this.rc}") - } - return vString - } -} - -class RunConfiguration { - def linuxBranch - def linuxTagId - def lttngBranch - def lttngModulesCommitId - def lttngToolsCommitId - def lttngUstCommitId - RunConfiguration(linuxBranch, linuxTagId, lttngBranch, lttngToolsCommitId, - lttngModulesCommitId, lttngUstCommitId) { - this.linuxBranch = linuxBranch - this.linuxTagId = linuxTagId - this.lttngBranch = lttngBranch - this.lttngModulesCommitId = lttngModulesCommitId - this.lttngToolsCommitId = lttngToolsCommitId - this.lttngUstCommitId = lttngUstCommitId - } - - String toString() { - return "${this.linuxBranch}:{${this.linuxTagId}}, ${this.lttngBranch}:{${this.lttngModulesCommitId}, ${this.lttngToolsCommitId}, ${this.lttngUstCommitId}}" - } -} - -def LoadPreviousIdsFromWorkspace = { ondiskpath -> - def previousIds = [] - try { - File myFile = new File(ondiskpath); - def input = new ObjectInputStream(new FileInputStream(ondiskpath)) - previousIds = input.readObject() - input.close() - } catch (all) { - println("Failed to load previous ids from disk.") - } - return previousIds -} - -def saveCurrentIdsToWorkspace = { currentIds, ondiskpath -> - try { - File myFile = new File(ondiskpath); - myFile.createNewFile(); - def out = new ObjectOutputStream(new FileOutputStream(ondiskpath)) - out.writeObject(currentIds) - out.close() - } catch (all) { - println("Failed to save previous ids from disk.") - } -} - -def GetHeadCommits = { remoteRepo, branchesOfInterest -> - def remoteHeads = [:] - def remoteHeadRefs = Git.lsRemoteRepository() - .setTags(false) - .setHeads(true) - .setRemote(remoteRepo).call() - - remoteHeadRefs.each { - def branch = it.getName().replaceAll('refs/heads/', '') - if (branchesOfInterest.contains(branch)) - remoteHeads[branch] = it.getObjectId().name() - } - - return remoteHeads -} - -def GetTagIds = { remoteRepo -> - def remoteTags = [:] - def remoteTagRefs = Git.lsRemoteRepository() - .setTags(true) - .setHeads(false) - .setRemote(remoteRepo).call() - - remoteTagRefs.each { - // Exclude release candidate tags - if (!it.getName().contains('-rc')) { - remoteTags[it.getName().replaceAll('refs/tags/', '')] = it.getObjectId().name() - } - } - - return remoteTags -} - -def GetLastTagOfBranch = { tagRefs, branch -> - def tagVersions = tagRefs.collect {new VanillaKVersion(it.key)} - def currMax = new VanillaKVersion('v0.0.0'); - if (!branch.contains('master')){ - def targetVersion = new VanillaKVersion(branch.replaceAll('linux-', 'v').replaceAll('.y', '')) - tagVersions.each { - if (it.isSameStable(targetVersion)) { - if (currMax < it) { - currMax = it; - } - } - } - } else { - tagVersions.each { - if (!it.isInStableBranch() && currMax < it) { - currMax = it; - } - } - } - return currMax.toString() -} - -// Returns the latest tags of each of the branches passed in the argument -def GetLastTagIds = { remoteRepo, branchesOfInterest -> - def remoteHeads = GetHeadCommits(remoteRepo, branchesOfInterest) - def remoteTagRefs = GetTagIds(remoteRepo) - def remoteLastTagCommit = [:] - - remoteTagRefs = remoteTagRefs.findAll { !it.key.contains("v2.") } - branchesOfInterest.each { - remoteLastTagCommit[it] = remoteTagRefs[GetLastTagOfBranch(remoteTagRefs, it)] - } - - return remoteLastTagCommit -} - -def CraftJobName = { jobType, runConfig -> - return "${jobType}_k${runConfig.linuxBranch}_l${runConfig.lttngBranch}" -} - -def LaunchJob = { jobName, runConfig -> - def job = Hudson.instance.getJob(jobName) - def params = [] - for (paramdef in job.getProperty(ParametersDefinitionProperty.class).getParameterDefinitions()) { - params += paramdef.getDefaultParameterValue(); - } - - params.add(new StringParameterValue('tools_commit_id', runConfig.lttngToolsCommitId)) - params.add(new StringParameterValue('modules_commit_id', runConfig.lttngModulesCommitId)) - params.add(new StringParameterValue('ust_commit_id', runConfig.lttngUstCommitId)) - params.add(new StringParameterValue('kernel_tag_id', runConfig.linuxTagId)) - job.scheduleBuild2(0, new Cause.UpstreamCause(build), new ParametersAction(params)) - println "Launching job: ${HyperlinkNote.encodeTo('/' + job.url, job.fullDisplayName)}" -} - -def jobTypes = ['baremetal_tests', 'vm_tests', 'baremetal_benchmarks'] -final String toolsRepo = "https://github.com/lttng/lttng-tools.git" -final String modulesRepo = "https://github.com/lttng/lttng-modules.git" -final String ustRepo = "https://github.com/lttng/lttng-ust.git" -final String linuxRepo = "git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git" - -final String toolsOnDiskPath = build.getEnvironment(listener).get('WORKSPACE') + "/on-disk-tools-ref" -final String modulesOnDiskPath = build.getEnvironment(listener).get('WORKSPACE') + "/on-disk-modules-ref" -final String ustOnDiskPath = build.getEnvironment(listener).get('WORKSPACE') + "/on-disk-ust-ref" -final String linuxOnDiskPath = build.getEnvironment(listener).get('WORKSPACE') + "/on-disk-linux-ref" - -def recentLttngBranchesOfInterest = ['master', 'stable-2.10', 'stable-2.9'] -def recentLinuxBranchesOfInterest = ['master', 'linux-4.9.y', 'linux-4.4.y'] - -def legacyLttngBranchesOfInterest = ['stable-2.7'] -def legacyLinuxBranchesOfInterest = ['linux-3.18.y', 'linux-4.4.y'] - -// Generate configurations of interest -def configurationOfInterest = [] as Set - -recentLttngBranchesOfInterest.each { lttngBranch -> - recentLinuxBranchesOfInterest.each { linuxBranch -> - configurationOfInterest.add([lttngBranch, linuxBranch]) - } -} - -legacyLttngBranchesOfInterest.each { lttngBranch -> - legacyLinuxBranchesOfInterest.each { linuxBranch -> - configurationOfInterest.add([lttngBranch, linuxBranch]) - } -} - -def lttngBranchesOfInterest = recentLttngBranchesOfInterest + legacyLttngBranchesOfInterest -def linuxBranchesOfInterest = recentLinuxBranchesOfInterest + legacyLinuxBranchesOfInterest - -// For Linux branches, we look for new non-RC tags -def toolsHeadCommits = GetHeadCommits(toolsRepo, lttngBranchesOfInterest) -def modulesHeadCommits = GetHeadCommits(modulesRepo, lttngBranchesOfInterest) -def ustHeadCommits = GetHeadCommits(ustRepo, lttngBranchesOfInterest) - -// For LTTng branches, we look for new commits -def linuxLastTagIds = GetLastTagIds(linuxRepo, linuxBranchesOfInterest) - -// Load previously build Linux tag ids -def oldLinuxTags = LoadPreviousIdsFromWorkspace(linuxOnDiskPath) as Set - -// Load previously built LTTng commit ids -def oldToolsHeadCommits = LoadPreviousIdsFromWorkspace(toolsOnDiskPath) as Set -def oldModulesHeadCommits = LoadPreviousIdsFromWorkspace(modulesOnDiskPath) as Set -def oldUstHeadCommits = LoadPreviousIdsFromWorkspace(ustOnDiskPath) as Set - -def newOldLinuxTags = oldLinuxTags -def newOldToolsHeadCommits = oldToolsHeadCommits -def newOldModulesHeadCommits = oldModulesHeadCommits -def newOldUstHeadCommits = oldUstHeadCommits - -def canaryRunConfigs = [] as Set -canaryRunConfigs.add( - ['v4.4.9', '1a1a512b983108015ced1e7a7c7775cfeec42d8c', 'v2.8.1','d11e0db', '7fd9215', '514a87f'] as RunConfiguration) - -def runConfigs = [] as Set - -// For each top of branch kernel tags that were not seen before, schedule one -// job for each lttng/linux tracked configurations -linuxLastTagIds.each { linuxTag -> - if (!oldLinuxTags.contains(linuxTag.value)) { - lttngBranchesOfInterest.each { lttngBranch -> - if (configurationOfInterest.contains([lttngBranch, linuxTag.key])) { - runConfigs.add([linuxTag.key, linuxTag.value, - lttngBranch, toolsHeadCommits[lttngBranch], - modulesHeadCommits[lttngBranch], ustHeadCommits[lttngBranch]] - as RunConfiguration) - - newOldLinuxTags.add(linuxTag.value) - } - } - } -} - -// For each top of branch commits that were not seen before, schedule one job -// for each lttng/linux tracked configurations -toolsHeadCommits.each { toolsHead -> - if (!oldToolsHeadCommits.contains(toolsHead.value)) { - linuxLastTagIds.each { linuxTag -> - def lttngBranch = toolsHead.key - if (configurationOfInterest.contains([lttngBranch, linuxTag.key])) { - runConfigs.add([linuxTag.key, linuxTag.value, - lttngBranch, toolsHeadCommits[lttngBranch], - modulesHeadCommits[lttngBranch], ustHeadCommits[lttngBranch]] - as RunConfiguration) - - newOldToolsHeadCommits.add(toolsHead.value) - } - } - } -} - -// For each top of branch commits that were not seen before, schedule one job -// for each lttng/linux tracked configurations -modulesHeadCommits.each { modulesHead -> - if (!oldModulesHeadCommits.contains(modulesHead.value)) { - linuxLastTagIds.each { linuxTag -> - def lttngBranch = modulesHead.key - if (configurationOfInterest.contains([lttngBranch, linuxTag.key])) { - runConfigs.add([linuxTag.key, linuxTag.value, - lttngBranch, toolsHeadCommits[lttngBranch], - modulesHeadCommits[lttngBranch], ustHeadCommits[lttngBranch]] - as RunConfiguration) - - newOldModulesHeadCommits.add(modulesHead.value) - } - } - } -} - -// For each top of branch commits that were not seen before, schedule one job -// for each lttng/linux tracked configurations -ustHeadCommits.each { ustHead -> - if (!oldUstHeadCommits.contains(ustHead.value)) { - linuxLastTagIds.each { linuxTag -> - def lttngBranch = ustHead.key - if (configurationOfInterest.contains([lttngBranch, linuxTag.key])) { - runConfigs.add([linuxTag.key, linuxTag.value, - lttngBranch, toolsHeadCommits[lttngBranch], - modulesHeadCommits[lttngBranch], ustHeadCommits[lttngBranch]] - as RunConfiguration) - - newOldUstHeadCommits.add(ustHead.value) - } - } - } -} - -// Save the tag and commit IDs scheduled in the past and during this run to the workspace -saveCurrentIdsToWorkspace(newOldLinuxTags, linuxOnDiskPath) -saveCurrentIdsToWorkspace(newOldToolsHeadCommits, toolsOnDiskPath) -saveCurrentIdsToWorkspace(newOldModulesHeadCommits, modulesOnDiskPath) -saveCurrentIdsToWorkspace(newOldUstHeadCommits, ustOnDiskPath) - -// Launch jobs -println("Schedule canary jobs once a day") -canaryRunConfigs.each { config -> - jobTypes.each { type -> - LaunchJob(type + '_canary', config) - } -} - -if (runConfigs.size() > 0) { - println("Schedule jobs because of code changes.") - runConfigs.each { config -> - jobTypes.each { type -> - LaunchJob(CraftJobName(type, config), config); - } - - // Jobs to run only on master branchs of both linux and lttng - if (config.linuxBranch.contains('master') && - config.lttngBranch.contains('master')) { - LaunchJob(CraftJobName('vm_tests_fuzzing', config), config) - } - } -} else { - println("No new commit or tags, nothing more to do.") -} diff --git a/scripts/lttng-baremetal-tests/build-kernel.sh b/scripts/lttng-baremetal-tests/build-kernel.sh deleted file mode 100644 index 7f0cca7..0000000 --- a/scripts/lttng-baremetal-tests/build-kernel.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -xeu -# Copyright (C) 2016 - Francis Deslauriers -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -echo 'kernel-built.txt does not exist' -echo 'So we build it' - -make --directory="$LINUX_PATH" "-j$NPROC" bzImage modules -make --directory="$LINUX_PATH" INSTALL_MOD_PATH="$MODULES_INSTALL_FOLDER" modules_install - -cp "$LINUX_PATH"/arch/x86/boot/bzImage "$DEPLOYDIR"/"$KERNEL_COMMIT_ID".bzImage -cp "$LINUX_PATH"/.config "$DEPLOYDIR"/"$KERNEL_COMMIT_ID".config - -tar -czf "$DEPLOYDIR/$KERNEL_COMMIT_ID.linux.modules.tar.gz" -C "$MODULES_INSTALL_FOLDER/" ./ - -$SCP_COMMAND "$DEPLOYDIR/$KERNEL_COMMIT_ID.bzImage" "$STORAGE_USER@$STORAGE_HOST:$STORAGE_KERNEL_IMAGE" -$SCP_COMMAND "$DEPLOYDIR/$KERNEL_COMMIT_ID.config" "$STORAGE_USER@$STORAGE_HOST:$STORAGE_KERNEL_CONFIG" -$SCP_COMMAND "$DEPLOYDIR/$KERNEL_COMMIT_ID.linux.modules.tar.gz" "$STORAGE_USER@$STORAGE_HOST:$STORAGE_LINUX_MODULES" -$SCP_COMMAND "$LINUX_PATH/Module.symvers" "$STORAGE_USER@$STORAGE_HOST:$STORAGE_KERNEL_MODULE_SYMVERS" diff --git a/scripts/lttng-baremetal-tests/build-modules.sh b/scripts/lttng-baremetal-tests/build-modules.sh deleted file mode 100644 index 04a8c0f..0000000 --- a/scripts/lttng-baremetal-tests/build-modules.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -xeu -# Copyright (C) 2016 - Francis Deslauriers -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -echo 'modules-built.txt does not exist' -echo 'So we build them against the kernel' - -$SCP_COMMAND "$STORAGE_USER@$STORAGE_HOST:$STORAGE_KERNEL_MODULE_SYMVERS" "$LINUX_PATH/Module.symvers" - -KERNELDIR="$LINUX_PATH" make -j"$NPROC" --directory="$LTTNG_MODULES_PATH" - -KERNELDIR="$LINUX_PATH" make -j"$NPROC" --directory="$LTTNG_MODULES_PATH" modules_install INSTALL_MOD_PATH="$MODULES_INSTALL_FOLDER" - -tar -czf "$DEPLOYDIR/$BUILD_NAME.lttng.modules.tar.gz" -C "$MODULES_INSTALL_FOLDER/" ./ - -$SCP_COMMAND "$DEPLOYDIR/$BUILD_NAME.lttng.modules.tar.gz" "$STORAGE_USER@$STORAGE_HOST:$STORAGE_LTTNG_MODULES" diff --git a/scripts/lttng-baremetal-tests/check-build-needs.sh b/scripts/lttng-baremetal-tests/check-build-needs.sh deleted file mode 100644 index 625047e..0000000 --- a/scripts/lttng-baremetal-tests/check-build-needs.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash -xeu -# Copyright (C) 2016 - Francis Deslauriers -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -mkdir -p "$DEPLOYDIR" - -NEED_MODULES_BUILD=0 -NEED_KERNEL_BUILD=0 - -set +e -$SSH_COMMAND "$STORAGE_USER@$STORAGE_HOST" ls "$STORAGE_KERNEL_IMAGE" -if [ $? -ne 0 ]; then - NEED_KERNEL_BUILD=1 - # We need to build the lttng modules if the kernel has changed. - NEED_MODULES_BUILD=1 -fi - -$SSH_COMMAND "$STORAGE_USER@$STORAGE_HOST" ls "$STORAGE_LTTNG_MODULES" -if [ $? -ne 0 ]; then - NEED_MODULES_BUILD=1 -fi -set -e - -# We need to fetch the kernel source and lttng-modules to build either the -# kernel or modules -if [ $NEED_MODULES_BUILD -eq 1 ] || [ $NEED_KERNEL_BUILD -eq 1 ] ; then - - git clone "$KGITREPO" "$LINUX_PATH" - pushd "$LINUX_PATH" - git checkout "$KERNEL_COMMIT_ID" - popd - git clone "$LTTNG_MODULES_GIT" "$LTTNG_MODULES_PATH" - pushd "$LTTNG_MODULES_PATH" - git checkout "$LTTNG_MODULES_COMMIT_ID" - popd - - git clone https://github.com/lttng/lttng-ci "$LTTNG_CI_PATH" - cp "$LTTNG_CI_PATH"/lava/kernel/vanilla/x86_64_server.config "$LINUX_PATH/.config" - make --directory="$LINUX_PATH" olddefconfig - - if [ $BUILD_DEVICE = 'kvm' ] ; then - make --directory="$LINUX_PATH" kvmconfig - fi - - make --directory="$LINUX_PATH" modules_prepare -fi - -#We create files to specify what needs to be built for the subsequent build steps -if [ $NEED_MODULES_BUILD -eq 0 ] ; then - touch modules-built.txt -fi -if [ $NEED_KERNEL_BUILD -eq 0 ] ; then - touch kernel-built.txt -fi diff --git a/scripts/lttng-baremetal-tests/generate-plots.py b/scripts/lttng-baremetal-tests/generate-plots.py deleted file mode 100644 index efc9877..0000000 --- a/scripts/lttng-baremetal-tests/generate-plots.py +++ /dev/null @@ -1,179 +0,0 @@ -# Copyright (C) 2017 - Francis Deslauriers -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -import os, sys -import numpy as np -import pandas as pd - -#Set Matplotlib to use the PNG non interactive backend -import matplotlib as mpl -mpl.use('Agg') - -import matplotlib.pyplot as plt -from matplotlib.ticker import MaxNLocator -from cycler import cycler -from collections import OrderedDict - -def rename_cols(df): - new_cols = {'baseline_1thr_peritermean': 'basel_1thr', - 'baseline_2thr_peritermean': 'basel_2thr', - 'baseline_4thr_peritermean': 'basel_4thr', - 'baseline_8thr_peritermean': 'basel_8thr', - 'baseline_16thr_peritermean': 'basel_16thr', - 'lttng_1thr_peritermean': 'lttng_1thr', - 'lttng_2thr_peritermean': 'lttng_2thr', - 'lttng_4thr_peritermean': 'lttng_4thr', - 'lttng_8thr_peritermean': 'lttng_8thr', - 'lttng_16thr_peritermean': 'lttng_16thr', - 'baseline_1thr_periterstdev': 'basel_1thr_stdev', - 'baseline_2thr_periterstdev': 'basel_2thr_stdev', - 'baseline_4thr_periterstdev': 'basel_4thr_stdev', - 'baseline_8thr_periterstdev': 'basel_8thr_stdev', - 'baseline_16thr_periterstdev': 'basel_16thr_stdev', - 'lttng_1thr_periterstdev': 'lttng_1thr_stdev', - 'lttng_2thr_periterstdev': 'lttng_2thr_stdev', - 'lttng_4thr_periterstdev': 'lttng_4thr_stdev', - 'lttng_8thr_periterstdev': 'lttng_8thr_stdev', - 'lttng_16thr_periterstdev': 'lttng_16thr_stdev' - } - df.rename(columns=new_cols, inplace=True) - return df - -def convert_us_to_ns(df): - cols = [col for col in df.columns if 'periter' in col] - df[cols] = df[cols].apply(lambda x: x*1000) - return df - -def create_plot(df, graph_type): - # We map all test configurations and their - # respective color - conf_to_color = OrderedDict([ - ('basel_1thr','lightcoral'), - ('lttng_1thr','red'), - ('basel_2thr','gray'), - ('lttng_2thr','black'), - ('basel_4thr','chartreuse'), - ('lttng_4thr','forestgreen'), - ('basel_8thr','deepskyblue'), - ('lttng_8thr','mediumblue'), - ('basel_16thr','orange'), - ('lttng_16thr','saddlebrown')]) - - # We create a list for each of the subplots - baseline = [x for x in conf_to_color.keys() if 'basel' in x] - lttng = [x for x in conf_to_color.keys() if 'lttng' in x] - one_thr = [x for x in conf_to_color.keys() if '_1thr' in x] - two_thr = [x for x in conf_to_color.keys() if '_2thr' in x] - four_thr = [x for x in conf_to_color.keys() if '_4thr' in x] - eight_thr = [x for x in conf_to_color.keys() if '_8thr' in x] - sixteen_thr = [x for x in conf_to_color.keys() if '_16thr' in x] - - plots = [baseline, lttng, one_thr, two_thr, four_thr, eight_thr, sixteen_thr] - - title='Meantime per event for {} testcase'.format(graph_type) - - # Create a axe object for each sub-plots - f, arrax = plt.subplots(len(plots), sharex=True, figsize=(16, 25)) - f.suptitle(title, fontsize=20) - - for (ax, data_cols) in zip(arrax, plots): - curr_df = df[data_cols] - - stdev_cols = ['{}_stdev'.format(x) for x in data_cols] - # Extract the color for each configuration - colors = [conf_to_color[x] for x in data_cols] - - # set the color cycler for this plot - ax.set_prop_cycle(cycler('color', colors)) - - # Plot each line and its errorbars - for (data, stdev) in zip(data_cols, stdev_cols): - ax.errorbar(x=df.index.values, y=df[data].values, yerr=df[stdev].values, marker='o') - - ax.set_ylim(0) - ax.grid() - ax.set_xlabel('Jenkins Build ID') - ax.set_ylabel('Meantime per event [us]') - - ax.xaxis.set_major_locator(MaxNLocator(integer=True, nbins=30)) - - ax.legend(prop={'family': 'monospace'}, - labels=curr_df.columns.values, bbox_to_anchor=(1.2,1)) - - plt.subplots_adjust(top=0.95) - plt.savefig('{}.png'.format(graph_type), bbox_inches='tight') - -# Writes a file that contains commit id of all configurations shown in the -# plots -def create_metadata_file(res_dir): - list_ = [] - for dirname, dirnames, res_files in os.walk('./'+res_dir): - if len(dirnames) > 0: - continue - try: - metadata = pd.read_csv(os.path.join(dirname, 'metadata.csv')) - except Exception: - print('Omitting run {} because metadata.csv is missing'.format(dirname)) - continue - list_.append(metadata) - - df = pd.concat(list_) - df.index=df.build_id - df.sort_index(inplace=True) - df.to_csv('metadata.csv', index=False) - -#Iterates over a result directory and creates the plots for the different -#testcases -def create_plots(res_dir): - df = pd.DataFrame() - metadata_df = pd.DataFrame() - list_ = [] - for dirname, dirnames, res_files in os.walk('./'+res_dir): - if len(dirnames) > 0: - continue - try: - metadata = pd.read_csv(os.path.join(dirname, 'metadata.csv')) - except Exception: - print('Omitting run {} because metadata.csv is missing'.format(dirname)) - continue - - for res in res_files: - if res in 'metadata.csv': - continue - tmp = pd.read_csv(os.path.join(dirname, res)) - #Use the build id as the index for the dataframe for filtering - tmp.index = metadata.build_id - #Add the testcase name to the row for later filtering - tmp['testcase'] = res.split('.')[0] - list_.append(tmp) - - df = pd.concat(list_) - df = convert_us_to_ns(df) - df = rename_cols(df) - df.sort_index(inplace=True) - - #Go over the entire dataframe by testcase and create a plot for each type - for testcase in df.testcase.unique(): - df_testcase = df.loc[df['testcase'] == testcase] - create_plot(df=df_testcase, graph_type=testcase) - -def main(): - res_path = sys.argv[1] - create_plots(os.path.join(res_path)) - create_metadata_file(os.path.join(res_path)) - -if __name__ == '__main__': - main() diff --git a/scripts/lttng-baremetal-tests/generate-properties-master.sh b/scripts/lttng-baremetal-tests/generate-properties-master.sh deleted file mode 100644 index 733b6ed..0000000 --- a/scripts/lttng-baremetal-tests/generate-properties-master.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -xeu -# Copyright (C) 2016 - Francis Deslauriers -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -touch properties.txt - -KERNEL_COMMIT_ID=$kernel_tag_id -LTTNG_MODULES_COMMIT_ID=$modules_commit_id -LTTNG_TOOLS_COMMIT_ID=$tools_commit_id -LTTNG_UST_COMMIT_ID=$ust_commit_id - -LTTNG_CI_PATH="$WORKSPACE/src/lttng-ci" -echo "LTTNG_CI_PATH=$LTTNG_CI_PATH" >> properties.txt - -echo "KERNEL_COMMIT_ID=$KERNEL_COMMIT_ID" >> properties.txt -echo "LTTNG_MODULES_COMMIT_ID=$LTTNG_MODULES_COMMIT_ID" >> properties.txt -echo "LTTNG_TOOLS_COMMIT_ID=$LTTNG_TOOLS_COMMIT_ID" >> properties.txt -echo "LTTNG_UST_COMMIT_ID=$LTTNG_UST_COMMIT_ID" >> properties.txt - -BASE_STORAGE_FOLDER="/storage/jenkins-lava/baremetal-tests" - -echo "BASE_STORAGE_FOLDER=$BASE_STORAGE_FOLDER" >> properties.txt -echo "STORAGE_HOST=storage.internal.efficios.com" >> properties.txt -echo "STORAGE_USER=jenkins-lava" >> properties.txt - -echo "BUILD_DEVICE=$BUILD_DEVICE" >> properties.txt -echo "KGITREPO=$kernel_repo" >> properties.txt -echo "STORAGE_KERNEL_FOLDER=$BASE_STORAGE_FOLDER/kernel" >> properties.txt -echo "STORAGE_KERNEL_IMAGE=$BASE_STORAGE_FOLDER/kernel/$KERNEL_COMMIT_ID.$BUILD_DEVICE.bzImage" >> properties.txt -echo "STORAGE_LINUX_MODULES=$BASE_STORAGE_FOLDER/modules/linux/$KERNEL_COMMIT_ID.$BUILD_DEVICE.linux.modules.tar.gz" >> properties.txt -echo "STORAGE_LTTNG_MODULES=$BASE_STORAGE_FOLDER/modules/lttng/i$KERNEL_COMMIT_ID-$LTTNG_MODULES_COMMIT_ID.$BUILD_DEVICE.lttng.modules.tar.gz" >> properties.txt diff --git a/scripts/lttng-baremetal-tests/generate-properties-slave.sh b/scripts/lttng-baremetal-tests/generate-properties-slave.sh deleted file mode 100644 index d82119c..0000000 --- a/scripts/lttng-baremetal-tests/generate-properties-slave.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -xeu -# Copyright (C) 2016 - Francis Deslauriers -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -touch properties.txt - -# Use all CPU cores -NPROC=$(nproc) -echo "NPROC=$NPROC" >> properties.txt - -LTTNG_CI_PATH="$WORKSPACE/src/lttng-ci" -LINUX_PATH="$WORKSPACE/src/linux" -LTTNG_MODULES_PATH="$WORKSPACE/src/lttng-modules" - -echo "LTTNG_MODULES_GIT=$LTTNG_MODULES_REPO" >> properties.txt -echo "LTTNG_CI_PATH=$LTTNG_CI_PATH" >> properties.txt -echo "LINUX_PATH=$LINUX_PATH" >> properties.txt -echo "LTTNG_MODULES_PATH=$LTTNG_MODULES_PATH" >> properties.txt - -DEPLOYDIR="$WORKSPACE/deploy" -MODULES_INSTALL_FOLDER="$DEPLOYDIR/modules" - -echo "DEPLOYDIR=$DEPLOYDIR" >> properties.txt -echo "MODULES_INSTALL_FOLDER=$MODULES_INSTALL_FOLDER" >> properties.txt - -BUILD_NAME="$KERNEL_COMMIT_ID-$LTTNG_MODULES_COMMIT_ID" - -echo "KERNEL_COMMIT_ID=$KERNEL_COMMIT_ID" >> properties.txt -echo "LTTNG_MODULES_COMMIT_ID=$LTTNG_MODULES_COMMIT_ID" >> properties.txt -echo "BUILD_NAME=$BUILD_NAME" >> properties.txt -echo "BUILD_DEVICE=$BUILD_DEVICE" >> properties.txt - -echo "STORAGE_KERNEL_MODULE_SYMVERS=$STORAGE_KERNEL_FOLDER/symvers/$KERNEL_COMMIT_ID.$BUILD_DEVICE.symvers" >>properties.txt -echo "STORAGE_KERNEL_CONFIG=$STORAGE_KERNEL_FOLDER/config/$KERNEL_COMMIT_ID.$BUILD_DEVICE.config" >> properties.txt - -echo "STORAGE_HOST=storage.internal.efficios.com" >> properties.txt -echo "STORAGE_USER=jenkins-lava" >> properties.txt - -echo SSH_COMMAND="ssh -oStrictHostKeyChecking=no -i $identity_file" >> properties.txt -echo SCP_COMMAND="scp -oStrictHostKeyChecking=no -i $identity_file" >> properties.txt diff --git a/scripts/lttng-baremetal-tests/inject-ssh-commands.sh b/scripts/lttng-baremetal-tests/inject-ssh-commands.sh deleted file mode 100644 index 74e4dde..0000000 --- a/scripts/lttng-baremetal-tests/inject-ssh-commands.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -xeu -# Copyright (C) 2017 - Francis Deslauriers -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -echo SSH_COMMAND="ssh -oStrictHostKeyChecking=no -i $identity_file" >> properties.txt -echo SCP_COMMAND="scp -oStrictHostKeyChecking=no -i $identity_file" >> properties.txt diff --git a/scripts/lttng-baremetal-tests/kprobe-fuzzing.sh b/scripts/lttng-baremetal-tests/kprobe-fuzzing.sh deleted file mode 100755 index fab987b..0000000 --- a/scripts/lttng-baremetal-tests/kprobe-fuzzing.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -xeu -# Copyright (C) 2017 - Francis Deslauriers -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -NB_KPROBE_PER_ITER=500 -SESSION_NAME="my_kprobe_session" - -# Silence the script to avoid redirection of kallsyms to fill the screen -set +x -syms=$(awk '{print $3;}' /proc/kallsyms | sort -R) -nb_syms=$(echo "$syms" | wc -l) -set -x - -# Loop over the list of symbols and enable the symbols in groups of -# $NB_KPROBE_PER_ITER -for i in $(seq 0 "$NB_KPROBE_PER_ITER" "$nb_syms"); do - # Print time in UTC at each iteration to easily see when the script - # hangs - date --utc - - # Pick $NB_KPROBE_PER_ITER symbols to instrument, craft enable-event - # command and save them to a file. We craft the commands and executed - # them in two steps so that the pipeline can be done without the bash - # '-x' option that would fill the serial buffer because of the multiple - # pipe redirections. - set +x - echo "$syms" | head -n $((i+NB_KPROBE_PER_ITER)) | tail -n $NB_KPROBE_PER_ITER |awk '{print "lttng enable-event --kernel --function=" $1 " " $1}' > lttng-enable-event.sh - set -x - - # Print what iteration we are at - echo "$i" $((i+NB_KPROBE_PER_ITER)) - - # Destroy previous session and create a new one - lttng create "$SESSION_NAME" - - # Expect commands to fail, turn off early exit of shell script on - # non-zero return value - set +e - source ./lttng-enable-event.sh - set -e - - # Run stress util to generate some kernel activity - stress --cpu 2 --io 4 --vm 2 --vm-bytes 128M --hdd 3 --timeout 5s - - lttng list "$SESSION_NAME" - lttng destroy "$SESSION_NAME" -done diff --git a/scripts/lttng-baremetal-tests/lava-submit.py b/scripts/lttng-baremetal-tests/lava-submit.py deleted file mode 100644 index 5d92177..0000000 --- a/scripts/lttng-baremetal-tests/lava-submit.py +++ /dev/null @@ -1,490 +0,0 @@ -#!/usr/bin/python -# Copyright (C) 2016 - Francis Deslauriers -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import argparse -import base64 -import json -import os -import sys -import time -import xmlrpc.client -from collections import OrderedDict -from enum import Enum - -USERNAME = 'frdeso' -HOSTNAME = 'lava-master.internal.efficios.com' -SCP_PATH = 'scp://jenkins-lava@storage.internal.efficios.com' - -class TestType(Enum): - baremetal_benchmarks=1 - baremetal_tests=2 - kvm_tests=3 - kvm_fuzzing_tests=4 - -def get_job_bundle_content(server, job): - try: - bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1'] - bundle = server.dashboard.get(bundle_sha) - except xmlrpc.client.Fault as f: - print('Error while fetching results bundle', f.faultString) - raise f - - return json.loads(bundle['content']) - -# Parse the results bundle to see the run-tests testcase -# of the lttng-kernel-tests passed successfully -def check_job_all_test_cases_state_count(server, job): - content = get_job_bundle_content(server, job) - - # FIXME:Those tests are part of the boot actions and fail randomly but - # doesn't affect the behaviour of the tests. We should update our Lava - # installation and try to reproduce it. This error was encountered on - # Ubuntu 16.04. - tests_known_to_fail=['mount', 'df', 'ls', 'ip', 'wait_for_test_image_prompt'] - - passed_tests=0 - failed_tests=0 - for run in content['test_runs']: - for result in run['test_results']: - if 'test_case_id' in result : - if result['result'] in 'pass': - passed_tests+=1 - elif result['test_case_id'] in tests_known_to_fail: - pass - else: - failed_tests+=1 - return (passed_tests, failed_tests) - -# Get the benchmark results from the lava bundle -# save them as CSV files localy -def fetch_benchmark_results(server, job): - content = get_job_bundle_content(server, job) - testcases = ['processed_results_close.csv', - 'processed_results_ioctl.csv', - 'processed_results_open_efault.csv', - 'processed_results_open_enoent.csv', - 'processed_results_dup_close.csv', - 'processed_results_raw_syscall_getpid.csv', - 'processed_results_lttng_test_filter.csv'] - - # The result bundle is a large JSON containing the results of every testcase - # of the LAVA job as well as the files that were attached during the run. - # We need to iterate over this JSON to get the base64 representation of the - # benchmark results produced during the run. - for run in content['test_runs']: - # We only care of the benchmark testcases - if 'benchmark-' in run['test_id']: - if 'test_results' in run: - for res in run['test_results']: - if 'attachments' in res: - for a in res['attachments']: - # We only save the results file - if a['pathname'] in testcases: - with open(a['pathname'],'wb') as f: - # Convert the b64 representation of the - # result file and write it to a file - # in the current working directory - f.write(base64.b64decode(a['content'])) - -# Parse the attachment of the testcase to fetch the stdout of the test suite -def print_test_output(server, job): - content = get_job_bundle_content(server, job) - found = False - - for run in content['test_runs']: - if run['test_id'] in 'lttng-kernel-test': - for attachment in run['attachments']: - if attachment['pathname'] in 'stdout.log': - - # Decode the base64 file and split on newlines to iterate - # on list - testoutput = str(base64.b64decode(bytes(attachment['content'], encoding='UTF-8'))).split('\n') - - # Create a generator to iterate on the lines and keeping - # the state of the iterator across the two loops. - testoutput_iter = iter(testoutput) - for line in testoutput_iter: - - # Find the header of the test case and start printing - # from there - if 'LAVA_SIGNAL_STARTTC run-tests' in line: - found = True - print('---- TEST SUITE OUTPUT BEGIN ----') - for line in testoutput_iter: - if 'LAVA_SIGNAL_ENDTC run-tests' not in line: - print(line) - else: - # Print until we reach the end of the - # section - break - - if found is True: - print('----- TEST SUITE OUTPUT END -----') - break - -def create_new_job(name, build_device): - job = OrderedDict({ - 'health_check': False, - 'job_name': name, - 'device_type': build_device, - 'tags': [ ], - 'timeout': 7200, - 'actions': [] - }) - if build_device in 'x86': - job['tags'].append('dev-sda1') - - return job - -def get_boot_cmd(): - command = OrderedDict({ - 'command': 'boot_image' - }) - return command - -def get_config_cmd(build_device): - packages=['bsdtar', 'psmisc', 'wget', 'python3', 'python3-pip', \ - 'libglib2.0-dev', 'libffi-dev', 'elfutils', 'libdw-dev', \ - 'libelf-dev', 'libmount-dev', 'libxml2', 'libpfm4-dev', \ - 'libnuma-dev', 'python3-dev', 'swig', 'stress'] - command = OrderedDict({ - 'command': 'lava_command_run', - 'parameters': { - 'commands': [ - 'cat /etc/resolv.conf', - 'echo nameserver 172.18.0.12 > /etc/resolv.conf', - 'groupadd tracing' - ], - 'timeout':300 - } - }) - if build_device in 'x86': - command['parameters']['commands'].extend([ - 'mount /dev/sda1 /tmp', - 'rm -rf /tmp/*']) - - command['parameters']['commands'].extend([ - 'depmod -a', - 'locale-gen en_US.UTF-8', - 'apt-get update', - 'apt-get upgrade', - 'apt-get install -y {}'.format(' '.join(packages)) - ]) - return command - -def get_baremetal_benchmarks_cmd(): - command = OrderedDict({ - 'command': 'lava_test_shell', - 'parameters': { - 'testdef_repos': [ - { - 'git-repo': 'https://github.com/lttng/lttng-ci.git', - 'revision': 'master', - 'testdef': 'lava/baremetal-tests/failing-close.yml' - }, - { - 'git-repo': 'https://github.com/lttng/lttng-ci.git', - 'revision': 'master', - 'testdef': 'lava/baremetal-tests/failing-ioctl.yml' - }, - { - 'git-repo': 'https://github.com/lttng/lttng-ci.git', - 'revision': 'master', - 'testdef': 'lava/baremetal-tests/failing-open-efault.yml' - }, - { - 'git-repo': 'https://github.com/lttng/lttng-ci.git', - 'revision': 'master', - 'testdef': 'lava/baremetal-tests/success-dup-close.yml' - }, - { - 'git-repo': 'https://github.com/lttng/lttng-ci.git', - 'revision': 'master', - 'testdef': 'lava/baremetal-tests/raw-syscall-getpid.yml' - }, - { - 'git-repo': 'https://github.com/lttng/lttng-ci.git', - 'revision': 'master', - 'testdef': 'lava/baremetal-tests/failing-open-enoent.yml' - }, - { - 'git-repo': 'https://github.com/lttng/lttng-ci.git', - 'revision': 'master', - 'testdef': 'lava/baremetal-tests/lttng-test-filter.yml' - } - ], - 'timeout': 7200 - } - }) - return command - -def get_baremetal_tests_cmd(): - command = OrderedDict({ - 'command': 'lava_test_shell', - 'parameters': { - 'testdef_repos': [ - { - 'git-repo': 'https://github.com/lttng/lttng-ci.git', - 'revision': 'master', - 'testdef': 'lava/baremetal-tests/perf-tests.yml' - } - ], - 'timeout': 3600 - } - }) - return command - -def get_kvm_tests_cmd(): - command = OrderedDict({ - 'command': 'lava_test_shell', - 'parameters': { - 'testdef_repos': [ - { - 'git-repo': 'https://github.com/lttng/lttng-ci.git', - 'revision': 'master', - 'testdef': 'lava/baremetal-tests/kernel-tests.yml' - }, - { - 'git-repo': 'https://github.com/lttng/lttng-ci.git', - 'revision': 'master', - 'testdef': 'lava/baremetal-tests/destructive-tests.yml' - } - ], - 'timeout': 7200 - } - }) - return command -def get_kprobes_test_cmd(): - command = OrderedDict({ - 'command': 'lava_test_shell', - 'parameters': { - 'testdef_repos': [ - { - 'git-repo': 'https://github.com/lttng/lttng-ci.git', - 'revision': 'master', - 'testdef': 'lava/baremetal-tests/kprobe-fuzzing-tests.yml' - } - ], - 'timeout': 7200 - } - }) - return command - -def get_results_cmd(stream_name): - command = OrderedDict({ - 'command': 'submit_results', - 'parameters': { - 'server': 'http://lava-master.internal.efficios.com/RPC2/' - } - }) - command['parameters']['stream']='/anonymous/'+stream_name+'/' - return command - -def get_deploy_cmd_kvm(jenkins_job, kernel_path, linux_modules_path, lttng_modules_path): - command = OrderedDict({ - 'command': 'deploy_kernel', - 'metadata': {}, - 'parameters': { - 'customize': {}, - 'kernel': None, - 'target_type': 'ubuntu', - 'rootfs': 'file:///var/lib/lava-server/default/media/images/xenial.img.gz', - 'login_prompt': 'kvm02 login:', - 'username': 'root' - } - }) - - command['parameters']['customize'][SCP_PATH+linux_modules_path]=['rootfs:/','archive'] - command['parameters']['customize'][SCP_PATH+lttng_modules_path]=['rootfs:/','archive'] - command['parameters']['kernel'] = str(SCP_PATH+kernel_path) - command['metadata']['jenkins_jobname'] = jenkins_job - - return command - -def get_deploy_cmd_x86(jenkins_job, kernel_path, linux_modules_path, lttng_modules_path, nb_iter=None): - command = OrderedDict({ - 'command': 'deploy_kernel', - 'metadata': {}, - 'parameters': { - 'overlays': [], - 'kernel': None, - 'nfsrootfs': str(SCP_PATH+'/storage/jenkins-lava/rootfs/rootfs_amd64_trusty_2016-02-23-1134.tar.gz'), - 'target_type': 'ubuntu' - } - }) - - command['parameters']['overlays'].append( str(SCP_PATH+linux_modules_path)) - command['parameters']['overlays'].append( str(SCP_PATH+lttng_modules_path)) - command['parameters']['kernel'] = str(SCP_PATH+kernel_path) - command['metadata']['jenkins_jobname'] = jenkins_job - if nb_iter is not None: - command['metadata']['nb_iterations'] = nb_iter - - return command - - -def get_env_setup_cmd(build_device, lttng_tools_commit, lttng_ust_commit=None): - command = OrderedDict({ - 'command': 'lava_command_run', - 'parameters': { - 'commands': [ - 'pip3 install --upgrade pip', - 'hash -r', - 'pip3 install vlttng', - ], - 'timeout': 3600 - } - }) - - vlttng_cmd = 'vlttng --jobs=$(nproc) --profile urcu-master' \ - ' --override projects.babeltrace.build-env.PYTHON=python3' \ - ' --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config' \ - ' --profile babeltrace-stable-1.4' \ - ' --profile babeltrace-python' \ - ' --profile lttng-tools-master' \ - ' --override projects.lttng-tools.checkout='+lttng_tools_commit + \ - ' --profile lttng-tools-no-man-pages' - - if lttng_ust_commit is not None: - vlttng_cmd += ' --profile lttng-ust-master ' \ - ' --override projects.lttng-ust.checkout='+lttng_ust_commit+ \ - ' --profile lttng-ust-no-man-pages' - - virtenv_path = None - if build_device in 'kvm': - virtenv_path = '/root/virtenv' - else: - virtenv_path = '/tmp/virtenv' - - vlttng_cmd += ' '+virtenv_path - - command['parameters']['commands'].append(vlttng_cmd) - command['parameters']['commands'].append('ln -s '+virtenv_path+' /root/lttngvenv') - command['parameters']['commands'].append('sync') - - return command - -def main(): - test_type = None - parser = argparse.ArgumentParser(description='Launch baremetal test using Lava') - parser.add_argument('-t', '--type', required=True) - parser.add_argument('-j', '--jobname', required=True) - parser.add_argument('-k', '--kernel', required=True) - parser.add_argument('-km', '--kmodule', required=True) - parser.add_argument('-lm', '--lmodule', required=True) - parser.add_argument('-tc', '--tools-commit', required=True) - parser.add_argument('-uc', '--ust-commit', required=False) - args = parser.parse_args() - - if args.type in 'baremetal-benchmarks': - test_type = TestType.baremetal_benchmarks - elif args.type in 'baremetal-tests': - test_type = TestType.baremetal_tests - elif args.type in 'kvm-tests': - test_type = TestType.kvm_tests - elif args.type in 'kvm-fuzzing-tests': - test_type = TestType.kvm_fuzzing_tests - else: - print('argument -t/--type {} unrecognized. Exiting...'.format(args.type)) - return -1 - - lava_api_key = None - try: - lava_api_key = os.environ['LAVA_JENKINS_TOKEN'] - except Exception as e: - print('LAVA_JENKINS_TOKEN not found in the environment variable. Exiting...', e ) - return -1 - - if test_type is TestType.baremetal_benchmarks: - j = create_new_job(args.jobname, build_device='x86') - j['actions'].append(get_deploy_cmd_x86(args.jobname, args.kernel, args.kmodule, args.lmodule)) - elif test_type is TestType.baremetal_tests: - j = create_new_job(args.jobname, build_device='x86') - j['actions'].append(get_deploy_cmd_x86(args.jobname, args.kernel, args.kmodule, args.lmodule)) - elif test_type is TestType.kvm_tests or test_type is TestType.kvm_fuzzing_tests: - j = create_new_job(args.jobname, build_device='kvm') - j['actions'].append(get_deploy_cmd_kvm(args.jobname, args.kernel, args.kmodule, args.lmodule)) - - j['actions'].append(get_boot_cmd()) - - if test_type is TestType.baremetal_benchmarks: - j['actions'].append(get_config_cmd('x86')) - j['actions'].append(get_env_setup_cmd('x86', args.tools_commit)) - j['actions'].append(get_baremetal_benchmarks_cmd()) - j['actions'].append(get_results_cmd(stream_name='benchmark-kernel')) - elif test_type is TestType.baremetal_tests: - if args.ust_commit is None: - print('Tests runs need -uc/--ust-commit options. Exiting...') - return -1 - j['actions'].append(get_config_cmd('x86')) - j['actions'].append(get_env_setup_cmd('x86', args.tools_commit, args.ust_commit)) - j['actions'].append(get_baremetal_tests_cmd()) - j['actions'].append(get_results_cmd(stream_name='tests-kernel')) - elif test_type is TestType.kvm_tests: - if args.ust_commit is None: - print('Tests runs need -uc/--ust-commit options. Exiting...') - return -1 - j['actions'].append(get_config_cmd('kvm')) - j['actions'].append(get_env_setup_cmd('kvm', args.tools_commit, args.ust_commit)) - j['actions'].append(get_kvm_tests_cmd()) - j['actions'].append(get_results_cmd(stream_name='tests-kernel')) - elif test_type is TestType.kvm_fuzzing_tests: - if args.ust_commit is None: - print('Tests runs need -uc/--ust-commit options. Exiting...') - return -1 - j['actions'].append(get_config_cmd('kvm')) - j['actions'].append(get_env_setup_cmd('kvm', args.tools_commit, args.ust_commit)) - j['actions'].append(get_kprobes_test_cmd()) - j['actions'].append(get_results_cmd(stream_name='tests-kernel')) - else: - assert False, 'Unknown test type' - - server = xmlrpc.client.ServerProxy('http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME)) - - jobid = server.scheduler.submit_job(json.dumps(j)) - - print('Lava jobid:{}'.format(jobid)) - print('Lava job URL: http://lava-master.internal.efficios.com/scheduler/job/{}/log_file'.format(jobid)) - - #Check the status of the job every 30 seconds - jobstatus = server.scheduler.job_status(jobid)['job_status'] - not_running = False - while jobstatus in 'Submitted' or jobstatus in 'Running': - if not_running is False and jobstatus in 'Running': - print('Job started running') - not_running = True - time.sleep(30) - jobstatus = server.scheduler.job_status(jobid)['job_status'] - - if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests: - print_test_output(server, jobid) - elif test_type is TestType.baremetal_benchmarks: - fetch_benchmark_results(server, jobid) - - print('Job ended with {} status.'.format(jobstatus)) - if jobstatus not in 'Complete': - return -1 - else: - passed, failed=check_job_all_test_cases_state_count(server, jobid) - print('With {} passed and {} failed Lava test cases.'.format(passed, failed)) - - if failed == 0: - return 0 - else: - return -1 - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/lttng-baremetal-tests/parse-results.py b/scripts/lttng-baremetal-tests/parse-results.py deleted file mode 100755 index b518436..0000000 --- a/scripts/lttng-baremetal-tests/parse-results.py +++ /dev/null @@ -1,92 +0,0 @@ -#! /usr/bin/python3 -from subprocess import call -from collections import defaultdict -import csv -import numpy as np -import pandas as pd -import sys - -def test_case(df): - # Duration is in usec - # usecPecIter = Duration/(average number of iteration per thread) - df['usecperiter'] = (df['nbthreads'] * df['duration']) / df['nbiter'] - - periter_mean = pd.DataFrame({'periter_mean' : - df.groupby(['nbthreads', 'tracer', 'testcase','sleeptime'])['usecperiter'].mean()}).reset_index() - - periter_stdev = pd.DataFrame({'periter_stdev' : - df.groupby(['nbthreads', 'tracer', 'testcase','sleeptime'])['usecperiter'].std()}).reset_index() - - nbiter_mean = pd.DataFrame({'nbiter_mean' : - df.groupby(['nbthreads', 'tracer', 'testcase','sleeptime'])['nbiter'].mean()}).reset_index() - - nbiter_stdev = pd.DataFrame({'nbiter_stdev' : - df.groupby(['nbthreads', 'tracer', 'testcase','sleeptime'])['nbiter'].std()}).reset_index() - - duration_mean = pd.DataFrame({'duration_mean' : - df.groupby(['nbthreads', 'tracer', 'testcase','sleeptime'])['duration'].mean()}).reset_index() - - duration_stdev = pd.DataFrame({'duration_stdev' : - df.groupby(['nbthreads', 'tracer', 'testcase','sleeptime'])['duration'].std()}).reset_index() - - tmp = periter_mean.merge(periter_stdev) - - tmp = tmp.merge(nbiter_mean) - tmp = tmp.merge(nbiter_stdev) - - tmp = tmp.merge(duration_mean) - tmp = tmp.merge(duration_stdev) - - # if there is any NaN or None value in the DF we raise an exeception - if tmp.isnull().values.any(): - raise Exception('NaN value found in dataframe') - - for i, row in tmp.iterrows(): - testcase_name='_'.join([row['tracer'],str(row['nbthreads'])+'thr', 'peritermean']) - yield( {"name": testcase_name, "result": "pass", "units": "usec/iter", - "measurement": str(row['periter_mean'])}) - - testcase_name='_'.join([row['tracer'],str(row['nbthreads'])+'thr', 'periterstdev']) - yield( {"name": testcase_name, "result": "pass", "units": "usec/iter", - "measurement": str(row['periter_stdev'])}) - - testcase_name='_'.join([row['tracer'],str(row['nbthreads'])+'thr', 'nbitermean']) - yield( {"name": testcase_name, "result": "pass", "units": "iterations", - "measurement": str(row['nbiter_mean'])}) - - testcase_name='_'.join([row['tracer'],str(row['nbthreads'])+'thr', 'nbiterstdev']) - yield( {"name": testcase_name, "result": "pass", "units": "iterations", - "measurement": str(row['nbiter_stdev'])}) - - testcase_name='_'.join([row['tracer'],str(row['nbthreads'])+'thr', 'durationmean']) - yield( {"name": testcase_name, "result": "pass", "units": "usec", - "measurement": str(row['duration_mean'])}) - - testcase_name='_'.join([row['tracer'],str(row['nbthreads'])+'thr', 'durationstdev']) - yield( {"name": testcase_name, "result": "pass", "units": "usec", - "measurement": str(row['duration_stdev'])}) - -def main(): - results_file=sys.argv[1] - df = pd.read_csv(results_file) - results=defaultdict() - data = test_case(df) - for res in data: - call( - ['lava-test-case', - res['name'], - '--result', res['result'], - '--measurement', res['measurement'], - '--units', res['units']]) - - # Save the results to write to the CSV file - results[res['name']]=res['measurement'] - - # Write the dictionnary to a csv file where each key is a column - with open('processed_results.csv', 'w') as output_csv: - dict_csv_write=csv.DictWriter(output_csv, results.keys()) - dict_csv_write.writeheader() - dict_csv_write.writerow(results) - -if __name__ == '__main__': - main() diff --git a/scripts/lttng-baremetal-tests/run-baremetal-benchmarks.sh b/scripts/lttng-baremetal-tests/run-baremetal-benchmarks.sh deleted file mode 100644 index aa6be86..0000000 --- a/scripts/lttng-baremetal-tests/run-baremetal-benchmarks.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -xeu -# Copyright (C) 2016 - Francis Deslauriers -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -echo 'At this point, we built the modules and kernel if we needed to.' -echo 'We can now launch the lava job using those artefacts' - -git clone https://github.com/lttng/lttng-ci "$LTTNG_CI_PATH" - -python3 -u "$LTTNG_CI_PATH"/scripts/lttng-baremetal-tests/lava-submit.py \ - -t baremetal-benchmarks \ - -j "$JOB_NAME" \ - -k "$STORAGE_KERNEL_IMAGE" \ - -km "$STORAGE_LINUX_MODULES" \ - -lm "$STORAGE_LTTNG_MODULES" \ - -tc "$LTTNG_TOOLS_COMMIT_ID" - -# Create a results folder for this job -RESULT_STORAGE_FOLDER="$BASE_STORAGE_FOLDER/benchmark-results/$JOB_NAME/$BUILD_NUMBER" -$SSH_COMMAND "$STORAGE_USER@$STORAGE_HOST" mkdir -p "$RESULT_STORAGE_FOLDER" - -# Create a metadata file for this job containing the build_id, timestamp and the commit ids -TIMESTAMP=$(/bin/date --iso-8601=seconds) -LTTNG_CI_COMMIT_ID="$(git --git-dir="$LTTNG_CI_PATH"/.git/ --work-tree="$LTTNG_CI_PATH" rev-parse --short HEAD)" - -echo "build_id,timestamp,kernel_commit,modules_commit,tools_commit,ci_commit" > metadata.csv -echo "$BUILD_NUMBER,$TIMESTAMP,$KERNEL_COMMIT_ID,$LTTNG_MODULES_COMMIT_ID,$LTTNG_TOOLS_COMMIT_ID,$LTTNG_CI_COMMIT_ID" >> metadata.csv - -# Copy the result files for each benchmark and metadata on storage server -$SCP_COMMAND ./processed_results_close.csv "$STORAGE_USER@$STORAGE_HOST:$RESULT_STORAGE_FOLDER/close.csv" -$SCP_COMMAND ./processed_results_ioctl.csv "$STORAGE_USER@$STORAGE_HOST:$RESULT_STORAGE_FOLDER/ioctl.csv" -$SCP_COMMAND ./processed_results_open_efault.csv "$STORAGE_USER@$STORAGE_HOST:$RESULT_STORAGE_FOLDER/open-efault.csv" -$SCP_COMMAND ./processed_results_open_enoent.csv "$STORAGE_USER@$STORAGE_HOST:$RESULT_STORAGE_FOLDER/open-enoent.csv" -$SCP_COMMAND ./processed_results_dup_close.csv "$STORAGE_USER@$STORAGE_HOST:$RESULT_STORAGE_FOLDER/dup-close.csv" -$SCP_COMMAND ./processed_results_lttng_test_filter.csv "$STORAGE_USER@$STORAGE_HOST:$RESULT_STORAGE_FOLDER/lttng-test-filter.csv" -$SCP_COMMAND ./processed_results_raw_syscall_getpid.csv "$STORAGE_USER@$STORAGE_HOST:$RESULT_STORAGE_FOLDER/raw_syscall_getpid.csv" -$SCP_COMMAND ./metadata.csv "$STORAGE_USER@$STORAGE_HOST:$RESULT_STORAGE_FOLDER/metadata.csv" diff --git a/scripts/lttng-baremetal-tests/run-baremetal-tests.sh b/scripts/lttng-baremetal-tests/run-baremetal-tests.sh deleted file mode 100644 index 2f9eee8..0000000 --- a/scripts/lttng-baremetal-tests/run-baremetal-tests.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -xeu -# Copyright (C) 2016 - Francis Deslauriers -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -echo 'At this point, we built the modules and kernel if we needed to.' -echo 'We can now launch the lava job using those artefacts' -git clone https://github.com/lttng/lttng-ci "$LTTNG_CI_PATH" - -python3 -u "$LTTNG_CI_PATH"/scripts/lttng-baremetal-tests/lava-submit.py \ - -t baremetal-tests \ - -j "$JOB_NAME" \ - -k "$STORAGE_KERNEL_IMAGE" \ - -km "$STORAGE_LINUX_MODULES" \ - -lm "$STORAGE_LTTNG_MODULES" \ - -tc "$LTTNG_TOOLS_COMMIT_ID" \ - -uc "$LTTNG_UST_COMMIT_ID" diff --git a/scripts/lttng-baremetal-tests/run-kvm-fuzzing-tests.sh b/scripts/lttng-baremetal-tests/run-kvm-fuzzing-tests.sh deleted file mode 100644 index a7ba525..0000000 --- a/scripts/lttng-baremetal-tests/run-kvm-fuzzing-tests.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -xeu -# Copyright (C) 2017 - Francis Deslauriers -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -echo 'At this point, we built the modules and kernel if we needed to.' -echo 'We can now launch the lava job using those artefacts' -git clone https://github.com/lttng/lttng-ci "$LTTNG_CI_PATH" - -python3 -u "$LTTNG_CI_PATH"/scripts/lttng-baremetal-tests/lava-submit.py \ - -t kvm-fuzzing-tests \ - -j "$JOB_NAME" \ - -k "$STORAGE_KERNEL_IMAGE" \ - -km "$STORAGE_LINUX_MODULES" \ - -lm "$STORAGE_LTTNG_MODULES" \ - -tc "$LTTNG_TOOLS_COMMIT_ID" \ - -uc "$LTTNG_UST_COMMIT_ID" diff --git a/scripts/lttng-baremetal-tests/run-kvm-tests.sh b/scripts/lttng-baremetal-tests/run-kvm-tests.sh deleted file mode 100644 index 05eb693..0000000 --- a/scripts/lttng-baremetal-tests/run-kvm-tests.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -xeu -# Copyright (C) 2016 - Francis Deslauriers -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -echo 'At this point, we built the modules and kernel if we needed to.' -echo 'We can now launch the lava job using those artefacts' -git clone https://github.com/lttng/lttng-ci "$LTTNG_CI_PATH" - -python3 -u "$LTTNG_CI_PATH"/scripts/lttng-baremetal-tests/lava-submit.py \ - -t kvm-tests \ - -j "$JOB_NAME" \ - -k "$STORAGE_KERNEL_IMAGE" \ - -km "$STORAGE_LINUX_MODULES" \ - -lm "$STORAGE_LTTNG_MODULES" \ - -tc "$LTTNG_TOOLS_COMMIT_ID" \ - -uc "$LTTNG_UST_COMMIT_ID" diff --git a/scripts/lttng-baremetal-tests/summarize-results.sh b/scripts/lttng-baremetal-tests/summarize-results.sh deleted file mode 100644 index fdd26d4..0000000 --- a/scripts/lttng-baremetal-tests/summarize-results.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -xeu -# Copyright (C) 2017 - Francis Deslauriers -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -$SCP_COMMAND -r "$STORAGE_USER@$STORAGE_HOST:$BASE_STORAGE_FOLDER/benchmark-results/$JOB_NAME/" ./plot-data/ - -PYTHON3="python3" - -PYENV_HOME=$WORKSPACE/.pyenv/ - -# Delete previously built virtualenv -if [ -d "$PYENV_HOME" ]; then - rm -rf "$PYENV_HOME" -fi - -# Create virtualenv and install necessary packages -virtualenv -p $PYTHON3 "$PYENV_HOME" - -set +ux -. "$PYENV_HOME/bin/activate" -set -ux - -pip install pandas -pip install matplotlib - -python3 "$LTTNG_CI_PATH"/scripts/lttng-baremetal-tests/generate-plots.py ./plot-data/ diff --git a/scripts/system-tests/build-kernel.sh b/scripts/system-tests/build-kernel.sh new file mode 100644 index 0000000..7f0cca7 --- /dev/null +++ b/scripts/system-tests/build-kernel.sh @@ -0,0 +1,31 @@ +#!/bin/bash -xeu +# Copyright (C) 2016 - Francis Deslauriers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +echo 'kernel-built.txt does not exist' +echo 'So we build it' + +make --directory="$LINUX_PATH" "-j$NPROC" bzImage modules +make --directory="$LINUX_PATH" INSTALL_MOD_PATH="$MODULES_INSTALL_FOLDER" modules_install + +cp "$LINUX_PATH"/arch/x86/boot/bzImage "$DEPLOYDIR"/"$KERNEL_COMMIT_ID".bzImage +cp "$LINUX_PATH"/.config "$DEPLOYDIR"/"$KERNEL_COMMIT_ID".config + +tar -czf "$DEPLOYDIR/$KERNEL_COMMIT_ID.linux.modules.tar.gz" -C "$MODULES_INSTALL_FOLDER/" ./ + +$SCP_COMMAND "$DEPLOYDIR/$KERNEL_COMMIT_ID.bzImage" "$STORAGE_USER@$STORAGE_HOST:$STORAGE_KERNEL_IMAGE" +$SCP_COMMAND "$DEPLOYDIR/$KERNEL_COMMIT_ID.config" "$STORAGE_USER@$STORAGE_HOST:$STORAGE_KERNEL_CONFIG" +$SCP_COMMAND "$DEPLOYDIR/$KERNEL_COMMIT_ID.linux.modules.tar.gz" "$STORAGE_USER@$STORAGE_HOST:$STORAGE_LINUX_MODULES" +$SCP_COMMAND "$LINUX_PATH/Module.symvers" "$STORAGE_USER@$STORAGE_HOST:$STORAGE_KERNEL_MODULE_SYMVERS" diff --git a/scripts/system-tests/build-modules.sh b/scripts/system-tests/build-modules.sh new file mode 100644 index 0000000..04a8c0f --- /dev/null +++ b/scripts/system-tests/build-modules.sh @@ -0,0 +1,28 @@ +#!/bin/bash -xeu +# Copyright (C) 2016 - Francis Deslauriers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +echo 'modules-built.txt does not exist' +echo 'So we build them against the kernel' + +$SCP_COMMAND "$STORAGE_USER@$STORAGE_HOST:$STORAGE_KERNEL_MODULE_SYMVERS" "$LINUX_PATH/Module.symvers" + +KERNELDIR="$LINUX_PATH" make -j"$NPROC" --directory="$LTTNG_MODULES_PATH" + +KERNELDIR="$LINUX_PATH" make -j"$NPROC" --directory="$LTTNG_MODULES_PATH" modules_install INSTALL_MOD_PATH="$MODULES_INSTALL_FOLDER" + +tar -czf "$DEPLOYDIR/$BUILD_NAME.lttng.modules.tar.gz" -C "$MODULES_INSTALL_FOLDER/" ./ + +$SCP_COMMAND "$DEPLOYDIR/$BUILD_NAME.lttng.modules.tar.gz" "$STORAGE_USER@$STORAGE_HOST:$STORAGE_LTTNG_MODULES" diff --git a/scripts/system-tests/check-build-needs.sh b/scripts/system-tests/check-build-needs.sh new file mode 100644 index 0000000..625047e --- /dev/null +++ b/scripts/system-tests/check-build-needs.sh @@ -0,0 +1,66 @@ +#!/bin/bash -xeu +# Copyright (C) 2016 - Francis Deslauriers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +mkdir -p "$DEPLOYDIR" + +NEED_MODULES_BUILD=0 +NEED_KERNEL_BUILD=0 + +set +e +$SSH_COMMAND "$STORAGE_USER@$STORAGE_HOST" ls "$STORAGE_KERNEL_IMAGE" +if [ $? -ne 0 ]; then + NEED_KERNEL_BUILD=1 + # We need to build the lttng modules if the kernel has changed. + NEED_MODULES_BUILD=1 +fi + +$SSH_COMMAND "$STORAGE_USER@$STORAGE_HOST" ls "$STORAGE_LTTNG_MODULES" +if [ $? -ne 0 ]; then + NEED_MODULES_BUILD=1 +fi +set -e + +# We need to fetch the kernel source and lttng-modules to build either the +# kernel or modules +if [ $NEED_MODULES_BUILD -eq 1 ] || [ $NEED_KERNEL_BUILD -eq 1 ] ; then + + git clone "$KGITREPO" "$LINUX_PATH" + pushd "$LINUX_PATH" + git checkout "$KERNEL_COMMIT_ID" + popd + git clone "$LTTNG_MODULES_GIT" "$LTTNG_MODULES_PATH" + pushd "$LTTNG_MODULES_PATH" + git checkout "$LTTNG_MODULES_COMMIT_ID" + popd + + git clone https://github.com/lttng/lttng-ci "$LTTNG_CI_PATH" + cp "$LTTNG_CI_PATH"/lava/kernel/vanilla/x86_64_server.config "$LINUX_PATH/.config" + make --directory="$LINUX_PATH" olddefconfig + + if [ $BUILD_DEVICE = 'kvm' ] ; then + make --directory="$LINUX_PATH" kvmconfig + fi + + make --directory="$LINUX_PATH" modules_prepare +fi + +#We create files to specify what needs to be built for the subsequent build steps +if [ $NEED_MODULES_BUILD -eq 0 ] ; then + touch modules-built.txt +fi +if [ $NEED_KERNEL_BUILD -eq 0 ] ; then + touch kernel-built.txt +fi diff --git a/scripts/system-tests/generate-plots.py b/scripts/system-tests/generate-plots.py new file mode 100644 index 0000000..efc9877 --- /dev/null +++ b/scripts/system-tests/generate-plots.py @@ -0,0 +1,179 @@ +# Copyright (C) 2017 - Francis Deslauriers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import os, sys +import numpy as np +import pandas as pd + +#Set Matplotlib to use the PNG non interactive backend +import matplotlib as mpl +mpl.use('Agg') + +import matplotlib.pyplot as plt +from matplotlib.ticker import MaxNLocator +from cycler import cycler +from collections import OrderedDict + +def rename_cols(df): + new_cols = {'baseline_1thr_peritermean': 'basel_1thr', + 'baseline_2thr_peritermean': 'basel_2thr', + 'baseline_4thr_peritermean': 'basel_4thr', + 'baseline_8thr_peritermean': 'basel_8thr', + 'baseline_16thr_peritermean': 'basel_16thr', + 'lttng_1thr_peritermean': 'lttng_1thr', + 'lttng_2thr_peritermean': 'lttng_2thr', + 'lttng_4thr_peritermean': 'lttng_4thr', + 'lttng_8thr_peritermean': 'lttng_8thr', + 'lttng_16thr_peritermean': 'lttng_16thr', + 'baseline_1thr_periterstdev': 'basel_1thr_stdev', + 'baseline_2thr_periterstdev': 'basel_2thr_stdev', + 'baseline_4thr_periterstdev': 'basel_4thr_stdev', + 'baseline_8thr_periterstdev': 'basel_8thr_stdev', + 'baseline_16thr_periterstdev': 'basel_16thr_stdev', + 'lttng_1thr_periterstdev': 'lttng_1thr_stdev', + 'lttng_2thr_periterstdev': 'lttng_2thr_stdev', + 'lttng_4thr_periterstdev': 'lttng_4thr_stdev', + 'lttng_8thr_periterstdev': 'lttng_8thr_stdev', + 'lttng_16thr_periterstdev': 'lttng_16thr_stdev' + } + df.rename(columns=new_cols, inplace=True) + return df + +def convert_us_to_ns(df): + cols = [col for col in df.columns if 'periter' in col] + df[cols] = df[cols].apply(lambda x: x*1000) + return df + +def create_plot(df, graph_type): + # We map all test configurations and their + # respective color + conf_to_color = OrderedDict([ + ('basel_1thr','lightcoral'), + ('lttng_1thr','red'), + ('basel_2thr','gray'), + ('lttng_2thr','black'), + ('basel_4thr','chartreuse'), + ('lttng_4thr','forestgreen'), + ('basel_8thr','deepskyblue'), + ('lttng_8thr','mediumblue'), + ('basel_16thr','orange'), + ('lttng_16thr','saddlebrown')]) + + # We create a list for each of the subplots + baseline = [x for x in conf_to_color.keys() if 'basel' in x] + lttng = [x for x in conf_to_color.keys() if 'lttng' in x] + one_thr = [x for x in conf_to_color.keys() if '_1thr' in x] + two_thr = [x for x in conf_to_color.keys() if '_2thr' in x] + four_thr = [x for x in conf_to_color.keys() if '_4thr' in x] + eight_thr = [x for x in conf_to_color.keys() if '_8thr' in x] + sixteen_thr = [x for x in conf_to_color.keys() if '_16thr' in x] + + plots = [baseline, lttng, one_thr, two_thr, four_thr, eight_thr, sixteen_thr] + + title='Meantime per event for {} testcase'.format(graph_type) + + # Create a axe object for each sub-plots + f, arrax = plt.subplots(len(plots), sharex=True, figsize=(16, 25)) + f.suptitle(title, fontsize=20) + + for (ax, data_cols) in zip(arrax, plots): + curr_df = df[data_cols] + + stdev_cols = ['{}_stdev'.format(x) for x in data_cols] + # Extract the color for each configuration + colors = [conf_to_color[x] for x in data_cols] + + # set the color cycler for this plot + ax.set_prop_cycle(cycler('color', colors)) + + # Plot each line and its errorbars + for (data, stdev) in zip(data_cols, stdev_cols): + ax.errorbar(x=df.index.values, y=df[data].values, yerr=df[stdev].values, marker='o') + + ax.set_ylim(0) + ax.grid() + ax.set_xlabel('Jenkins Build ID') + ax.set_ylabel('Meantime per event [us]') + + ax.xaxis.set_major_locator(MaxNLocator(integer=True, nbins=30)) + + ax.legend(prop={'family': 'monospace'}, + labels=curr_df.columns.values, bbox_to_anchor=(1.2,1)) + + plt.subplots_adjust(top=0.95) + plt.savefig('{}.png'.format(graph_type), bbox_inches='tight') + +# Writes a file that contains commit id of all configurations shown in the +# plots +def create_metadata_file(res_dir): + list_ = [] + for dirname, dirnames, res_files in os.walk('./'+res_dir): + if len(dirnames) > 0: + continue + try: + metadata = pd.read_csv(os.path.join(dirname, 'metadata.csv')) + except Exception: + print('Omitting run {} because metadata.csv is missing'.format(dirname)) + continue + list_.append(metadata) + + df = pd.concat(list_) + df.index=df.build_id + df.sort_index(inplace=True) + df.to_csv('metadata.csv', index=False) + +#Iterates over a result directory and creates the plots for the different +#testcases +def create_plots(res_dir): + df = pd.DataFrame() + metadata_df = pd.DataFrame() + list_ = [] + for dirname, dirnames, res_files in os.walk('./'+res_dir): + if len(dirnames) > 0: + continue + try: + metadata = pd.read_csv(os.path.join(dirname, 'metadata.csv')) + except Exception: + print('Omitting run {} because metadata.csv is missing'.format(dirname)) + continue + + for res in res_files: + if res in 'metadata.csv': + continue + tmp = pd.read_csv(os.path.join(dirname, res)) + #Use the build id as the index for the dataframe for filtering + tmp.index = metadata.build_id + #Add the testcase name to the row for later filtering + tmp['testcase'] = res.split('.')[0] + list_.append(tmp) + + df = pd.concat(list_) + df = convert_us_to_ns(df) + df = rename_cols(df) + df.sort_index(inplace=True) + + #Go over the entire dataframe by testcase and create a plot for each type + for testcase in df.testcase.unique(): + df_testcase = df.loc[df['testcase'] == testcase] + create_plot(df=df_testcase, graph_type=testcase) + +def main(): + res_path = sys.argv[1] + create_plots(os.path.join(res_path)) + create_metadata_file(os.path.join(res_path)) + +if __name__ == '__main__': + main() diff --git a/scripts/system-tests/generate-properties-master.sh b/scripts/system-tests/generate-properties-master.sh new file mode 100644 index 0000000..733b6ed --- /dev/null +++ b/scripts/system-tests/generate-properties-master.sh @@ -0,0 +1,43 @@ +#!/bin/bash -xeu +# Copyright (C) 2016 - Francis Deslauriers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +touch properties.txt + +KERNEL_COMMIT_ID=$kernel_tag_id +LTTNG_MODULES_COMMIT_ID=$modules_commit_id +LTTNG_TOOLS_COMMIT_ID=$tools_commit_id +LTTNG_UST_COMMIT_ID=$ust_commit_id + +LTTNG_CI_PATH="$WORKSPACE/src/lttng-ci" +echo "LTTNG_CI_PATH=$LTTNG_CI_PATH" >> properties.txt + +echo "KERNEL_COMMIT_ID=$KERNEL_COMMIT_ID" >> properties.txt +echo "LTTNG_MODULES_COMMIT_ID=$LTTNG_MODULES_COMMIT_ID" >> properties.txt +echo "LTTNG_TOOLS_COMMIT_ID=$LTTNG_TOOLS_COMMIT_ID" >> properties.txt +echo "LTTNG_UST_COMMIT_ID=$LTTNG_UST_COMMIT_ID" >> properties.txt + +BASE_STORAGE_FOLDER="/storage/jenkins-lava/baremetal-tests" + +echo "BASE_STORAGE_FOLDER=$BASE_STORAGE_FOLDER" >> properties.txt +echo "STORAGE_HOST=storage.internal.efficios.com" >> properties.txt +echo "STORAGE_USER=jenkins-lava" >> properties.txt + +echo "BUILD_DEVICE=$BUILD_DEVICE" >> properties.txt +echo "KGITREPO=$kernel_repo" >> properties.txt +echo "STORAGE_KERNEL_FOLDER=$BASE_STORAGE_FOLDER/kernel" >> properties.txt +echo "STORAGE_KERNEL_IMAGE=$BASE_STORAGE_FOLDER/kernel/$KERNEL_COMMIT_ID.$BUILD_DEVICE.bzImage" >> properties.txt +echo "STORAGE_LINUX_MODULES=$BASE_STORAGE_FOLDER/modules/linux/$KERNEL_COMMIT_ID.$BUILD_DEVICE.linux.modules.tar.gz" >> properties.txt +echo "STORAGE_LTTNG_MODULES=$BASE_STORAGE_FOLDER/modules/lttng/i$KERNEL_COMMIT_ID-$LTTNG_MODULES_COMMIT_ID.$BUILD_DEVICE.lttng.modules.tar.gz" >> properties.txt diff --git a/scripts/system-tests/generate-properties-slave.sh b/scripts/system-tests/generate-properties-slave.sh new file mode 100644 index 0000000..d82119c --- /dev/null +++ b/scripts/system-tests/generate-properties-slave.sh @@ -0,0 +1,52 @@ +#!/bin/bash -xeu +# Copyright (C) 2016 - Francis Deslauriers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +touch properties.txt + +# Use all CPU cores +NPROC=$(nproc) +echo "NPROC=$NPROC" >> properties.txt + +LTTNG_CI_PATH="$WORKSPACE/src/lttng-ci" +LINUX_PATH="$WORKSPACE/src/linux" +LTTNG_MODULES_PATH="$WORKSPACE/src/lttng-modules" + +echo "LTTNG_MODULES_GIT=$LTTNG_MODULES_REPO" >> properties.txt +echo "LTTNG_CI_PATH=$LTTNG_CI_PATH" >> properties.txt +echo "LINUX_PATH=$LINUX_PATH" >> properties.txt +echo "LTTNG_MODULES_PATH=$LTTNG_MODULES_PATH" >> properties.txt + +DEPLOYDIR="$WORKSPACE/deploy" +MODULES_INSTALL_FOLDER="$DEPLOYDIR/modules" + +echo "DEPLOYDIR=$DEPLOYDIR" >> properties.txt +echo "MODULES_INSTALL_FOLDER=$MODULES_INSTALL_FOLDER" >> properties.txt + +BUILD_NAME="$KERNEL_COMMIT_ID-$LTTNG_MODULES_COMMIT_ID" + +echo "KERNEL_COMMIT_ID=$KERNEL_COMMIT_ID" >> properties.txt +echo "LTTNG_MODULES_COMMIT_ID=$LTTNG_MODULES_COMMIT_ID" >> properties.txt +echo "BUILD_NAME=$BUILD_NAME" >> properties.txt +echo "BUILD_DEVICE=$BUILD_DEVICE" >> properties.txt + +echo "STORAGE_KERNEL_MODULE_SYMVERS=$STORAGE_KERNEL_FOLDER/symvers/$KERNEL_COMMIT_ID.$BUILD_DEVICE.symvers" >>properties.txt +echo "STORAGE_KERNEL_CONFIG=$STORAGE_KERNEL_FOLDER/config/$KERNEL_COMMIT_ID.$BUILD_DEVICE.config" >> properties.txt + +echo "STORAGE_HOST=storage.internal.efficios.com" >> properties.txt +echo "STORAGE_USER=jenkins-lava" >> properties.txt + +echo SSH_COMMAND="ssh -oStrictHostKeyChecking=no -i $identity_file" >> properties.txt +echo SCP_COMMAND="scp -oStrictHostKeyChecking=no -i $identity_file" >> properties.txt diff --git a/scripts/system-tests/inject-ssh-commands.sh b/scripts/system-tests/inject-ssh-commands.sh new file mode 100644 index 0000000..74e4dde --- /dev/null +++ b/scripts/system-tests/inject-ssh-commands.sh @@ -0,0 +1,18 @@ +#!/bin/bash -xeu +# Copyright (C) 2017 - Francis Deslauriers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +echo SSH_COMMAND="ssh -oStrictHostKeyChecking=no -i $identity_file" >> properties.txt +echo SCP_COMMAND="scp -oStrictHostKeyChecking=no -i $identity_file" >> properties.txt diff --git a/scripts/system-tests/kprobe-fuzzing.sh b/scripts/system-tests/kprobe-fuzzing.sh new file mode 100755 index 0000000..fab987b --- /dev/null +++ b/scripts/system-tests/kprobe-fuzzing.sh @@ -0,0 +1,59 @@ +#!/bin/bash -xeu +# Copyright (C) 2017 - Francis Deslauriers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +NB_KPROBE_PER_ITER=500 +SESSION_NAME="my_kprobe_session" + +# Silence the script to avoid redirection of kallsyms to fill the screen +set +x +syms=$(awk '{print $3;}' /proc/kallsyms | sort -R) +nb_syms=$(echo "$syms" | wc -l) +set -x + +# Loop over the list of symbols and enable the symbols in groups of +# $NB_KPROBE_PER_ITER +for i in $(seq 0 "$NB_KPROBE_PER_ITER" "$nb_syms"); do + # Print time in UTC at each iteration to easily see when the script + # hangs + date --utc + + # Pick $NB_KPROBE_PER_ITER symbols to instrument, craft enable-event + # command and save them to a file. We craft the commands and executed + # them in two steps so that the pipeline can be done without the bash + # '-x' option that would fill the serial buffer because of the multiple + # pipe redirections. + set +x + echo "$syms" | head -n $((i+NB_KPROBE_PER_ITER)) | tail -n $NB_KPROBE_PER_ITER |awk '{print "lttng enable-event --kernel --function=" $1 " " $1}' > lttng-enable-event.sh + set -x + + # Print what iteration we are at + echo "$i" $((i+NB_KPROBE_PER_ITER)) + + # Destroy previous session and create a new one + lttng create "$SESSION_NAME" + + # Expect commands to fail, turn off early exit of shell script on + # non-zero return value + set +e + source ./lttng-enable-event.sh + set -e + + # Run stress util to generate some kernel activity + stress --cpu 2 --io 4 --vm 2 --vm-bytes 128M --hdd 3 --timeout 5s + + lttng list "$SESSION_NAME" + lttng destroy "$SESSION_NAME" +done diff --git a/scripts/system-tests/lava-submit.py b/scripts/system-tests/lava-submit.py new file mode 100644 index 0000000..5d92177 --- /dev/null +++ b/scripts/system-tests/lava-submit.py @@ -0,0 +1,490 @@ +#!/usr/bin/python +# Copyright (C) 2016 - Francis Deslauriers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import argparse +import base64 +import json +import os +import sys +import time +import xmlrpc.client +from collections import OrderedDict +from enum import Enum + +USERNAME = 'frdeso' +HOSTNAME = 'lava-master.internal.efficios.com' +SCP_PATH = 'scp://jenkins-lava@storage.internal.efficios.com' + +class TestType(Enum): + baremetal_benchmarks=1 + baremetal_tests=2 + kvm_tests=3 + kvm_fuzzing_tests=4 + +def get_job_bundle_content(server, job): + try: + bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1'] + bundle = server.dashboard.get(bundle_sha) + except xmlrpc.client.Fault as f: + print('Error while fetching results bundle', f.faultString) + raise f + + return json.loads(bundle['content']) + +# Parse the results bundle to see the run-tests testcase +# of the lttng-kernel-tests passed successfully +def check_job_all_test_cases_state_count(server, job): + content = get_job_bundle_content(server, job) + + # FIXME:Those tests are part of the boot actions and fail randomly but + # doesn't affect the behaviour of the tests. We should update our Lava + # installation and try to reproduce it. This error was encountered on + # Ubuntu 16.04. + tests_known_to_fail=['mount', 'df', 'ls', 'ip', 'wait_for_test_image_prompt'] + + passed_tests=0 + failed_tests=0 + for run in content['test_runs']: + for result in run['test_results']: + if 'test_case_id' in result : + if result['result'] in 'pass': + passed_tests+=1 + elif result['test_case_id'] in tests_known_to_fail: + pass + else: + failed_tests+=1 + return (passed_tests, failed_tests) + +# Get the benchmark results from the lava bundle +# save them as CSV files localy +def fetch_benchmark_results(server, job): + content = get_job_bundle_content(server, job) + testcases = ['processed_results_close.csv', + 'processed_results_ioctl.csv', + 'processed_results_open_efault.csv', + 'processed_results_open_enoent.csv', + 'processed_results_dup_close.csv', + 'processed_results_raw_syscall_getpid.csv', + 'processed_results_lttng_test_filter.csv'] + + # The result bundle is a large JSON containing the results of every testcase + # of the LAVA job as well as the files that were attached during the run. + # We need to iterate over this JSON to get the base64 representation of the + # benchmark results produced during the run. + for run in content['test_runs']: + # We only care of the benchmark testcases + if 'benchmark-' in run['test_id']: + if 'test_results' in run: + for res in run['test_results']: + if 'attachments' in res: + for a in res['attachments']: + # We only save the results file + if a['pathname'] in testcases: + with open(a['pathname'],'wb') as f: + # Convert the b64 representation of the + # result file and write it to a file + # in the current working directory + f.write(base64.b64decode(a['content'])) + +# Parse the attachment of the testcase to fetch the stdout of the test suite +def print_test_output(server, job): + content = get_job_bundle_content(server, job) + found = False + + for run in content['test_runs']: + if run['test_id'] in 'lttng-kernel-test': + for attachment in run['attachments']: + if attachment['pathname'] in 'stdout.log': + + # Decode the base64 file and split on newlines to iterate + # on list + testoutput = str(base64.b64decode(bytes(attachment['content'], encoding='UTF-8'))).split('\n') + + # Create a generator to iterate on the lines and keeping + # the state of the iterator across the two loops. + testoutput_iter = iter(testoutput) + for line in testoutput_iter: + + # Find the header of the test case and start printing + # from there + if 'LAVA_SIGNAL_STARTTC run-tests' in line: + found = True + print('---- TEST SUITE OUTPUT BEGIN ----') + for line in testoutput_iter: + if 'LAVA_SIGNAL_ENDTC run-tests' not in line: + print(line) + else: + # Print until we reach the end of the + # section + break + + if found is True: + print('----- TEST SUITE OUTPUT END -----') + break + +def create_new_job(name, build_device): + job = OrderedDict({ + 'health_check': False, + 'job_name': name, + 'device_type': build_device, + 'tags': [ ], + 'timeout': 7200, + 'actions': [] + }) + if build_device in 'x86': + job['tags'].append('dev-sda1') + + return job + +def get_boot_cmd(): + command = OrderedDict({ + 'command': 'boot_image' + }) + return command + +def get_config_cmd(build_device): + packages=['bsdtar', 'psmisc', 'wget', 'python3', 'python3-pip', \ + 'libglib2.0-dev', 'libffi-dev', 'elfutils', 'libdw-dev', \ + 'libelf-dev', 'libmount-dev', 'libxml2', 'libpfm4-dev', \ + 'libnuma-dev', 'python3-dev', 'swig', 'stress'] + command = OrderedDict({ + 'command': 'lava_command_run', + 'parameters': { + 'commands': [ + 'cat /etc/resolv.conf', + 'echo nameserver 172.18.0.12 > /etc/resolv.conf', + 'groupadd tracing' + ], + 'timeout':300 + } + }) + if build_device in 'x86': + command['parameters']['commands'].extend([ + 'mount /dev/sda1 /tmp', + 'rm -rf /tmp/*']) + + command['parameters']['commands'].extend([ + 'depmod -a', + 'locale-gen en_US.UTF-8', + 'apt-get update', + 'apt-get upgrade', + 'apt-get install -y {}'.format(' '.join(packages)) + ]) + return command + +def get_baremetal_benchmarks_cmd(): + command = OrderedDict({ + 'command': 'lava_test_shell', + 'parameters': { + 'testdef_repos': [ + { + 'git-repo': 'https://github.com/lttng/lttng-ci.git', + 'revision': 'master', + 'testdef': 'lava/baremetal-tests/failing-close.yml' + }, + { + 'git-repo': 'https://github.com/lttng/lttng-ci.git', + 'revision': 'master', + 'testdef': 'lava/baremetal-tests/failing-ioctl.yml' + }, + { + 'git-repo': 'https://github.com/lttng/lttng-ci.git', + 'revision': 'master', + 'testdef': 'lava/baremetal-tests/failing-open-efault.yml' + }, + { + 'git-repo': 'https://github.com/lttng/lttng-ci.git', + 'revision': 'master', + 'testdef': 'lava/baremetal-tests/success-dup-close.yml' + }, + { + 'git-repo': 'https://github.com/lttng/lttng-ci.git', + 'revision': 'master', + 'testdef': 'lava/baremetal-tests/raw-syscall-getpid.yml' + }, + { + 'git-repo': 'https://github.com/lttng/lttng-ci.git', + 'revision': 'master', + 'testdef': 'lava/baremetal-tests/failing-open-enoent.yml' + }, + { + 'git-repo': 'https://github.com/lttng/lttng-ci.git', + 'revision': 'master', + 'testdef': 'lava/baremetal-tests/lttng-test-filter.yml' + } + ], + 'timeout': 7200 + } + }) + return command + +def get_baremetal_tests_cmd(): + command = OrderedDict({ + 'command': 'lava_test_shell', + 'parameters': { + 'testdef_repos': [ + { + 'git-repo': 'https://github.com/lttng/lttng-ci.git', + 'revision': 'master', + 'testdef': 'lava/baremetal-tests/perf-tests.yml' + } + ], + 'timeout': 3600 + } + }) + return command + +def get_kvm_tests_cmd(): + command = OrderedDict({ + 'command': 'lava_test_shell', + 'parameters': { + 'testdef_repos': [ + { + 'git-repo': 'https://github.com/lttng/lttng-ci.git', + 'revision': 'master', + 'testdef': 'lava/baremetal-tests/kernel-tests.yml' + }, + { + 'git-repo': 'https://github.com/lttng/lttng-ci.git', + 'revision': 'master', + 'testdef': 'lava/baremetal-tests/destructive-tests.yml' + } + ], + 'timeout': 7200 + } + }) + return command +def get_kprobes_test_cmd(): + command = OrderedDict({ + 'command': 'lava_test_shell', + 'parameters': { + 'testdef_repos': [ + { + 'git-repo': 'https://github.com/lttng/lttng-ci.git', + 'revision': 'master', + 'testdef': 'lava/baremetal-tests/kprobe-fuzzing-tests.yml' + } + ], + 'timeout': 7200 + } + }) + return command + +def get_results_cmd(stream_name): + command = OrderedDict({ + 'command': 'submit_results', + 'parameters': { + 'server': 'http://lava-master.internal.efficios.com/RPC2/' + } + }) + command['parameters']['stream']='/anonymous/'+stream_name+'/' + return command + +def get_deploy_cmd_kvm(jenkins_job, kernel_path, linux_modules_path, lttng_modules_path): + command = OrderedDict({ + 'command': 'deploy_kernel', + 'metadata': {}, + 'parameters': { + 'customize': {}, + 'kernel': None, + 'target_type': 'ubuntu', + 'rootfs': 'file:///var/lib/lava-server/default/media/images/xenial.img.gz', + 'login_prompt': 'kvm02 login:', + 'username': 'root' + } + }) + + command['parameters']['customize'][SCP_PATH+linux_modules_path]=['rootfs:/','archive'] + command['parameters']['customize'][SCP_PATH+lttng_modules_path]=['rootfs:/','archive'] + command['parameters']['kernel'] = str(SCP_PATH+kernel_path) + command['metadata']['jenkins_jobname'] = jenkins_job + + return command + +def get_deploy_cmd_x86(jenkins_job, kernel_path, linux_modules_path, lttng_modules_path, nb_iter=None): + command = OrderedDict({ + 'command': 'deploy_kernel', + 'metadata': {}, + 'parameters': { + 'overlays': [], + 'kernel': None, + 'nfsrootfs': str(SCP_PATH+'/storage/jenkins-lava/rootfs/rootfs_amd64_trusty_2016-02-23-1134.tar.gz'), + 'target_type': 'ubuntu' + } + }) + + command['parameters']['overlays'].append( str(SCP_PATH+linux_modules_path)) + command['parameters']['overlays'].append( str(SCP_PATH+lttng_modules_path)) + command['parameters']['kernel'] = str(SCP_PATH+kernel_path) + command['metadata']['jenkins_jobname'] = jenkins_job + if nb_iter is not None: + command['metadata']['nb_iterations'] = nb_iter + + return command + + +def get_env_setup_cmd(build_device, lttng_tools_commit, lttng_ust_commit=None): + command = OrderedDict({ + 'command': 'lava_command_run', + 'parameters': { + 'commands': [ + 'pip3 install --upgrade pip', + 'hash -r', + 'pip3 install vlttng', + ], + 'timeout': 3600 + } + }) + + vlttng_cmd = 'vlttng --jobs=$(nproc) --profile urcu-master' \ + ' --override projects.babeltrace.build-env.PYTHON=python3' \ + ' --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config' \ + ' --profile babeltrace-stable-1.4' \ + ' --profile babeltrace-python' \ + ' --profile lttng-tools-master' \ + ' --override projects.lttng-tools.checkout='+lttng_tools_commit + \ + ' --profile lttng-tools-no-man-pages' + + if lttng_ust_commit is not None: + vlttng_cmd += ' --profile lttng-ust-master ' \ + ' --override projects.lttng-ust.checkout='+lttng_ust_commit+ \ + ' --profile lttng-ust-no-man-pages' + + virtenv_path = None + if build_device in 'kvm': + virtenv_path = '/root/virtenv' + else: + virtenv_path = '/tmp/virtenv' + + vlttng_cmd += ' '+virtenv_path + + command['parameters']['commands'].append(vlttng_cmd) + command['parameters']['commands'].append('ln -s '+virtenv_path+' /root/lttngvenv') + command['parameters']['commands'].append('sync') + + return command + +def main(): + test_type = None + parser = argparse.ArgumentParser(description='Launch baremetal test using Lava') + parser.add_argument('-t', '--type', required=True) + parser.add_argument('-j', '--jobname', required=True) + parser.add_argument('-k', '--kernel', required=True) + parser.add_argument('-km', '--kmodule', required=True) + parser.add_argument('-lm', '--lmodule', required=True) + parser.add_argument('-tc', '--tools-commit', required=True) + parser.add_argument('-uc', '--ust-commit', required=False) + args = parser.parse_args() + + if args.type in 'baremetal-benchmarks': + test_type = TestType.baremetal_benchmarks + elif args.type in 'baremetal-tests': + test_type = TestType.baremetal_tests + elif args.type in 'kvm-tests': + test_type = TestType.kvm_tests + elif args.type in 'kvm-fuzzing-tests': + test_type = TestType.kvm_fuzzing_tests + else: + print('argument -t/--type {} unrecognized. Exiting...'.format(args.type)) + return -1 + + lava_api_key = None + try: + lava_api_key = os.environ['LAVA_JENKINS_TOKEN'] + except Exception as e: + print('LAVA_JENKINS_TOKEN not found in the environment variable. Exiting...', e ) + return -1 + + if test_type is TestType.baremetal_benchmarks: + j = create_new_job(args.jobname, build_device='x86') + j['actions'].append(get_deploy_cmd_x86(args.jobname, args.kernel, args.kmodule, args.lmodule)) + elif test_type is TestType.baremetal_tests: + j = create_new_job(args.jobname, build_device='x86') + j['actions'].append(get_deploy_cmd_x86(args.jobname, args.kernel, args.kmodule, args.lmodule)) + elif test_type is TestType.kvm_tests or test_type is TestType.kvm_fuzzing_tests: + j = create_new_job(args.jobname, build_device='kvm') + j['actions'].append(get_deploy_cmd_kvm(args.jobname, args.kernel, args.kmodule, args.lmodule)) + + j['actions'].append(get_boot_cmd()) + + if test_type is TestType.baremetal_benchmarks: + j['actions'].append(get_config_cmd('x86')) + j['actions'].append(get_env_setup_cmd('x86', args.tools_commit)) + j['actions'].append(get_baremetal_benchmarks_cmd()) + j['actions'].append(get_results_cmd(stream_name='benchmark-kernel')) + elif test_type is TestType.baremetal_tests: + if args.ust_commit is None: + print('Tests runs need -uc/--ust-commit options. Exiting...') + return -1 + j['actions'].append(get_config_cmd('x86')) + j['actions'].append(get_env_setup_cmd('x86', args.tools_commit, args.ust_commit)) + j['actions'].append(get_baremetal_tests_cmd()) + j['actions'].append(get_results_cmd(stream_name='tests-kernel')) + elif test_type is TestType.kvm_tests: + if args.ust_commit is None: + print('Tests runs need -uc/--ust-commit options. Exiting...') + return -1 + j['actions'].append(get_config_cmd('kvm')) + j['actions'].append(get_env_setup_cmd('kvm', args.tools_commit, args.ust_commit)) + j['actions'].append(get_kvm_tests_cmd()) + j['actions'].append(get_results_cmd(stream_name='tests-kernel')) + elif test_type is TestType.kvm_fuzzing_tests: + if args.ust_commit is None: + print('Tests runs need -uc/--ust-commit options. Exiting...') + return -1 + j['actions'].append(get_config_cmd('kvm')) + j['actions'].append(get_env_setup_cmd('kvm', args.tools_commit, args.ust_commit)) + j['actions'].append(get_kprobes_test_cmd()) + j['actions'].append(get_results_cmd(stream_name='tests-kernel')) + else: + assert False, 'Unknown test type' + + server = xmlrpc.client.ServerProxy('http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME)) + + jobid = server.scheduler.submit_job(json.dumps(j)) + + print('Lava jobid:{}'.format(jobid)) + print('Lava job URL: http://lava-master.internal.efficios.com/scheduler/job/{}/log_file'.format(jobid)) + + #Check the status of the job every 30 seconds + jobstatus = server.scheduler.job_status(jobid)['job_status'] + not_running = False + while jobstatus in 'Submitted' or jobstatus in 'Running': + if not_running is False and jobstatus in 'Running': + print('Job started running') + not_running = True + time.sleep(30) + jobstatus = server.scheduler.job_status(jobid)['job_status'] + + if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests: + print_test_output(server, jobid) + elif test_type is TestType.baremetal_benchmarks: + fetch_benchmark_results(server, jobid) + + print('Job ended with {} status.'.format(jobstatus)) + if jobstatus not in 'Complete': + return -1 + else: + passed, failed=check_job_all_test_cases_state_count(server, jobid) + print('With {} passed and {} failed Lava test cases.'.format(passed, failed)) + + if failed == 0: + return 0 + else: + return -1 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/system-tests/parse-results.py b/scripts/system-tests/parse-results.py new file mode 100755 index 0000000..b518436 --- /dev/null +++ b/scripts/system-tests/parse-results.py @@ -0,0 +1,92 @@ +#! /usr/bin/python3 +from subprocess import call +from collections import defaultdict +import csv +import numpy as np +import pandas as pd +import sys + +def test_case(df): + # Duration is in usec + # usecPecIter = Duration/(average number of iteration per thread) + df['usecperiter'] = (df['nbthreads'] * df['duration']) / df['nbiter'] + + periter_mean = pd.DataFrame({'periter_mean' : + df.groupby(['nbthreads', 'tracer', 'testcase','sleeptime'])['usecperiter'].mean()}).reset_index() + + periter_stdev = pd.DataFrame({'periter_stdev' : + df.groupby(['nbthreads', 'tracer', 'testcase','sleeptime'])['usecperiter'].std()}).reset_index() + + nbiter_mean = pd.DataFrame({'nbiter_mean' : + df.groupby(['nbthreads', 'tracer', 'testcase','sleeptime'])['nbiter'].mean()}).reset_index() + + nbiter_stdev = pd.DataFrame({'nbiter_stdev' : + df.groupby(['nbthreads', 'tracer', 'testcase','sleeptime'])['nbiter'].std()}).reset_index() + + duration_mean = pd.DataFrame({'duration_mean' : + df.groupby(['nbthreads', 'tracer', 'testcase','sleeptime'])['duration'].mean()}).reset_index() + + duration_stdev = pd.DataFrame({'duration_stdev' : + df.groupby(['nbthreads', 'tracer', 'testcase','sleeptime'])['duration'].std()}).reset_index() + + tmp = periter_mean.merge(periter_stdev) + + tmp = tmp.merge(nbiter_mean) + tmp = tmp.merge(nbiter_stdev) + + tmp = tmp.merge(duration_mean) + tmp = tmp.merge(duration_stdev) + + # if there is any NaN or None value in the DF we raise an exeception + if tmp.isnull().values.any(): + raise Exception('NaN value found in dataframe') + + for i, row in tmp.iterrows(): + testcase_name='_'.join([row['tracer'],str(row['nbthreads'])+'thr', 'peritermean']) + yield( {"name": testcase_name, "result": "pass", "units": "usec/iter", + "measurement": str(row['periter_mean'])}) + + testcase_name='_'.join([row['tracer'],str(row['nbthreads'])+'thr', 'periterstdev']) + yield( {"name": testcase_name, "result": "pass", "units": "usec/iter", + "measurement": str(row['periter_stdev'])}) + + testcase_name='_'.join([row['tracer'],str(row['nbthreads'])+'thr', 'nbitermean']) + yield( {"name": testcase_name, "result": "pass", "units": "iterations", + "measurement": str(row['nbiter_mean'])}) + + testcase_name='_'.join([row['tracer'],str(row['nbthreads'])+'thr', 'nbiterstdev']) + yield( {"name": testcase_name, "result": "pass", "units": "iterations", + "measurement": str(row['nbiter_stdev'])}) + + testcase_name='_'.join([row['tracer'],str(row['nbthreads'])+'thr', 'durationmean']) + yield( {"name": testcase_name, "result": "pass", "units": "usec", + "measurement": str(row['duration_mean'])}) + + testcase_name='_'.join([row['tracer'],str(row['nbthreads'])+'thr', 'durationstdev']) + yield( {"name": testcase_name, "result": "pass", "units": "usec", + "measurement": str(row['duration_stdev'])}) + +def main(): + results_file=sys.argv[1] + df = pd.read_csv(results_file) + results=defaultdict() + data = test_case(df) + for res in data: + call( + ['lava-test-case', + res['name'], + '--result', res['result'], + '--measurement', res['measurement'], + '--units', res['units']]) + + # Save the results to write to the CSV file + results[res['name']]=res['measurement'] + + # Write the dictionnary to a csv file where each key is a column + with open('processed_results.csv', 'w') as output_csv: + dict_csv_write=csv.DictWriter(output_csv, results.keys()) + dict_csv_write.writeheader() + dict_csv_write.writerow(results) + +if __name__ == '__main__': + main() diff --git a/scripts/system-tests/run-baremetal-benchmarks.sh b/scripts/system-tests/run-baremetal-benchmarks.sh new file mode 100644 index 0000000..aa6be86 --- /dev/null +++ b/scripts/system-tests/run-baremetal-benchmarks.sh @@ -0,0 +1,49 @@ +#!/bin/bash -xeu +# Copyright (C) 2016 - Francis Deslauriers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +echo 'At this point, we built the modules and kernel if we needed to.' +echo 'We can now launch the lava job using those artefacts' + +git clone https://github.com/lttng/lttng-ci "$LTTNG_CI_PATH" + +python3 -u "$LTTNG_CI_PATH"/scripts/lttng-baremetal-tests/lava-submit.py \ + -t baremetal-benchmarks \ + -j "$JOB_NAME" \ + -k "$STORAGE_KERNEL_IMAGE" \ + -km "$STORAGE_LINUX_MODULES" \ + -lm "$STORAGE_LTTNG_MODULES" \ + -tc "$LTTNG_TOOLS_COMMIT_ID" + +# Create a results folder for this job +RESULT_STORAGE_FOLDER="$BASE_STORAGE_FOLDER/benchmark-results/$JOB_NAME/$BUILD_NUMBER" +$SSH_COMMAND "$STORAGE_USER@$STORAGE_HOST" mkdir -p "$RESULT_STORAGE_FOLDER" + +# Create a metadata file for this job containing the build_id, timestamp and the commit ids +TIMESTAMP=$(/bin/date --iso-8601=seconds) +LTTNG_CI_COMMIT_ID="$(git --git-dir="$LTTNG_CI_PATH"/.git/ --work-tree="$LTTNG_CI_PATH" rev-parse --short HEAD)" + +echo "build_id,timestamp,kernel_commit,modules_commit,tools_commit,ci_commit" > metadata.csv +echo "$BUILD_NUMBER,$TIMESTAMP,$KERNEL_COMMIT_ID,$LTTNG_MODULES_COMMIT_ID,$LTTNG_TOOLS_COMMIT_ID,$LTTNG_CI_COMMIT_ID" >> metadata.csv + +# Copy the result files for each benchmark and metadata on storage server +$SCP_COMMAND ./processed_results_close.csv "$STORAGE_USER@$STORAGE_HOST:$RESULT_STORAGE_FOLDER/close.csv" +$SCP_COMMAND ./processed_results_ioctl.csv "$STORAGE_USER@$STORAGE_HOST:$RESULT_STORAGE_FOLDER/ioctl.csv" +$SCP_COMMAND ./processed_results_open_efault.csv "$STORAGE_USER@$STORAGE_HOST:$RESULT_STORAGE_FOLDER/open-efault.csv" +$SCP_COMMAND ./processed_results_open_enoent.csv "$STORAGE_USER@$STORAGE_HOST:$RESULT_STORAGE_FOLDER/open-enoent.csv" +$SCP_COMMAND ./processed_results_dup_close.csv "$STORAGE_USER@$STORAGE_HOST:$RESULT_STORAGE_FOLDER/dup-close.csv" +$SCP_COMMAND ./processed_results_lttng_test_filter.csv "$STORAGE_USER@$STORAGE_HOST:$RESULT_STORAGE_FOLDER/lttng-test-filter.csv" +$SCP_COMMAND ./processed_results_raw_syscall_getpid.csv "$STORAGE_USER@$STORAGE_HOST:$RESULT_STORAGE_FOLDER/raw_syscall_getpid.csv" +$SCP_COMMAND ./metadata.csv "$STORAGE_USER@$STORAGE_HOST:$RESULT_STORAGE_FOLDER/metadata.csv" diff --git a/scripts/system-tests/run-baremetal-tests.sh b/scripts/system-tests/run-baremetal-tests.sh new file mode 100644 index 0000000..2f9eee8 --- /dev/null +++ b/scripts/system-tests/run-baremetal-tests.sh @@ -0,0 +1,28 @@ +#!/bin/bash -xeu +# Copyright (C) 2016 - Francis Deslauriers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +echo 'At this point, we built the modules and kernel if we needed to.' +echo 'We can now launch the lava job using those artefacts' +git clone https://github.com/lttng/lttng-ci "$LTTNG_CI_PATH" + +python3 -u "$LTTNG_CI_PATH"/scripts/lttng-baremetal-tests/lava-submit.py \ + -t baremetal-tests \ + -j "$JOB_NAME" \ + -k "$STORAGE_KERNEL_IMAGE" \ + -km "$STORAGE_LINUX_MODULES" \ + -lm "$STORAGE_LTTNG_MODULES" \ + -tc "$LTTNG_TOOLS_COMMIT_ID" \ + -uc "$LTTNG_UST_COMMIT_ID" diff --git a/scripts/system-tests/run-kvm-fuzzing-tests.sh b/scripts/system-tests/run-kvm-fuzzing-tests.sh new file mode 100644 index 0000000..a7ba525 --- /dev/null +++ b/scripts/system-tests/run-kvm-fuzzing-tests.sh @@ -0,0 +1,28 @@ +#!/bin/bash -xeu +# Copyright (C) 2017 - Francis Deslauriers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +echo 'At this point, we built the modules and kernel if we needed to.' +echo 'We can now launch the lava job using those artefacts' +git clone https://github.com/lttng/lttng-ci "$LTTNG_CI_PATH" + +python3 -u "$LTTNG_CI_PATH"/scripts/lttng-baremetal-tests/lava-submit.py \ + -t kvm-fuzzing-tests \ + -j "$JOB_NAME" \ + -k "$STORAGE_KERNEL_IMAGE" \ + -km "$STORAGE_LINUX_MODULES" \ + -lm "$STORAGE_LTTNG_MODULES" \ + -tc "$LTTNG_TOOLS_COMMIT_ID" \ + -uc "$LTTNG_UST_COMMIT_ID" diff --git a/scripts/system-tests/run-kvm-tests.sh b/scripts/system-tests/run-kvm-tests.sh new file mode 100644 index 0000000..05eb693 --- /dev/null +++ b/scripts/system-tests/run-kvm-tests.sh @@ -0,0 +1,28 @@ +#!/bin/bash -xeu +# Copyright (C) 2016 - Francis Deslauriers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +echo 'At this point, we built the modules and kernel if we needed to.' +echo 'We can now launch the lava job using those artefacts' +git clone https://github.com/lttng/lttng-ci "$LTTNG_CI_PATH" + +python3 -u "$LTTNG_CI_PATH"/scripts/lttng-baremetal-tests/lava-submit.py \ + -t kvm-tests \ + -j "$JOB_NAME" \ + -k "$STORAGE_KERNEL_IMAGE" \ + -km "$STORAGE_LINUX_MODULES" \ + -lm "$STORAGE_LTTNG_MODULES" \ + -tc "$LTTNG_TOOLS_COMMIT_ID" \ + -uc "$LTTNG_UST_COMMIT_ID" diff --git a/scripts/system-tests/summarize-results.sh b/scripts/system-tests/summarize-results.sh new file mode 100644 index 0000000..fdd26d4 --- /dev/null +++ b/scripts/system-tests/summarize-results.sh @@ -0,0 +1,38 @@ +#!/bin/bash -xeu +# Copyright (C) 2017 - Francis Deslauriers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +$SCP_COMMAND -r "$STORAGE_USER@$STORAGE_HOST:$BASE_STORAGE_FOLDER/benchmark-results/$JOB_NAME/" ./plot-data/ + +PYTHON3="python3" + +PYENV_HOME=$WORKSPACE/.pyenv/ + +# Delete previously built virtualenv +if [ -d "$PYENV_HOME" ]; then + rm -rf "$PYENV_HOME" +fi + +# Create virtualenv and install necessary packages +virtualenv -p $PYTHON3 "$PYENV_HOME" + +set +ux +. "$PYENV_HOME/bin/activate" +set -ux + +pip install pandas +pip install matplotlib + +python3 "$LTTNG_CI_PATH"/scripts/lttng-baremetal-tests/generate-plots.py ./plot-data/ diff --git a/scripts/system-tests/system-trigger.groovy b/scripts/system-tests/system-trigger.groovy new file mode 100644 index 0000000..394227c --- /dev/null +++ b/scripts/system-tests/system-trigger.groovy @@ -0,0 +1,462 @@ +/** + * Copyright (C) 2017 - Francis Deslauriers + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +import hudson.console.HyperlinkNote +import hudson.model.* +import java.io.File +import org.eclipse.jgit.api.Git +import org.eclipse.jgit.lib.Ref + +class InvalidKVersionException extends Exception { + public InvalidKVersionException(String message) { + super(message) + } +} + +class EmptyKVersionException extends Exception { + public EmptyKVersionException(String message) { + super(message) + } +} + +class VanillaKVersion implements Comparable { + + Integer major = 0 + Integer majorB = 0 + Integer minor = 0 + Integer patch = 0 + Integer rc = Integer.MAX_VALUE + Boolean inStable = false; + + VanillaKVersion() {} + + VanillaKVersion(version) { + this.parse(version) + } + + static VanillaKVersion minKVersion() { + return new VanillaKVersion("v0.0.0") + } + + static VanillaKVersion maxKVersion() { + return new VanillaKVersion("v" + Integer.MAX_VALUE + ".0.0") + } + + static VanillaKVersion factory(version) { + return new VanillaKVersion(version) + } + + def parse(version) { + this.major = 0 + this.majorB = 0 + this.minor = 0 + this.patch = 0 + this.rc = Integer.MAX_VALUE + + if (!version) { + throw new EmptyKVersionException("Empty kernel version") + } + + def match = version =~ /^v(\d+)\.(\d+)(\.(\d+))?(\.(\d+))?(-rc(\d+))?$/ + if (!match) { + throw new InvalidKVersionException("Invalid kernel version: ${version}") + } + + Integer offset = 0; + + // Major + this.major = Integer.parseInt(match.group(1)) + if (this.major <= 2) { + offset = 2 + this.majorB = Integer.parseInt(match.group(2)) + } + + // Minor + if (match.group(2 + offset) != null) { + this.minor = Integer.parseInt(match.group(2 + offset)) + } + + // Patch level + if (match.group(4 + offset) != null) { + this.patch = Integer.parseInt(match.group(4 + offset)) + this.inStable = true + } + + // RC + if (match.group(8) != null) { + this.rc = Integer.parseInt(match.group(8)) + } + } + + Boolean isInStableBranch() { + return this.inStable + } + + // Return true if both version are of the same stable branch + Boolean isSameStable(VanillaKVersion o) { + if (this.major != o.major) { + return false + } + if (this.majorB != o.majorB) { + return false + } + if (this.minor != o.minor) { + return false + } + + return true + } + + @Override int compareTo(VanillaKVersion o) { + if (this.major != o.major) { + return Integer.compare(this.major, o.major) + } + if (this.majorB != o.majorB) { + return Integer.compare(this.majorB, o.majorB) + } + if (this.minor != o.minor) { + return Integer.compare(this.minor, o.minor) + } + if (this.patch != o.patch) { + return Integer.compare(this.patch, o.patch) + } + if (this.rc != o.rc) { + return Integer.compare(this.rc, o.rc) + } + + // Same version + return 0; + } + + String toString() { + String vString = "v${this.major}" + + if (this.majorB > 0) { + vString = vString.concat(".${this.majorB}") + } + + vString = vString.concat(".${this.minor}") + + if (this.patch > 0) { + vString = vString.concat(".${this.patch}") + } + + if (this.rc > 0 && this.rc < Integer.MAX_VALUE) { + vString = vString.concat("-rc${this.rc}") + } + return vString + } +} + +class RunConfiguration { + def linuxBranch + def linuxTagId + def lttngBranch + def lttngModulesCommitId + def lttngToolsCommitId + def lttngUstCommitId + RunConfiguration(linuxBranch, linuxTagId, lttngBranch, lttngToolsCommitId, + lttngModulesCommitId, lttngUstCommitId) { + this.linuxBranch = linuxBranch + this.linuxTagId = linuxTagId + this.lttngBranch = lttngBranch + this.lttngModulesCommitId = lttngModulesCommitId + this.lttngToolsCommitId = lttngToolsCommitId + this.lttngUstCommitId = lttngUstCommitId + } + + String toString() { + return "${this.linuxBranch}:{${this.linuxTagId}}, ${this.lttngBranch}:{${this.lttngModulesCommitId}, ${this.lttngToolsCommitId}, ${this.lttngUstCommitId}}" + } +} + +def LoadPreviousIdsFromWorkspace = { ondiskpath -> + def previousIds = [] + try { + File myFile = new File(ondiskpath); + def input = new ObjectInputStream(new FileInputStream(ondiskpath)) + previousIds = input.readObject() + input.close() + } catch (all) { + println("Failed to load previous ids from disk.") + } + return previousIds +} + +def saveCurrentIdsToWorkspace = { currentIds, ondiskpath -> + try { + File myFile = new File(ondiskpath); + myFile.createNewFile(); + def out = new ObjectOutputStream(new FileOutputStream(ondiskpath)) + out.writeObject(currentIds) + out.close() + } catch (all) { + println("Failed to save previous ids from disk.") + } +} + +def GetHeadCommits = { remoteRepo, branchesOfInterest -> + def remoteHeads = [:] + def remoteHeadRefs = Git.lsRemoteRepository() + .setTags(false) + .setHeads(true) + .setRemote(remoteRepo).call() + + remoteHeadRefs.each { + def branch = it.getName().replaceAll('refs/heads/', '') + if (branchesOfInterest.contains(branch)) + remoteHeads[branch] = it.getObjectId().name() + } + + return remoteHeads +} + +def GetTagIds = { remoteRepo -> + def remoteTags = [:] + def remoteTagRefs = Git.lsRemoteRepository() + .setTags(true) + .setHeads(false) + .setRemote(remoteRepo).call() + + remoteTagRefs.each { + // Exclude release candidate tags + if (!it.getName().contains('-rc')) { + remoteTags[it.getName().replaceAll('refs/tags/', '')] = it.getObjectId().name() + } + } + + return remoteTags +} + +def GetLastTagOfBranch = { tagRefs, branch -> + def tagVersions = tagRefs.collect {new VanillaKVersion(it.key)} + def currMax = new VanillaKVersion('v0.0.0'); + if (!branch.contains('master')){ + def targetVersion = new VanillaKVersion(branch.replaceAll('linux-', 'v').replaceAll('.y', '')) + tagVersions.each { + if (it.isSameStable(targetVersion)) { + if (currMax < it) { + currMax = it; + } + } + } + } else { + tagVersions.each { + if (!it.isInStableBranch() && currMax < it) { + currMax = it; + } + } + } + return currMax.toString() +} + +// Returns the latest tags of each of the branches passed in the argument +def GetLastTagIds = { remoteRepo, branchesOfInterest -> + def remoteHeads = GetHeadCommits(remoteRepo, branchesOfInterest) + def remoteTagRefs = GetTagIds(remoteRepo) + def remoteLastTagCommit = [:] + + remoteTagRefs = remoteTagRefs.findAll { !it.key.contains("v2.") } + branchesOfInterest.each { + remoteLastTagCommit[it] = remoteTagRefs[GetLastTagOfBranch(remoteTagRefs, it)] + } + + return remoteLastTagCommit +} + +def CraftJobName = { jobType, runConfig -> + return "${jobType}_k${runConfig.linuxBranch}_l${runConfig.lttngBranch}" +} + +def LaunchJob = { jobName, runConfig -> + def job = Hudson.instance.getJob(jobName) + def params = [] + for (paramdef in job.getProperty(ParametersDefinitionProperty.class).getParameterDefinitions()) { + params += paramdef.getDefaultParameterValue(); + } + + params.add(new StringParameterValue('tools_commit_id', runConfig.lttngToolsCommitId)) + params.add(new StringParameterValue('modules_commit_id', runConfig.lttngModulesCommitId)) + params.add(new StringParameterValue('ust_commit_id', runConfig.lttngUstCommitId)) + params.add(new StringParameterValue('kernel_tag_id', runConfig.linuxTagId)) + job.scheduleBuild2(0, new Cause.UpstreamCause(build), new ParametersAction(params)) + println "Launching job: ${HyperlinkNote.encodeTo('/' + job.url, job.fullDisplayName)}" +} + +def jobTypes = ['baremetal_tests', 'vm_tests', 'baremetal_benchmarks'] +final String toolsRepo = "https://github.com/lttng/lttng-tools.git" +final String modulesRepo = "https://github.com/lttng/lttng-modules.git" +final String ustRepo = "https://github.com/lttng/lttng-ust.git" +final String linuxRepo = "git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git" + +final String toolsOnDiskPath = build.getEnvironment(listener).get('WORKSPACE') + "/on-disk-tools-ref" +final String modulesOnDiskPath = build.getEnvironment(listener).get('WORKSPACE') + "/on-disk-modules-ref" +final String ustOnDiskPath = build.getEnvironment(listener).get('WORKSPACE') + "/on-disk-ust-ref" +final String linuxOnDiskPath = build.getEnvironment(listener).get('WORKSPACE') + "/on-disk-linux-ref" + +def recentLttngBranchesOfInterest = ['master', 'stable-2.10', 'stable-2.9'] +def recentLinuxBranchesOfInterest = ['master', 'linux-4.9.y', 'linux-4.4.y'] + +def legacyLttngBranchesOfInterest = ['stable-2.7'] +def legacyLinuxBranchesOfInterest = ['linux-3.18.y', 'linux-4.4.y'] + +// Generate configurations of interest +def configurationOfInterest = [] as Set + +recentLttngBranchesOfInterest.each { lttngBranch -> + recentLinuxBranchesOfInterest.each { linuxBranch -> + configurationOfInterest.add([lttngBranch, linuxBranch]) + } +} + +legacyLttngBranchesOfInterest.each { lttngBranch -> + legacyLinuxBranchesOfInterest.each { linuxBranch -> + configurationOfInterest.add([lttngBranch, linuxBranch]) + } +} + +def lttngBranchesOfInterest = recentLttngBranchesOfInterest + legacyLttngBranchesOfInterest +def linuxBranchesOfInterest = recentLinuxBranchesOfInterest + legacyLinuxBranchesOfInterest + +// For Linux branches, we look for new non-RC tags +def toolsHeadCommits = GetHeadCommits(toolsRepo, lttngBranchesOfInterest) +def modulesHeadCommits = GetHeadCommits(modulesRepo, lttngBranchesOfInterest) +def ustHeadCommits = GetHeadCommits(ustRepo, lttngBranchesOfInterest) + +// For LTTng branches, we look for new commits +def linuxLastTagIds = GetLastTagIds(linuxRepo, linuxBranchesOfInterest) + +// Load previously build Linux tag ids +def oldLinuxTags = LoadPreviousIdsFromWorkspace(linuxOnDiskPath) as Set + +// Load previously built LTTng commit ids +def oldToolsHeadCommits = LoadPreviousIdsFromWorkspace(toolsOnDiskPath) as Set +def oldModulesHeadCommits = LoadPreviousIdsFromWorkspace(modulesOnDiskPath) as Set +def oldUstHeadCommits = LoadPreviousIdsFromWorkspace(ustOnDiskPath) as Set + +def newOldLinuxTags = oldLinuxTags +def newOldToolsHeadCommits = oldToolsHeadCommits +def newOldModulesHeadCommits = oldModulesHeadCommits +def newOldUstHeadCommits = oldUstHeadCommits + +def canaryRunConfigs = [] as Set +canaryRunConfigs.add( + ['v4.4.9', '1a1a512b983108015ced1e7a7c7775cfeec42d8c', 'v2.8.1','d11e0db', '7fd9215', '514a87f'] as RunConfiguration) + +def runConfigs = [] as Set + +// For each top of branch kernel tags that were not seen before, schedule one +// job for each lttng/linux tracked configurations +linuxLastTagIds.each { linuxTag -> + if (!oldLinuxTags.contains(linuxTag.value)) { + lttngBranchesOfInterest.each { lttngBranch -> + if (configurationOfInterest.contains([lttngBranch, linuxTag.key])) { + runConfigs.add([linuxTag.key, linuxTag.value, + lttngBranch, toolsHeadCommits[lttngBranch], + modulesHeadCommits[lttngBranch], ustHeadCommits[lttngBranch]] + as RunConfiguration) + + newOldLinuxTags.add(linuxTag.value) + } + } + } +} + +// For each top of branch commits that were not seen before, schedule one job +// for each lttng/linux tracked configurations +toolsHeadCommits.each { toolsHead -> + if (!oldToolsHeadCommits.contains(toolsHead.value)) { + linuxLastTagIds.each { linuxTag -> + def lttngBranch = toolsHead.key + if (configurationOfInterest.contains([lttngBranch, linuxTag.key])) { + runConfigs.add([linuxTag.key, linuxTag.value, + lttngBranch, toolsHeadCommits[lttngBranch], + modulesHeadCommits[lttngBranch], ustHeadCommits[lttngBranch]] + as RunConfiguration) + + newOldToolsHeadCommits.add(toolsHead.value) + } + } + } +} + +// For each top of branch commits that were not seen before, schedule one job +// for each lttng/linux tracked configurations +modulesHeadCommits.each { modulesHead -> + if (!oldModulesHeadCommits.contains(modulesHead.value)) { + linuxLastTagIds.each { linuxTag -> + def lttngBranch = modulesHead.key + if (configurationOfInterest.contains([lttngBranch, linuxTag.key])) { + runConfigs.add([linuxTag.key, linuxTag.value, + lttngBranch, toolsHeadCommits[lttngBranch], + modulesHeadCommits[lttngBranch], ustHeadCommits[lttngBranch]] + as RunConfiguration) + + newOldModulesHeadCommits.add(modulesHead.value) + } + } + } +} + +// For each top of branch commits that were not seen before, schedule one job +// for each lttng/linux tracked configurations +ustHeadCommits.each { ustHead -> + if (!oldUstHeadCommits.contains(ustHead.value)) { + linuxLastTagIds.each { linuxTag -> + def lttngBranch = ustHead.key + if (configurationOfInterest.contains([lttngBranch, linuxTag.key])) { + runConfigs.add([linuxTag.key, linuxTag.value, + lttngBranch, toolsHeadCommits[lttngBranch], + modulesHeadCommits[lttngBranch], ustHeadCommits[lttngBranch]] + as RunConfiguration) + + newOldUstHeadCommits.add(ustHead.value) + } + } + } +} + +// Save the tag and commit IDs scheduled in the past and during this run to the workspace +saveCurrentIdsToWorkspace(newOldLinuxTags, linuxOnDiskPath) +saveCurrentIdsToWorkspace(newOldToolsHeadCommits, toolsOnDiskPath) +saveCurrentIdsToWorkspace(newOldModulesHeadCommits, modulesOnDiskPath) +saveCurrentIdsToWorkspace(newOldUstHeadCommits, ustOnDiskPath) + +// Launch jobs +println("Schedule canary jobs once a day") +canaryRunConfigs.each { config -> + jobTypes.each { type -> + LaunchJob(type + '_canary', config) + } +} + +if (runConfigs.size() > 0) { + println("Schedule jobs because of code changes.") + runConfigs.each { config -> + jobTypes.each { type -> + LaunchJob(CraftJobName(type, config), config); + } + + // Jobs to run only on master branchs of both linux and lttng + if (config.linuxBranch.contains('master') && + config.lttngBranch.contains('master')) { + LaunchJob(CraftJobName('vm_tests_fuzzing', config), config) + } + } +} else { + println("No new commit or tags, nothing more to do.") +}