Lava: Run kernel tests in KVM instance instead of baremetal
authorFrancis Deslauriers <francis.deslauriers@efficios.com>
Wed, 23 Nov 2016 16:11:50 +0000 (11:11 -0500)
committerFrancis Deslauriers <francis.deslauriers@efficios.com>
Wed, 23 Nov 2016 17:25:18 +0000 (12:25 -0500)
Signed-off-by: Francis Deslauriers <francis.deslauriers@efficios.com>
jobs/lttng-baremetal-tests.yaml
lava/baremetal-tests/failing-close.yml
lava/baremetal-tests/failing-open-efault.yml
lava/baremetal-tests/failing-open-enoent.yml
lava/baremetal-tests/kernel-tests.yml
scripts/lttng-baremetal-tests/check-build-needs.sh
scripts/lttng-baremetal-tests/generate-properties-master.sh
scripts/lttng-baremetal-tests/generate-properties-slave.sh
scripts/lttng-baremetal-tests/lava-submit.py

index 9f6a664d276d80deb435be74a3afc1b64144969e..894f9b991c03d564fe77ed77dc7630871e9fee47 100644 (file)
       - pollscm:
           cron: "@hourly"
 
-    properties:
-      - throttle:
-          max-total: 3
-          option: 'category'
-          categories:
-            - 'baremetal-tests'
     publishers:
       - email:
           recipients: 'francis.deslauriers@efficios.com'
     description: |
       Runs baremetal kernel tests over different combination of kernel and lttng configurations.
     defaults: lttng-baremetal-tests
+    properties:
+      - throttle:
+          max-total: 2
+          option: 'category'
+          categories:
+            - 'kvm-tests'
     wrappers:
       - workspace-cleanup
       - timestamps
@@ -64,6 +64,7 @@
           properties-content: |
             TOOLS_BRANCH={lttngversion}
             UST_BRANCH={lttngversion}
+            BUILD_DEVICE=kvm
 
     builders:
       - shell: !include-raw-escape: scripts/lttng-baremetal-tests/generate-properties-master.sh
     logrotate:
       numToKeep: 5
 
+    properties:
+      - throttle:
+          max-total: 2
+          option: 'category'
+          categories:
+            - 'baremetal-tests'
     scm:
       - git:
           url: git://git-mirror.internal.efficios.com/lttng/lttng-tools.git
           - text:
               credential-id: jenkins_lava_key
               variable: LAVA_FRDESO_TOKEN
+      - inject:
+          properties-content: |
+            BUILD_DEVICE=baremetal
     builders:
       - shell: !include-raw-escape: scripts/lttng-baremetal-tests/generate-properties-master.sh
       - trigger-builds:
       - string:
           name: 'STORAGE_LTTNG_MODULES'
           description: 'Path to store the LTTng Modules'
+      - string:
+          name: 'BUILD_DEVICE'
+          description: 'The target device. (kvm or baremetal)'
 
 ## Project
 - project:
index ff872ccb9e8771e324df77935f188abbc75e4638..99ebc3738b426bb0ba2991889be24c3f0e668357 100644 (file)
@@ -20,7 +20,7 @@ install:
                 - echo "$(pwd)/coredump/core.%e.%p.%h.%t" > /proc/sys/kernel/core_pattern
 run:
         steps:
-                - source /tmp/virtenv/activate
+                - source /root/lttngvenv/activate
                 - cd benchmarks
                 - lava-test-case build-benchmarks --shell "make"
                 - lava-test-case run-benchmarks --shell "./run.sh failing-close"
index a05b1fdcad42c62318d9dfa233ab852b7288dc40..ece0c8ffe9db4caf4d30bbb2e6f4225025fe6fca 100644 (file)
@@ -20,7 +20,7 @@ install:
                 - echo "$(pwd)/coredump/core.%e.%p.%h.%t" > /proc/sys/kernel/core_pattern
 run:
         steps:
-                - source /tmp/virtenv/activate
+                - source /root/lttngvenv/activate
                 - cd benchmarks
                 - lava-test-case build-benchmarks --shell "make"
                 - lava-test-case run-benchmarks --shell "./run.sh failing-open-efault"
index 980c72e0d3c9fd944b6a106627f850e986c3deee..31c4a29c54315b8ff0db051de025f3a1cea99265 100644 (file)
@@ -20,7 +20,7 @@ install:
                 - echo "$(pwd)/coredump/core.%e.%p.%h.%t" > /proc/sys/kernel/core_pattern
 run:
         steps:
-                - source /tmp/virtenv/activate
+                - source /root/lttngvenv/activate
                 - cd benchmarks
                 - lava-test-case build-benchmarks --shell "make"
                 - lava-test-case run-benchmarks --shell "./run.sh failing-open-enoent"
index a079a67522a0fc4ab35453f19f3f5bd5ec45b3f0..35ee979103dadac432e3eb3582e3a32ed66e25b3 100644 (file)
@@ -11,11 +11,11 @@ install:
                 - echo "$(pwd)/coredump/core.%e.%p.%h.%t" > /proc/sys/kernel/core_pattern
 run:
         steps:
-                - source /tmp/virtenv/activate
-                - cd /tmp/virtenv/src/lttng-tools
+                - source /root/lttngvenv/activate
+                - cd /root/lttngvenv/src/lttng-tools
                 - lava-test-case build-test-suite --shell "make"
                 - cd tests
-                - lava-test-case run-tests --shell "./run.sh root_regression"
+                - lava-test-case run-tests --shell "prove --verbose --merge --exec '' - < root_regression"
                 - cd
                 - tar czf coredump.tar.gz coredump
                 - lava-test-case-attach run-benchmarks coredump.tar.gz
index 52aea76e2e82b5175bea68235625de7abf1ef7ac..625047e7943b74a4e3b791e6d3852a2b8c3ecba9 100644 (file)
@@ -47,9 +47,13 @@ if [ $NEED_MODULES_BUILD -eq 1 ] || [ $NEED_KERNEL_BUILD -eq 1 ] ; then
   popd
 
   git clone https://github.com/lttng/lttng-ci "$LTTNG_CI_PATH"
-  cp "$LTTNG_CI_PATH"/lava/kernel/vanilla/x86_64_server.config "$LINUX_PATH"/.config
-
+  cp "$LTTNG_CI_PATH"/lava/kernel/vanilla/x86_64_server.config "$LINUX_PATH/.config"
   make --directory="$LINUX_PATH" olddefconfig
+
+  if [ $BUILD_DEVICE = 'kvm' ] ; then
+    make --directory="$LINUX_PATH" kvmconfig
+  fi
+
   make --directory="$LINUX_PATH" modules_prepare
 fi
 
index 74ce81865e8ce0f5de458a8e64a2f55959c66d12..22111a8b03563dcb90036b19ba28be5af3b2fe5b 100644 (file)
@@ -48,8 +48,9 @@ echo "LTTNG_TOOLS_COMMIT_ID=$LTTNG_TOOLS_COMMIT_ID" >> properties.txt
 
 BASE_STORAGE_FOLDER="/storage/jenkins-lava/baremetal-tests"
 
+echo "BUILD_DEVICE=$BUILD_DEVICE" >> properties.txt
 echo "KGITREPO=git://git-mirror.internal.efficios.com/git/linux-stable.git" >> properties.txt
 echo "STORAGE_KERNEL_FOLDER=$BASE_STORAGE_FOLDER/kernel" >> properties.txt
-echo "STORAGE_KERNEL_IMAGE=$BASE_STORAGE_FOLDER/kernel/$KERNEL_COMMIT_ID.bzImage" >> properties.txt
-echo "STORAGE_LINUX_MODULES=$BASE_STORAGE_FOLDER/modules/linux/$KERNEL_COMMIT_ID.linux.modules.tar.gz" >> properties.txt
-echo "STORAGE_LTTNG_MODULES=$BASE_STORAGE_FOLDER/modules/lttng/$KERNEL_COMMIT_ID-$LTTNG_MODULES_COMMIT_ID.lttng.modules.tar.gz" >> properties.txt
+echo "STORAGE_KERNEL_IMAGE=$BASE_STORAGE_FOLDER/kernel/$KERNEL_COMMIT_ID.$BUILD_DEVICE.bzImage" >> properties.txt
+echo "STORAGE_LINUX_MODULES=$BASE_STORAGE_FOLDER/modules/linux/$KERNEL_COMMIT_ID.$BUILD_DEVICE.linux.modules.tar.gz" >> properties.txt
+echo "STORAGE_LTTNG_MODULES=$BASE_STORAGE_FOLDER/modules/lttng/$KERNEL_COMMIT_ID-$LTTNG_MODULES_COMMIT_ID.$BUILD_DEVICE.lttng.modules.tar.gz" >> properties.txt
index 449a9daed173351374ca3576e3516f60c72cbd83..017f1f653ebb026f694734f04014580bda639db5 100644 (file)
@@ -40,9 +40,10 @@ BUILD_NAME="$KERNEL_COMMIT_ID-$LTTNG_MODULES_COMMIT_ID"
 echo "KERNEL_COMMIT_ID=$KERNEL_COMMIT_ID" >> properties.txt
 echo "LTTNG_MODULES_COMMIT_ID=$LTTNG_MODULES_COMMIT_ID" >> properties.txt
 echo "BUILD_NAME=$BUILD_NAME" >> properties.txt
+echo "BUILD_DEVICE=$BUILD_DEVICE" >> properties.txt
 
-echo "STORAGE_KERNEL_MODULE_SYMVERS=$STORAGE_KERNEL_FOLDER/$KERNEL_COMMIT_ID.symvers" >>properties.txt
-echo "STORAGE_KERNEL_CONFIG=$STORAGE_KERNEL_FOLDER/$KERNEL_COMMIT_ID.config" >> properties.txt
+echo "STORAGE_KERNEL_MODULE_SYMVERS=$STORAGE_KERNEL_FOLDER/$KERNEL_COMMIT_ID.$BUILD_DEVICE.symvers" >>properties.txt
+echo "STORAGE_KERNEL_CONFIG=$STORAGE_KERNEL_FOLDER/$KERNEL_COMMIT_ID.$BUILD_DEVICE.config" >> properties.txt
 
 echo "STORAGE_HOST=storage01.internal.efficios.com" >> properties.txt
 echo "STORAGE_USER=jenkins-lava" >> properties.txt
index a0e340748d50fb2b0a850ad08a00d0cd825da18b..52bc4d7be5b3f4de7a1b2bdb3b7b348ec6e03b68 100644 (file)
@@ -90,15 +90,18 @@ def print_test_output(server, job):
                             print('----- TEST SUITE OUTPUT END -----')
                             break
 
-def create_new_job(name):
+def create_new_job(name, build_device):
     job = OrderedDict({
         'health_check': False,
         'job_name': name,
-        'device_type': 'x86',
-        'tags': [ 'dev-sda1' ],
+        'device_type':build_device,
+        'tags': [ ],
         'timeout': 18000,
         'actions': []
     })
+    if build_device in 'x86':
+        job['tags'].append('dev-sda1')
+
     return job
 
 def get_boot_cmd():
@@ -107,7 +110,7 @@ def get_boot_cmd():
         })
     return command
 
-def get_config_cmd():
+def get_config_cmd(build_device):
     packages=['bsdtar', 'psmisc', 'wget', 'python3', 'python3-pip', \
             'libglib2.0-dev', 'libffi-dev', 'elfutils', 'libdw-dev', \
             'libelf-dev', 'libmount-dev', 'libxml2', 'python3-pandas', \
@@ -120,15 +123,21 @@ def get_config_cmd():
                 'route -n',
                 'cat /etc/resolv.conf',
                 'echo nameserver 172.18.0.12 > /etc/resolv.conf',
-                'mount /dev/sda1 /tmp',
-                'rm -rf /tmp/*',
-                'depmod -a',
-                'locale-gen en_US.UTF-8',
-                'apt-get update',
-                'apt-get install -y {}'.format(' '.join(packages)),
+                'groupadd tracing'
                 ]
             }
         })
+    if build_device in 'x86':
+        command['parameters']['commands'].extend([
+                    'mount /dev/sda1 /tmp',
+                    'rm -rf /tmp/*'])
+
+    command['parameters']['commands'].extend([
+                    'depmod -a',
+                    'locale-gen en_US.UTF-8',
+                    'apt-get update',
+                    'apt-get install -y {}'.format(' '.join(packages))
+                ])
     return command
 
 def get_benchmarks_cmd():
@@ -183,7 +192,26 @@ def get_results_cmd(stream_name):
     command['parameters']['stream']='/anonymous/'+stream_name+'/'
     return command
 
-def get_deploy_cmd(jenkins_job, kernel_path, linux_modules_path, lttng_modules_path, nb_iter=None):
+def get_deploy_cmd_kvm(jenkins_job, kernel_path, linux_modules_path, lttng_modules_path):
+    command = OrderedDict({
+            'command': 'deploy_kernel',
+            'metadata': {},
+            'parameters': {
+                'customize': {},
+                'kernel': None,
+                'rootfs': 'file:///var/lib/lava-server/default/media/images/trusty-grub.img.gz',
+                'target_type': 'ubuntu'
+                }
+            })
+
+    command['parameters']['customize'][SCP_PATH+linux_modules_path]=['rootfs:/','archive']
+    command['parameters']['customize'][SCP_PATH+lttng_modules_path]=['rootfs:/','archive']
+    command['parameters']['kernel'] = str(SCP_PATH+kernel_path)
+    command['metadata']['jenkins_jobname'] = jenkins_job
+
+    return command
+
+def get_deploy_cmd_x86(jenkins_job, kernel_path, linux_modules_path, lttng_modules_path, nb_iter=None):
     command = OrderedDict({
             'command': 'deploy_kernel',
             'metadata': {},
@@ -198,14 +226,14 @@ def get_deploy_cmd(jenkins_job, kernel_path, linux_modules_path, lttng_modules_p
     command['parameters']['overlays'].append( str(SCP_PATH+linux_modules_path))
     command['parameters']['overlays'].append( str(SCP_PATH+lttng_modules_path))
     command['parameters']['kernel'] = str(SCP_PATH+kernel_path)
-    command['metadata']['jenkins_jobname'] = jenkins_job    
+    command['metadata']['jenkins_jobname'] = jenkins_job
     if nb_iter is not None:
         command['metadata']['nb_iterations'] = nb_iter
 
     return command
 
 
-def get_env_setup_cmd(lttng_tools_commit, lttng_ust_commit=None):
+def get_env_setup_cmd(build_device, lttng_tools_commit, lttng_ust_commit=None):
     command = OrderedDict({
         'command': 'lava_command_run',
         'parameters': {
@@ -228,9 +256,18 @@ def get_env_setup_cmd(lttng_tools_commit, lttng_ust_commit=None):
                     ' --override projects.lttng-ust.checkout='+lttng_ust_commit+ \
                     ' --profile lttng-ust-no-man-pages'
 
-    vlttng_cmd += " /tmp/virtenv"
+    virtenv_path = None
+    if build_device in 'kvm':
+        virtenv_path = '/root/virtenv'
+    else:
+        virtenv_path = '/tmp/virtenv'
+
+    vlttng_cmd += ' '+virtenv_path
 
     command['parameters']['commands'].append(vlttng_cmd)
+    command['parameters']['commands'].append('ln -s '+virtenv_path+' /root/lttngvenv')
+    command['parameters']['commands'].append('sync')
+
     return command
 
 def main():
@@ -246,12 +283,6 @@ def main():
     parser.add_argument('-uc', '--ust-commit', required=False)
     args = parser.parse_args()
 
-
-    j = create_new_job(args.jobname)
-    j['actions'].append(get_deploy_cmd(args.jobname, args.kernel, args.kmodule, args.lmodule))
-    j['actions'].append(get_boot_cmd())
-    j['actions'].append(get_config_cmd())
-
     if args.type in 'benchmarks':
         test_type = TestType.benchmarks
     elif args.type in 'tests':
@@ -261,14 +292,25 @@ def main():
         return -1
 
     if test_type is TestType.benchmarks:
-        j['actions'].append(get_env_setup_cmd(args.tools_commit))
+        j = create_new_job(args.jobname, build_device='x86')
+        j['actions'].append(get_deploy_cmd_x86(args.jobname, args.kernel, args.kmodule, args.lmodule))
+    elif test_type  is TestType.tests:
+        j = create_new_job(args.jobname, build_device='kvm')
+        j['actions'].append(get_deploy_cmd_kvm(args.jobname, args.kernel, args.kmodule, args.lmodule))
+
+    j['actions'].append(get_boot_cmd())
+
+    if test_type is TestType.benchmarks:
+        j['actions'].append(get_config_cmd('x86'))
+        j['actions'].append(get_env_setup_cmd('x86', args.tools_commit))
         j['actions'].append(get_benchmarks_cmd())
         j['actions'].append(get_results_cmd(stream_name='benchmark-kernel'))
     elif test_type  is TestType.tests:
         if args.ust_commit is None:
             print('Tests runs need -uc/--ust-commit options. Exiting...')
             return -1
-        j['actions'].append(get_env_setup_cmd(args.tools_commit, args.ust_commit))
+        j['actions'].append(get_config_cmd('kvm'))
+        j['actions'].append(get_env_setup_cmd('kvm', args.tools_commit, args.ust_commit))
         j['actions'].append(get_tests_cmd())
         j['actions'].append(get_results_cmd(stream_name='tests-kernel'))
     else:
This page took 0.030702 seconds and 4 git commands to generate.