scripts/lttng-baremetal-tests/lava-submit.py

   1 #!/usr/bin/python
   2 # Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import argparse
  18 import base64
  19 import json
  20 import os
  21 import sys
  22 import time
  23 import xmlrpclib
  24 from collections import OrderedDict
  25 from enum import Enum
  26
  27 USERNAME = 'frdeso'
  28 HOSTNAME = 'lava-master.internal.efficios.com'
  29 SCP_PATH = 'scp://jenkins-lava@storage.internal.efficios.com'
  30
  31 class TestType(Enum):
  32     benchmarks=1
  33     tests=2
  34
  35 def get_job_bundle_content(server, job):
  36     bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
  37     bundle = server.dashboard.get(bundle_sha)
  38
  39     return json.loads(bundle['content'])
  40
  41 # Parse the results bundle to see the run-tests testcase
  42 # of the lttng-kernel-tests passed successfully
  43 def check_job_all_test_cases_state_count(server, job):
  44     content = get_job_bundle_content(server, job)
  45
  46     passed_tests=0
  47     failed_tests=0
  48     for run in content['test_runs']:
  49         for result in run['test_results']:
  50             if 'test_case_id' in result:
  51                 if result['result'] in 'pass':
  52                     passed_tests+=1
  53                 else:
  54                     failed_tests+=1
  55     return (passed_tests, failed_tests)
  56
  57 # Parse the attachment of the testcase to fetch the stdout of the test suite
  58 def print_test_output(server, job):
  59     content = get_job_bundle_content(server, job)
  60     found = False
  61
  62     for run in content['test_runs']:
  63         if run['test_id'] in 'lttng-kernel-test':
  64             for attachment in run['attachments']:
  65                 if attachment['pathname'] in 'stdout.log':
  66
  67                     # Decode the base64 file and split on newlines to iterate
  68                     # on list
  69                     testoutput = base64.b64decode(attachment['content']).split('\n')
  70
  71                     # Create a generator to iterate on the lines and keeping
  72                     # the state of the iterator across the two loops.
  73                     testoutput_iter = iter(testoutput)
  74                     for line in testoutput_iter:
  75
  76                         # Find the header of the test case and start printing
  77                         # from there
  78                         if 'LAVA_SIGNAL_STARTTC run-tests' in line:
  79                             found = True
  80                             print('---- TEST SUITE OUTPUT BEGIN ----')
  81                             for line in testoutput_iter:
  82                                 if 'LAVA_SIGNAL_ENDTC run-tests' not in line:
  83                                     print(line)
  84                                 else:
  85                                     # Print until we reach the end of the
  86                                     # section
  87                                     break
  88
  89                         if found is True:
  90                             print('----- TEST SUITE OUTPUT END -----')
  91                             break
  92
  93 def create_new_job(name, build_device):
  94     job = OrderedDict({
  95         'health_check': False,
  96         'job_name': name,
  97         'device_type':build_device,
  98         'tags': [ ],
  99         'timeout': 18000,
 100         'actions': []
 101     })
 102     if build_device in 'x86':
 103         job['tags'].append('dev-sda1')
 104
 105     return job
 106
 107 def get_boot_cmd():
 108     command = OrderedDict({
 109         'command': 'boot_image'
 110         })
 111     return command
 112
 113 def get_config_cmd(build_device):
 114     packages=['bsdtar', 'psmisc', 'wget', 'python3', 'python3-pip', \
 115             'libglib2.0-dev', 'libffi-dev', 'elfutils', 'libdw-dev', \
 116             'libelf-dev', 'libmount-dev', 'libxml2']
 117     command = OrderedDict({
 118         'command': 'lava_command_run',
 119         'parameters': {
 120             'commands': [
 121                 'ifup eth0',
 122                 'route -n',
 123                 'cat /etc/resolv.conf',
 124                 'echo nameserver 172.18.0.12 > /etc/resolv.conf',
 125                 'groupadd tracing'
 126                 ]
 127             }
 128         })
 129     if build_device in 'x86':
 130         command['parameters']['commands'].extend([
 131                     'mount /dev/sda1 /tmp',
 132                     'rm -rf /tmp/*'])
 133
 134     command['parameters']['commands'].extend([
 135                     'depmod -a',
 136                     'locale-gen en_US.UTF-8',
 137                     'apt-get update',
 138                     'apt-get install -y {}'.format(' '.join(packages))
 139                 ])
 140     return command
 141
 142 def get_benchmarks_cmd():
 143     command = OrderedDict({
 144         'command': 'lava_test_shell',
 145         'parameters': {
 146             'testdef_repos': [
 147                 {
 148                     'git-repo': 'https://github.com/lttng/lttng-ci.git',
 149                     'revision': 'master',
 150                     'testdef': 'lava/baremetal-tests/failing-close.yml'
 151                 },
 152                 {
 153                     'git-repo': 'https://github.com/lttng/lttng-ci.git',
 154                     'revision': 'master',
 155                     'testdef': 'lava/baremetal-tests/failing-open-efault.yml'
 156                 },
 157                 {
 158                     'git-repo': 'https://github.com/lttng/lttng-ci.git',
 159                     'revision': 'master',
 160                     'testdef': 'lava/baremetal-tests/failing-open-enoent.yml'
 161                 },
 162                 {
 163                     'git-repo': 'https://github.com/lttng/lttng-ci.git',
 164                     'revision': 'master',
 165                     'testdef': 'lava/baremetal-tests/perf-tests.yml'
 166                 }
 167                 ],
 168             'timeout': 18000
 169             }
 170         })
 171     return command
 172
 173 def get_tests_cmd():
 174     command = OrderedDict({
 175         'command': 'lava_test_shell',
 176         'parameters': {
 177             'testdef_repos': [
 178                 {
 179                     'git-repo': 'https://github.com/lttng/lttng-ci.git',
 180                     'revision': 'master',
 181                     'testdef': 'lava/baremetal-tests/kernel-tests.yml'
 182                 }
 183                 ],
 184             'timeout': 18000
 185             }
 186         })
 187     return command
 188
 189 def get_results_cmd(stream_name):
 190     command = OrderedDict({
 191             'command': 'submit_results',
 192             'parameters': {
 193                 'server': 'http://lava-master.internal.efficios.com/RPC2/'
 194             }
 195         })
 196     command['parameters']['stream']='/anonymous/'+stream_name+'/'
 197     return command
 198
 199 def get_deploy_cmd_kvm(jenkins_job, kernel_path, linux_modules_path, lttng_modules_path):
 200     command = OrderedDict({
 201             'command': 'deploy_kernel',
 202             'metadata': {},
 203             'parameters': {
 204                 'customize': {},
 205                 'kernel': None,
 206                 'rootfs': 'file:///var/lib/lava-server/default/media/images/trusty-grub.img.gz',
 207                 'target_type': 'ubuntu'
 208                 }
 209             })
 210
 211     command['parameters']['customize'][SCP_PATH+linux_modules_path]=['rootfs:/','archive']
 212     command['parameters']['customize'][SCP_PATH+lttng_modules_path]=['rootfs:/','archive']
 213     command['parameters']['kernel'] = str(SCP_PATH+kernel_path)
 214     command['metadata']['jenkins_jobname'] = jenkins_job
 215
 216     return command
 217
 218 def get_deploy_cmd_x86(jenkins_job, kernel_path, linux_modules_path, lttng_modules_path, nb_iter=None):
 219     command = OrderedDict({
 220             'command': 'deploy_kernel',
 221             'metadata': {},
 222             'parameters': {
 223                 'overlays': [],
 224                 'kernel': None,
 225                 'nfsrootfs': str(SCP_PATH+'/storage/jenkins-lava/rootfs/rootfs_amd64_trusty_2016-02-23-1134.tar.gz'),
 226                 'target_type': 'ubuntu'
 227                 }
 228             })
 229
 230     command['parameters']['overlays'].append( str(SCP_PATH+linux_modules_path))
 231     command['parameters']['overlays'].append( str(SCP_PATH+lttng_modules_path))
 232     command['parameters']['kernel'] = str(SCP_PATH+kernel_path)
 233     command['metadata']['jenkins_jobname'] = jenkins_job
 234     if nb_iter is not None:
 235         command['metadata']['nb_iterations'] = nb_iter
 236
 237     return command
 238
 239
 240 def get_env_setup_cmd(build_device, lttng_tools_commit, lttng_ust_commit=None):
 241     command = OrderedDict({
 242         'command': 'lava_command_run',
 243         'parameters': {
 244             'commands': [
 245                 'git clone https://github.com/frdeso/syscall-bench-it.git bm',
 246                 'pip3 install vlttng',
 247                         ],
 248             'timeout': 18000
 249             }
 250         })
 251
 252     vlttng_cmd = 'vlttng --jobs=16 --profile urcu-master' \
 253                     ' --profile babeltrace-stable-1.4 ' \
 254                     ' --profile lttng-tools-master' \
 255                     ' --override projects.lttng-tools.checkout='+lttng_tools_commit + \
 256                     ' --profile lttng-tools-no-man-pages'
 257
 258     if lttng_ust_commit is not None:
 259         vlttng_cmd += ' --profile lttng-ust-master ' \
 260                     ' --override projects.lttng-ust.checkout='+lttng_ust_commit+ \
 261                     ' --profile lttng-ust-no-man-pages'
 262
 263     virtenv_path = None
 264     if build_device in 'kvm':
 265         virtenv_path = '/root/virtenv'
 266     else:
 267         virtenv_path = '/tmp/virtenv'
 268
 269     vlttng_cmd += ' '+virtenv_path
 270
 271     command['parameters']['commands'].append(vlttng_cmd)
 272     command['parameters']['commands'].append('ln -s '+virtenv_path+' /root/lttngvenv')
 273     command['parameters']['commands'].append('sync')
 274
 275     return command
 276
 277 def main():
 278     test_type = None
 279     parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
 280     parser.add_argument('-t', '--type', required=True)
 281     parser.add_argument('-j', '--jobname', required=True)
 282     parser.add_argument('-k', '--kernel', required=True)
 283     parser.add_argument('-km', '--kmodule', required=True)
 284     parser.add_argument('-lm', '--lmodule', required=True)
 285     parser.add_argument('-l', '--lava-key', required=True)
 286     parser.add_argument('-tc', '--tools-commit', required=True)
 287     parser.add_argument('-uc', '--ust-commit', required=False)
 288     args = parser.parse_args()
 289
 290     if args.type in 'benchmarks':
 291         test_type = TestType.benchmarks
 292     elif args.type in 'tests':
 293         test_type = TestType.tests
 294     else:
 295         print('argument -t/--type {} unrecognized. Exiting...'.format(args.type))
 296         return -1
 297
 298     if test_type is TestType.benchmarks:
 299         j = create_new_job(args.jobname, build_device='x86')
 300         j['actions'].append(get_deploy_cmd_x86(args.jobname, args.kernel, args.kmodule, args.lmodule))
 301     elif test_type  is TestType.tests:
 302         j = create_new_job(args.jobname, build_device='kvm')
 303         j['actions'].append(get_deploy_cmd_kvm(args.jobname, args.kernel, args.kmodule, args.lmodule))
 304
 305     j['actions'].append(get_boot_cmd())
 306
 307     if test_type is TestType.benchmarks:
 308         j['actions'].append(get_config_cmd('x86'))
 309         j['actions'].append(get_env_setup_cmd('x86', args.tools_commit))
 310         j['actions'].append(get_benchmarks_cmd())
 311         j['actions'].append(get_results_cmd(stream_name='benchmark-kernel'))
 312     elif test_type  is TestType.tests:
 313         if args.ust_commit is None:
 314             print('Tests runs need -uc/--ust-commit options. Exiting...')
 315             return -1
 316         j['actions'].append(get_config_cmd('kvm'))
 317         j['actions'].append(get_env_setup_cmd('kvm', args.tools_commit, args.ust_commit))
 318         j['actions'].append(get_tests_cmd())
 319         j['actions'].append(get_results_cmd(stream_name='tests-kernel'))
 320     else:
 321         assert False, 'Unknown test type'
 322
 323     server = xmlrpclib.ServerProxy('http://%s:%s@%s/RPC2' % (USERNAME, args.lava_key, HOSTNAME))
 324
 325     jobid = server.scheduler.submit_job(json.dumps(j))
 326
 327     print('Lava jobid:{}'.format(jobid))
 328
 329     #Check the status of the job every 30 seconds
 330     jobstatus = server.scheduler.job_status(jobid)['job_status']
 331     while jobstatus in 'Submitted' or jobstatus in 'Running':
 332         time.sleep(30)
 333         jobstatus = server.scheduler.job_status(jobid)['job_status']
 334
 335     print('Job ended with {} status.'.format(jobstatus))
 336     if jobstatus not in 'Complete':
 337         return -1
 338
 339     passed, failed=check_job_all_test_cases_state_count(server, jobid)
 340
 341     print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
 342
 343     if test_type is TestType.tests:
 344         print_test_output(server, jobid)
 345
 346     if failed == 0:
 347         return 0
 348     else:
 349         return -1
 350
 351 if __name__ == "__main__":
 352     sys.exit(main())