scripts/lttng-baremetal-tests/lava-submit.py

   1 #!/usr/bin/python
   2 # Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import argparse
  18 import base64
  19 import json
  20 import os
  21 import sys
  22 import time
  23 import xmlrpclib
  24 from collections import OrderedDict
  25 from enum import Enum
  26
  27 USERNAME = 'frdeso'
  28 HOSTNAME = 'lava-master.internal.efficios.com'
  29 SCP_PATH = 'scp://jenkins-lava@storage.internal.efficios.com'
  30
  31 class TestType(Enum):
  32     benchmarks=1
  33     tests=2
  34
  35 def get_job_bundle_content(server, job):
  36     bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
  37     bundle = server.dashboard.get(bundle_sha)
  38
  39     return json.loads(bundle['content'])
  40
  41 # Parse the results bundle to see the run-tests testcase
  42 # of the lttng-kernel-tests passed successfully
  43 def check_job_all_test_cases_state_count(server, job):
  44     content = get_job_bundle_content(server, job)
  45
  46     passed_tests=0
  47     failed_tests=0
  48     for run in content['test_runs']:
  49         for result in run['test_results']:
  50             if 'test_case_id' in result:
  51                 if result['result'] in 'pass':
  52                     passed_tests+=1
  53                 else:
  54                     failed_tests+=1
  55     return (passed_tests, failed_tests)
  56
  57 # Parse the attachment of the testcase to fetch the stdout of the test suite
  58 def print_test_output(server, job):
  59     content = get_job_bundle_content(server, job)
  60     found = False
  61
  62     for run in content['test_runs']:
  63         if run['test_id'] in 'lttng-kernel-test':
  64             for attachment in run['attachments']:
  65                 if attachment['pathname'] in 'stdout.log':
  66
  67                     # Decode the base64 file and split on newlines to iterate
  68                     # on list
  69                     testoutput = base64.b64decode(attachment['content']).split('\n')
  70
  71                     # Create a generator to iterate on the lines and keeping
  72                     # the state of the iterator across the two loops.
  73                     testoutput_iter = iter(testoutput)
  74                     for line in testoutput_iter:
  75
  76                         # Find the header of the test case and start printing
  77                         # from there
  78                         if 'LAVA_SIGNAL_STARTTC run-tests' in line:
  79                             found = True
  80                             print('---- TEST SUITE OUTPUT BEGIN ----')
  81                             for line in testoutput_iter:
  82                                 if 'LAVA_SIGNAL_ENDTC run-tests' not in line:
  83                                     print(line)
  84                                 else:
  85                                     # Print until we reach the end of the
  86                                     # section
  87                                     break
  88
  89                         if found is True:
  90                             print('----- TEST SUITE OUTPUT END -----')
  91                             break
  92
  93 def create_new_job(name, build_device):
  94     job = OrderedDict({
  95         'health_check': False,
  96         'job_name': name,
  97         'device_type':build_device,
  98         'tags': [ ],
  99         'timeout': 18000,
 100         'actions': []
 101     })
 102     if build_device in 'x86':
 103         job['tags'].append('dev-sda1')
 104
 105     return job
 106
 107 def get_boot_cmd():
 108     command = OrderedDict({
 109         'command': 'boot_image'
 110         })
 111     return command
 112
 113 def get_config_cmd(build_device):
 114     packages=['bsdtar', 'psmisc', 'wget', 'python3', 'python3-pip', \
 115             'libglib2.0-dev', 'libffi-dev', 'elfutils', 'libdw-dev', \
 116             'libelf-dev', 'libmount-dev', 'libxml2', 'python3-pandas', \
 117             'python3-numpy']
 118     command = OrderedDict({
 119         'command': 'lava_command_run',
 120         'parameters': {
 121             'commands': [
 122                 'ifup eth0',
 123                 'route -n',
 124                 'cat /etc/resolv.conf',
 125                 'echo nameserver 172.18.0.12 > /etc/resolv.conf',
 126                 'groupadd tracing'
 127                 ]
 128             }
 129         })
 130     if build_device in 'x86':
 131         command['parameters']['commands'].extend([
 132                     'mount /dev/sda1 /tmp',
 133                     'rm -rf /tmp/*'])
 134
 135     command['parameters']['commands'].extend([
 136                     'depmod -a',
 137                     'locale-gen en_US.UTF-8',
 138                     'apt-get update',
 139                     'apt-get install -y {}'.format(' '.join(packages))
 140                 ])
 141     return command
 142
 143 def get_benchmarks_cmd():
 144     command = OrderedDict({
 145         'command': 'lava_test_shell',
 146         'parameters': {
 147             'testdef_repos': [
 148                 {
 149                     'git-repo': 'https://github.com/lttng/lttng-ci.git',
 150                     'revision': 'master',
 151                     'testdef': 'lava/baremetal-tests/failing-close.yml'
 152                 },
 153                 {
 154                     'git-repo': 'https://github.com/lttng/lttng-ci.git',
 155                     'revision': 'master',
 156                     'testdef': 'lava/baremetal-tests/failing-open-efault.yml'
 157                 },
 158                 {
 159                     'git-repo': 'https://github.com/lttng/lttng-ci.git',
 160                     'revision': 'master',
 161                     'testdef': 'lava/baremetal-tests/failing-open-enoent.yml'
 162                 }
 163                 ],
 164             'timeout': 18000
 165             }
 166         })
 167     return command
 168
 169 def get_tests_cmd():
 170     command = OrderedDict({
 171         'command': 'lava_test_shell',
 172         'parameters': {
 173             'testdef_repos': [
 174                 {
 175                     'git-repo': 'https://github.com/lttng/lttng-ci.git',
 176                     'revision': 'master',
 177                     'testdef': 'lava/baremetal-tests/kernel-tests.yml'
 178                 }
 179                 ],
 180             'timeout': 18000
 181             }
 182         })
 183     return command
 184
 185 def get_results_cmd(stream_name):
 186     command = OrderedDict({
 187             'command': 'submit_results',
 188             'parameters': {
 189                 'server': 'http://lava-master.internal.efficios.com/RPC2/'
 190             }
 191         })
 192     command['parameters']['stream']='/anonymous/'+stream_name+'/'
 193     return command
 194
 195 def get_deploy_cmd_kvm(jenkins_job, kernel_path, linux_modules_path, lttng_modules_path):
 196     command = OrderedDict({
 197             'command': 'deploy_kernel',
 198             'metadata': {},
 199             'parameters': {
 200                 'customize': {},
 201                 'kernel': None,
 202                 'rootfs': 'file:///var/lib/lava-server/default/media/images/trusty-grub.img.gz',
 203                 'target_type': 'ubuntu'
 204                 }
 205             })
 206
 207     command['parameters']['customize'][SCP_PATH+linux_modules_path]=['rootfs:/','archive']
 208     command['parameters']['customize'][SCP_PATH+lttng_modules_path]=['rootfs:/','archive']
 209     command['parameters']['kernel'] = str(SCP_PATH+kernel_path)
 210     command['metadata']['jenkins_jobname'] = jenkins_job
 211
 212     return command
 213
 214 def get_deploy_cmd_x86(jenkins_job, kernel_path, linux_modules_path, lttng_modules_path, nb_iter=None):
 215     command = OrderedDict({
 216             'command': 'deploy_kernel',
 217             'metadata': {},
 218             'parameters': {
 219                 'overlays': [],
 220                 'kernel': None,
 221                 'nfsrootfs': str(SCP_PATH+'/storage/jenkins-lava/rootfs/rootfs_amd64_trusty_2016-02-23-1134.tar.gz'),
 222                 'target_type': 'ubuntu'
 223                 }
 224             })
 225
 226     command['parameters']['overlays'].append( str(SCP_PATH+linux_modules_path))
 227     command['parameters']['overlays'].append( str(SCP_PATH+lttng_modules_path))
 228     command['parameters']['kernel'] = str(SCP_PATH+kernel_path)
 229     command['metadata']['jenkins_jobname'] = jenkins_job
 230     if nb_iter is not None:
 231         command['metadata']['nb_iterations'] = nb_iter
 232
 233     return command
 234
 235
 236 def get_env_setup_cmd(build_device, lttng_tools_commit, lttng_ust_commit=None):
 237     command = OrderedDict({
 238         'command': 'lava_command_run',
 239         'parameters': {
 240             'commands': [
 241                 'git clone https://github.com/frdeso/syscall-bench-it.git bm',
 242                 'pip3 install vlttng',
 243                         ],
 244             'timeout': 18000
 245             }
 246         })
 247
 248     vlttng_cmd = 'vlttng --jobs=16 --profile urcu-master' \
 249                     ' --profile babeltrace-stable-1.4 ' \
 250                     ' --profile lttng-tools-master' \
 251                     ' --override projects.lttng-tools.checkout='+lttng_tools_commit + \
 252                     ' --profile lttng-tools-no-man-pages'
 253
 254     if lttng_ust_commit is not None:
 255         vlttng_cmd += ' --profile lttng-ust-master ' \
 256                     ' --override projects.lttng-ust.checkout='+lttng_ust_commit+ \
 257                     ' --profile lttng-ust-no-man-pages'
 258
 259     virtenv_path = None
 260     if build_device in 'kvm':
 261         virtenv_path = '/root/virtenv'
 262     else:
 263         virtenv_path = '/tmp/virtenv'
 264
 265     vlttng_cmd += ' '+virtenv_path
 266
 267     command['parameters']['commands'].append(vlttng_cmd)
 268     command['parameters']['commands'].append('ln -s '+virtenv_path+' /root/lttngvenv')
 269     command['parameters']['commands'].append('sync')
 270
 271     return command
 272
 273 def main():
 274     test_type = None
 275     parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
 276     parser.add_argument('-t', '--type', required=True)
 277     parser.add_argument('-j', '--jobname', required=True)
 278     parser.add_argument('-k', '--kernel', required=True)
 279     parser.add_argument('-km', '--kmodule', required=True)
 280     parser.add_argument('-lm', '--lmodule', required=True)
 281     parser.add_argument('-l', '--lava-key', required=True)
 282     parser.add_argument('-tc', '--tools-commit', required=True)
 283     parser.add_argument('-uc', '--ust-commit', required=False)
 284     args = parser.parse_args()
 285
 286     if args.type in 'benchmarks':
 287         test_type = TestType.benchmarks
 288     elif args.type in 'tests':
 289         test_type = TestType.tests
 290     else:
 291         print('argument -t/--type {} unrecognized. Exiting...'.format(args.type))
 292         return -1
 293
 294     if test_type is TestType.benchmarks:
 295         j = create_new_job(args.jobname, build_device='x86')
 296         j['actions'].append(get_deploy_cmd_x86(args.jobname, args.kernel, args.kmodule, args.lmodule))
 297     elif test_type  is TestType.tests:
 298         j = create_new_job(args.jobname, build_device='kvm')
 299         j['actions'].append(get_deploy_cmd_kvm(args.jobname, args.kernel, args.kmodule, args.lmodule))
 300
 301     j['actions'].append(get_boot_cmd())
 302
 303     if test_type is TestType.benchmarks:
 304         j['actions'].append(get_config_cmd('x86'))
 305         j['actions'].append(get_env_setup_cmd('x86', args.tools_commit))
 306         j['actions'].append(get_benchmarks_cmd())
 307         j['actions'].append(get_results_cmd(stream_name='benchmark-kernel'))
 308     elif test_type  is TestType.tests:
 309         if args.ust_commit is None:
 310             print('Tests runs need -uc/--ust-commit options. Exiting...')
 311             return -1
 312         j['actions'].append(get_config_cmd('kvm'))
 313         j['actions'].append(get_env_setup_cmd('kvm', args.tools_commit, args.ust_commit))
 314         j['actions'].append(get_tests_cmd())
 315         j['actions'].append(get_results_cmd(stream_name='tests-kernel'))
 316     else:
 317         assert False, 'Unknown test type'
 318
 319     server = xmlrpclib.ServerProxy('http://%s:%s@%s/RPC2' % (USERNAME, args.lava_key, HOSTNAME))
 320
 321     jobid = server.scheduler.submit_job(json.dumps(j))
 322
 323     print('Lava jobid:{}'.format(jobid))
 324
 325     #Check the status of the job every 30 seconds
 326     jobstatus = server.scheduler.job_status(jobid)['job_status']
 327     while jobstatus in 'Submitted' or jobstatus in 'Running':
 328         time.sleep(30)
 329         jobstatus = server.scheduler.job_status(jobid)['job_status']
 330
 331     print('Job ended with {} status.'.format(jobstatus))
 332     if jobstatus not in 'Complete':
 333         return -1
 334
 335     passed, failed=check_job_all_test_cases_state_count(server, jobid)
 336
 337     print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
 338
 339     if test_type is TestType.tests:
 340         print_test_output(server, jobid)
 341
 342     if failed == 0:
 343         return 0
 344     else:
 345         return -1
 346
 347 if __name__ == "__main__":
 348     sys.exit(main())