scripts/system-tests/lava2-submit.py

   1 #!/usr/bin/python
   2 # Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import argparse
  18 import base64
  19 import json
  20 import os
  21 import random
  22 import sys
  23 import time
  24 import yaml
  25 import xmlrpc.client
  26 import pprint
  27
  28 from jinja2 import Environment, FileSystemLoader, meta
  29
  30 USERNAME = 'lava-jenkins'
  31 HOSTNAME = 'lava-master-02.internal.efficios.com'
  32
  33 class TestType():
  34     baremetal_benchmarks=1
  35     baremetal_tests=2
  36     kvm_tests=3
  37     kvm_fuzzing_tests=4
  38     values = {
  39         'baremetal-benchmarks' : baremetal_benchmarks,
  40         'baremetal-tests' : baremetal_tests,
  41         'kvm-tests' : kvm_tests,
  42         'kvm-fuzzin-tests' : kvm_fuzzing_tests,
  43     }
  44
  45 class DeviceType():
  46     x86 = 'x86'
  47     kvm = 'qemu'
  48     values = {
  49         'kvm' : kvm,
  50         'x86' : x86,
  51     }
  52
  53 def get_job_bundle_content(server, job):
  54     try:
  55         bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
  56         bundle = server.dashboard.get(bundle_sha)
  57     except xmlrpc.client.Fault as f:
  58         print('Error while fetching results bundle', f.faultString)
  59         raise f
  60
  61     return json.loads(bundle['content'])
  62
  63 # Parse the results bundle to see the run-tests testcase
  64 # of the lttng-kernel-tests passed successfully
  65 def check_job_all_test_cases_state_count(server, job):
  66     content = get_job_bundle_content(server, job)
  67
  68     # FIXME:Those tests are part of the boot actions and fail randomly but
  69     # doesn't affect the behaviour of the tests. We should update our Lava
  70     # installation and try to reproduce it. This error was encountered on
  71     # Ubuntu 16.04.
  72     tests_known_to_fail=['mount', 'df', 'ls', 'ip', 'wait_for_test_image_prompt']
  73
  74     passed_tests=0
  75     failed_tests=0
  76     for run in content['test_runs']:
  77         for result in run['test_results']:
  78             if 'test_case_id' in result :
  79                 if result['result'] in 'pass':
  80                     passed_tests+=1
  81                 elif result['test_case_id'] in tests_known_to_fail:
  82                     pass
  83                 else:
  84                     failed_tests+=1
  85     return (passed_tests, failed_tests)
  86
  87 # Get the benchmark results from the lava bundle
  88 # save them as CSV files localy
  89 def fetch_benchmark_results(server, job):
  90     content = get_job_bundle_content(server, job)
  91     testcases = ['processed_results_close.csv',
  92             'processed_results_ioctl.csv',
  93             'processed_results_open_efault.csv',
  94             'processed_results_open_enoent.csv',
  95             'processed_results_dup_close.csv',
  96             'processed_results_raw_syscall_getpid.csv',
  97             'processed_results_lttng_test_filter.csv']
  98
  99     # The result bundle is a large JSON containing the results of every testcase
 100     # of the LAVA job as well as the files that were attached during the run.
 101     # We need to iterate over this JSON to get the base64 representation of the
 102     # benchmark results produced during the run.
 103     for run in content['test_runs']:
 104         # We only care of the benchmark testcases
 105         if 'benchmark-' in run['test_id']:
 106             if 'test_results' in run:
 107                 for res in run['test_results']:
 108                     if 'attachments' in res:
 109                         for a in res['attachments']:
 110                             # We only save the results file
 111                             if a['pathname'] in testcases:
 112                                 with open(a['pathname'],'wb') as f:
 113                                     # Convert the b64 representation of the
 114                                     # result file and write it to a file
 115                                     # in the current working directory
 116                                     f.write(base64.b64decode(a['content']))
 117
 118 # Parse the attachment of the testcase to fetch the stdout of the test suite
 119 def print_test_output(server, job):
 120     content = get_job_bundle_content(server, job)
 121     found = False
 122
 123     for run in content['test_runs']:
 124         if run['test_id'] in 'lttng-kernel-test':
 125             for attachment in run['attachments']:
 126                 if attachment['pathname'] in 'stdout.log':
 127
 128                     # Decode the base64 file and split on newlines to iterate
 129                     # on list
 130                     testoutput = str(base64.b64decode(bytes(attachment['content'], encoding='UTF-8')))
 131
 132                     testoutput = testoutput.replace('\\n', '\n')
 133
 134                     # Create a generator to iterate on the lines and keeping
 135                     # the state of the iterator across the two loops.
 136                     testoutput_iter = iter(testoutput.split('\n'))
 137                     for line in testoutput_iter:
 138
 139                         # Find the header of the test case and start printing
 140                         # from there
 141                         if 'LAVA_SIGNAL_STARTTC run-tests' in line:
 142                             print('---- TEST SUITE OUTPUT BEGIN ----')
 143                             for line in testoutput_iter:
 144                                 if 'LAVA_SIGNAL_ENDTC run-tests' not in line:
 145                                     print(line)
 146                                 else:
 147                                     # Print until we reach the end of the
 148                                     # section
 149                                     break
 150
 151                             print('----- TEST SUITE OUTPUT END -----')
 152                             break
 153
 154 def get_vlttng_cmd(device, lttng_tools_commit, lttng_ust_commit=None):
 155
 156     vlttng_cmd = 'vlttng --jobs=$(nproc) --profile urcu-master' \
 157                     ' --override projects.babeltrace.build-env.PYTHON=python3' \
 158                     ' --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config' \
 159                     ' --profile babeltrace-stable-1.4' \
 160                     ' --profile babeltrace-python' \
 161                     ' --profile lttng-tools-master' \
 162                     ' --override projects.lttng-tools.checkout='+lttng_tools_commit + \
 163                     ' --profile lttng-tools-no-man-pages'
 164
 165     if lttng_ust_commit is not None:
 166         vlttng_cmd += ' --profile lttng-ust-master ' \
 167                     ' --override projects.lttng-ust.checkout='+lttng_ust_commit+ \
 168                     ' --profile lttng-ust-no-man-pages'
 169
 170     if device is DeviceType.kvm:
 171         vlttng_path = '/root/virtenv'
 172     else:
 173         vlttng_path = '/tmp/virtenv'
 174
 175     vlttng_cmd += ' ' + vlttng_path
 176
 177     return vlttng_cmd
 178
 179 def main():
 180     nfsrootfs = "https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_bionic_2018-11-29.tar.gz"
 181     test_type = None
 182     parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
 183     parser.add_argument('-t', '--type', required=True)
 184     parser.add_argument('-j', '--jobname', required=True)
 185     parser.add_argument('-k', '--kernel', required=True)
 186     parser.add_argument('-lm', '--lmodule', required=True)
 187     parser.add_argument('-tc', '--tools-commit', required=True)
 188     parser.add_argument('-id', '--build-id', required=True)
 189     parser.add_argument('-uc', '--ust-commit', required=False)
 190     parser.add_argument('-d', '--debug', required=False, action='store_true')
 191     args = parser.parse_args()
 192
 193     if args.type not in TestType.values:
 194         print('argument -t/--type {} unrecognized.'.format(args.type))
 195         print('Possible values are:')
 196         for k in TestType.values:
 197             print('\t {}'.format(k))
 198         return -1
 199
 200     lava_api_key = None
 201     if not args.debug:
 202         try:
 203             lava_api_key = os.environ['LAVA2_JENKINS_TOKEN']
 204         except Exception as e:
 205             print('LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...', e )
 206             return -1
 207
 208     jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__)))
 209     jinja_env = Environment(loader=jinja_loader, trim_blocks=True,
 210             lstrip_blocks= True)
 211     jinja_template = jinja_env.get_template('template_lava_job.jinja2')
 212     template_source = jinja_env.loader.get_source(jinja_env, 'template_lava_job.jinja2')
 213     parsed_content = jinja_env.parse(template_source)
 214     undef = meta.find_undeclared_variables(parsed_content)
 215
 216     test_type = TestType.values[args.type]
 217
 218     if test_type in [TestType.baremetal_benchmarks, TestType.baremetal_tests]:
 219         device_type = DeviceType.x86
 220         vlttng_path = '/tmp/virtenv'
 221
 222     else:
 223         device_type = DeviceType.kvm
 224         vlttng_path = '/root/virtenv'
 225
 226     vlttng_cmd = get_vlttng_cmd(device_type, args.tools_commit, args.ust_commit)
 227
 228     context = dict()
 229     context['DeviceType'] = DeviceType
 230     context['TestType'] = TestType
 231
 232     context['job_name'] = args.jobname
 233     context['test_type'] = test_type
 234     context['random_seed'] = random.randint(0, 1000000)
 235     context['device_type'] = device_type
 236
 237     context['vlttng_cmd'] = vlttng_cmd
 238     context['vlttng_path'] = vlttng_path
 239
 240     context['kernel_url'] = args.kernel
 241     context['nfsrootfs_url'] = nfsrootfs
 242     context['lttng_modules_url'] = args.lmodule
 243     context['jenkins_build_id'] = args.build_id
 244
 245     context['kprobe_round_nb'] = 10
 246
 247     render = jinja_template.render(context)
 248
 249     print('Job to be submitted:')
 250
 251     print(render)
 252
 253     if args.debug:
 254         return 0
 255
 256     server = xmlrpc.client.ServerProxy('http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME))
 257
 258     jobid = server.scheduler.submit_job(render)
 259
 260     print('Lava jobid:{}'.format(jobid))
 261     print('Lava job URL: http://lava-master-02.internal.efficios.com/scheduler/job/{}/log_file'.format(jobid))
 262
 263     #Check the status of the job every 30 seconds
 264     jobstatus = server.scheduler.job_status(jobid)['job_status']
 265     not_running = False
 266     while jobstatus in 'Submitted' or jobstatus in 'Running':
 267         if not_running is False and jobstatus in 'Running':
 268             print('Job started running')
 269             not_running = True
 270         time.sleep(30)
 271         jobstatus = server.scheduler.job_status(jobid)['job_status']
 272
 273 #    Do not fetch result for now
 274 #    if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests:
 275 #        print_test_output(server, jobid)
 276 #    elif test_type is TestType.baremetal_benchmarks:
 277 #        fetch_benchmark_results(server, jobid)
 278
 279     print('Job ended with {} status.'.format(jobstatus))
 280     if jobstatus not in 'Complete':
 281         return -1
 282     else:
 283         passed, failed=check_job_all_test_cases_state_count(server, jobid)
 284         print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
 285
 286         if failed == 0:
 287             return 0
 288         else:
 289             return -1
 290
 291 if __name__ == "__main__":
 292     sys.exit(main())