scripts/system-tests/lava2-submit.py

   1 #!/usr/bin/python
   2 # Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import argparse
  18 import base64
  19 import json
  20 import os
  21 import random
  22 import sys
  23 import time
  24 import yaml
  25 import xmlrpc.client
  26 import pprint
  27
  28 from jinja2 import Environment, FileSystemLoader, meta
  29
  30 USERNAME = 'lava-jenkins'
  31 HOSTNAME = 'lava-master-02.internal.efficios.com'
  32 OBJSTORE_URL = "https://obj.internal.efficios.com/lava/results/"
  33
  34 class TestType():
  35     baremetal_benchmarks=1
  36     baremetal_tests=2
  37     kvm_tests=3
  38     kvm_fuzzing_tests=4
  39     values = {
  40         'baremetal-benchmarks' : baremetal_benchmarks,
  41         'baremetal-tests' : baremetal_tests,
  42         'kvm-tests' : kvm_tests,
  43         'kvm-fuzzin-tests' : kvm_fuzzing_tests,
  44     }
  45
  46 class DeviceType():
  47     x86 = 'x86'
  48     kvm = 'qemu'
  49     values = {
  50         'kvm' : kvm,
  51         'x86' : x86,
  52     }
  53
  54 def get_job_bundle_content(server, job):
  55     try:
  56         bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
  57         bundle = server.dashboard.get(bundle_sha)
  58     except xmlrpc.client.Fault as f:
  59         print('Error while fetching results bundle', f.faultString)
  60         raise f
  61
  62     return json.loads(bundle['content'])
  63
  64 # Parse the results bundle to see the run-tests testcase
  65 # of the lttng-kernel-tests passed successfully
  66 def check_job_all_test_cases_state_count(server, job):
  67     print("Testcase result:")
  68     content = server.results.get_testjob_results_yaml(str(job))
  69     testcases = yaml.load(content)
  70
  71     passed_tests=0
  72     failed_tests=0
  73     for testcase in testcases:
  74         if testcase['result'] != 'pass':
  75             print("\tFAILED {}\n\t\t See http://{}{}".format(
  76                 testcase['name'],
  77                 HOSTNAME,
  78                 testcase['url']
  79             ))
  80             failed_tests+=1
  81         else:
  82             passed_tests+=1
  83     return (passed_tests, failed_tests)
  84
  85 # Get the benchmark results from the objstore
  86 # save them as CSV files localy
  87 def fetch_benchmark_results(build_id):
  88     testcases = ['processed_results_close.csv',
  89             'processed_results_ioctl.csv',
  90             'processed_results_open_efault.csv',
  91             'processed_results_open_enoent.csv',
  92             'processed_results_dup_close.csv',
  93             'processed_results_raw_syscall_getpid.csv',
  94             'processed_results_lttng_test_filter.csv']
  95     for testcase in testcases:
  96         url = urljoin(OBJSTORE_URL, "{:s}/{:s}".format(build_id, testcase))
  97         urlretrieve(url, testcase)
  98
  99 # Parse the attachment of the testcase to fetch the stdout of the test suite
 100 def print_test_output(server, job):
 101     job_finished, log = server.scheduler.jobs.logs(str(job))
 102     logs = yaml.load(log.data.decode('ascii'))
 103     print_line = False
 104     for line in logs:
 105         if line['lvl'] != 'target':
 106             continue
 107         if line['msg'] == '<LAVA_SIGNAL_STARTTC run-tests>':
 108             print('---- TEST SUITE OUTPUT BEGIN ----')
 109             print_line = True
 110             continue
 111         if line['msg'] == '<LAVA_SIGNAL_ENDTC run-tests>':
 112             print('----- TEST SUITE OUTPUT END -----')
 113             break
 114         if print_line:
 115             print("{} {}".format(line['dt'], line['msg']))
 116
 117 def get_vlttng_cmd(device, lttng_tools_commit, lttng_ust_commit=None):
 118
 119     vlttng_cmd = 'vlttng --jobs=$(nproc) --profile urcu-master' \
 120                     ' --override projects.babeltrace.build-env.PYTHON=python3' \
 121                     ' --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config' \
 122                     ' --profile babeltrace-stable-1.4' \
 123                     ' --profile babeltrace-python' \
 124                     ' --profile lttng-tools-master' \
 125                     ' --override projects.lttng-tools.checkout='+lttng_tools_commit + \
 126                     ' --profile lttng-tools-no-man-pages' \
 127                     ' --verbose'
 128
 129     if lttng_ust_commit is not None:
 130         vlttng_cmd += ' --profile lttng-ust-master ' \
 131                     ' --override projects.lttng-ust.checkout='+lttng_ust_commit+ \
 132                     ' --profile lttng-ust-no-man-pages'
 133
 134     vlttng_path = '/tmp/virtenv'
 135
 136     vlttng_cmd += ' ' + vlttng_path
 137
 138     return vlttng_cmd
 139
 140 def main():
 141     nfsrootfs = "https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_xenial_2018-12-05.tar.gz"
 142     test_type = None
 143     parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
 144     parser.add_argument('-t', '--type', required=True)
 145     parser.add_argument('-j', '--jobname', required=True)
 146     parser.add_argument('-k', '--kernel', required=True)
 147     parser.add_argument('-lm', '--lmodule', required=True)
 148     parser.add_argument('-tc', '--tools-commit', required=True)
 149     parser.add_argument('-id', '--build-id', required=True)
 150     parser.add_argument('-uc', '--ust-commit', required=False)
 151     parser.add_argument('-d', '--debug', required=False, action='store_true')
 152     args = parser.parse_args()
 153
 154     if args.type not in TestType.values:
 155         print('argument -t/--type {} unrecognized.'.format(args.type))
 156         print('Possible values are:')
 157         for k in TestType.values:
 158             print('\t {}'.format(k))
 159         return -1
 160
 161     lava_api_key = None
 162     if not args.debug:
 163         try:
 164             lava_api_key = os.environ['LAVA2_JENKINS_TOKEN']
 165         except Exception as e:
 166             print('LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...', e )
 167             return -1
 168
 169     jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__)))
 170     jinja_env = Environment(loader=jinja_loader, trim_blocks=True,
 171             lstrip_blocks= True)
 172     jinja_template = jinja_env.get_template('template_lava_job.jinja2')
 173     template_source = jinja_env.loader.get_source(jinja_env, 'template_lava_job.jinja2')
 174     parsed_content = jinja_env.parse(template_source)
 175     undef = meta.find_undeclared_variables(parsed_content)
 176
 177     test_type = TestType.values[args.type]
 178
 179     if test_type in [TestType.baremetal_benchmarks, TestType.baremetal_tests]:
 180         device_type = DeviceType.x86
 181     else:
 182         device_type = DeviceType.kvm
 183
 184     vlttng_path = '/tmp/virtenv'
 185
 186     vlttng_cmd = get_vlttng_cmd(device_type, args.tools_commit, args.ust_commit)
 187
 188     context = dict()
 189     context['DeviceType'] = DeviceType
 190     context['TestType'] = TestType
 191
 192     context['job_name'] = args.jobname
 193     context['test_type'] = test_type
 194     context['random_seed'] = random.randint(0, 1000000)
 195     context['device_type'] = device_type
 196
 197     context['vlttng_cmd'] = vlttng_cmd
 198     context['vlttng_path'] = vlttng_path
 199
 200     context['kernel_url'] = args.kernel
 201     context['nfsrootfs_url'] = nfsrootfs
 202     context['lttng_modules_url'] = args.lmodule
 203     context['jenkins_build_id'] = args.build_id
 204
 205     context['kprobe_round_nb'] = 10
 206
 207     render = jinja_template.render(context)
 208
 209     print('Job to be submitted:')
 210
 211     print(render)
 212
 213     if args.debug:
 214         return 0
 215
 216     server = xmlrpc.client.ServerProxy('http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME))
 217
 218     jobid = server.scheduler.submit_job(render)
 219
 220     print('Lava jobid:{}'.format(jobid))
 221     print('Lava job URL: http://lava-master-02.internal.efficios.com/scheduler/job/{}'.format(jobid))
 222
 223     #Check the status of the job every 30 seconds
 224     jobstatus = server.scheduler.job_state(jobid)['job_state']
 225     running = False
 226     while jobstatus in ['Submitted','Scheduling','Scheduled','Running']:
 227         if not running and jobstatus == 'Running':
 228             print('Job started running')
 229             running = True
 230         time.sleep(30)
 231         jobstatus = server.scheduler.job_state(jobid)['job_state']
 232     print('Job ended with {} status.'.format(jobstatus))
 233
 234     if jobstatus != 'Finished':
 235         return -1
 236
 237     if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests:
 238         print_test_output(server, jobid)
 239     elif test_type is TestType.baremetal_benchmarks:
 240         fetch_benchmark_results(args.build_id)
 241
 242     passed, failed=check_job_all_test_cases_state_count(server, jobid)
 243     print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
 244
 245     if failed == 0:
 246         return 0
 247     else:
 248         return -1
 249
 250 if __name__ == "__main__":
 251     sys.exit(main())