scripts/system-tests/lava2-submit.py

   1 #!/usr/bin/python3
   2 # Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import argparse
  18 import json
  19 import os
  20 import random
  21 import sys
  22 import time
  23 import xmlrpc.client
  24 from urllib.parse import urljoin
  25 from urllib.request import urlretrieve
  26 import yaml
  27 from jinja2 import Environment, FileSystemLoader
  28
  29 USERNAME = 'lava-jenkins'
  30 HOSTNAME = 'lava-master-02.internal.efficios.com'
  31 OBJSTORE_URL = "https://obj.internal.efficios.com/lava/results/"
  32
  33 class TestType():
  34     """ Enum like for test type """
  35     baremetal_benchmarks = 1
  36     baremetal_tests = 2
  37     kvm_tests = 3
  38     kvm_fuzzing_tests = 4
  39     values = {
  40         'baremetal-benchmarks' : baremetal_benchmarks,
  41         'baremetal-tests' : baremetal_tests,
  42         'kvm-tests' : kvm_tests,
  43         'kvm-fuzzing-tests' : kvm_fuzzing_tests,
  44     }
  45
  46 class DeviceType():
  47     """ Enum like for device type """
  48     x86 = 'x86'
  49     kvm = 'qemu'
  50     values = {
  51         'kvm' : kvm,
  52         'x86' : x86,
  53     }
  54
  55 def get_job_bundle_content(server, job):
  56     try:
  57         bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
  58         bundle = server.dashboard.get(bundle_sha)
  59     except xmlrpc.client.Fault as error:
  60         print('Error while fetching results bundle', error.faultString)
  61         raise error
  62
  63     return json.loads(bundle['content'])
  64
  65 def check_job_all_test_cases_state_count(server, job):
  66     """
  67     Parse the results bundle to see the run-tests testcase
  68     of the lttng-kernel-tests passed successfully
  69     """
  70     print("Testcase result:")
  71     content = server.results.get_testjob_results_yaml(str(job))
  72     testcases = yaml.load(content)
  73
  74     passed_tests = 0
  75     failed_tests = 0
  76     for testcase in testcases:
  77         if testcase['result'] != 'pass':
  78             print("\tFAILED {}\n\t\t See http://{}{}".format(
  79                 testcase['name'],
  80                 HOSTNAME,
  81                 testcase['url']
  82             ))
  83             failed_tests += 1
  84         else:
  85             passed_tests += 1
  86     return (passed_tests, failed_tests)
  87
  88 def fetch_benchmark_results(build_id):
  89     """
  90     Get the benchmark results from the objstore
  91     save them as CSV files localy
  92     """
  93     testcases = ['processed_results_close.csv',
  94                  'processed_results_ioctl.csv',
  95                  'processed_results_open_efault.csv',
  96                  'processed_results_open_enoent.csv',
  97                  'processed_results_dup_close.csv',
  98                  'processed_results_raw_syscall_getpid.csv',
  99                  'processed_results_lttng_test_filter.csv']
 100     for testcase in testcases:
 101         url = urljoin(OBJSTORE_URL, "{:s}/{:s}".format(build_id, testcase))
 102         print('Fetching {}'.format(url))
 103         urlretrieve(url, testcase)
 104
 105 def print_test_output(server, job):
 106     """
 107     Parse the attachment of the testcase to fetch the stdout of the test suite
 108     """
 109     job_finished, log = server.scheduler.jobs.logs(str(job))
 110     logs = yaml.load(log.data.decode('ascii'))
 111     print_line = False
 112     for line in logs:
 113         if line['lvl'] != 'target':
 114             continue
 115         if line['msg'] == '<LAVA_SIGNAL_STARTTC run-tests>':
 116             print('---- TEST SUITE OUTPUT BEGIN ----')
 117             print_line = True
 118             continue
 119         if line['msg'] == '<LAVA_SIGNAL_ENDTC run-tests>':
 120             print('----- TEST SUITE OUTPUT END -----')
 121             break
 122         if print_line:
 123             print("{} {}".format(line['dt'], line['msg']))
 124
 125 def get_vlttng_cmd(lttng_tools_url, lttng_tools_commit, lttng_ust_url=None, lttng_ust_commit=None):
 126     """
 127     Return vlttng cmd to be used in the job template for setup.
 128     """
 129
 130     vlttng_cmd = 'vlttng --jobs=$(nproc) --profile urcu-master' \
 131                     ' --override projects.babeltrace.build-env.PYTHON=python3' \
 132                     ' --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config' \
 133                     ' --profile babeltrace-stable-1.4' \
 134                     ' --profile babeltrace-python' \
 135                     ' --profile lttng-tools-master' \
 136                     ' --override projects.lttng-tools.source='+lttng_tools_url + \
 137                     ' --override projects.lttng-tools.checkout='+lttng_tools_commit + \
 138                     ' --profile lttng-tools-no-man-pages'
 139
 140     if lttng_ust_commit is not None:
 141         vlttng_cmd += ' --profile lttng-ust-master ' \
 142                     ' --override projects.lttng-ust.source='+lttng_ust_url + \
 143                     ' --override projects.lttng-ust.checkout='+lttng_ust_commit+ \
 144                     ' --profile lttng-ust-no-man-pages'
 145
 146     vlttng_path = '/tmp/virtenv'
 147
 148     vlttng_cmd += ' ' + vlttng_path
 149
 150     return vlttng_cmd
 151
 152 def main():
 153     nfsrootfs = "https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_xenial_2018-12-05.tar.gz"
 154     test_type = None
 155     parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
 156     parser.add_argument('-t', '--type', required=True)
 157     parser.add_argument('-j', '--jobname', required=True)
 158     parser.add_argument('-k', '--kernel', required=True)
 159     parser.add_argument('-lm', '--lmodule', required=True)
 160     parser.add_argument('-tu', '--tools-url', required=True)
 161     parser.add_argument('-tc', '--tools-commit', required=True)
 162     parser.add_argument('-id', '--build-id', required=True)
 163     parser.add_argument('-uu', '--ust-url', required=False)
 164     parser.add_argument('-uc', '--ust-commit', required=False)
 165     parser.add_argument('-d', '--debug', required=False, action='store_true')
 166     args = parser.parse_args()
 167
 168     if args.type not in TestType.values:
 169         print('argument -t/--type {} unrecognized.'.format(args.type))
 170         print('Possible values are:')
 171         for k in TestType.values:
 172             print('\t {}'.format(k))
 173         return -1
 174
 175     lava_api_key = None
 176     if not args.debug:
 177         try:
 178             lava_api_key = os.environ['LAVA2_JENKINS_TOKEN']
 179         except Exception as error:
 180             print('LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...',
 181                   error)
 182             return -1
 183
 184     jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__)))
 185     jinja_env = Environment(loader=jinja_loader, trim_blocks=True,
 186                             lstrip_blocks=True)
 187     jinja_template = jinja_env.get_template('template_lava_job.jinja2')
 188
 189     test_type = TestType.values[args.type]
 190
 191     if test_type in [TestType.baremetal_benchmarks, TestType.baremetal_tests]:
 192         device_type = DeviceType.x86
 193     else:
 194         device_type = DeviceType.kvm
 195
 196     vlttng_path = '/tmp/virtenv'
 197
 198     vlttng_cmd = get_vlttng_cmd(args.tools_url, args.tools_commit, args.ust_url, args.ust_commit)
 199
 200     context = dict()
 201     context['DeviceType'] = DeviceType
 202     context['TestType'] = TestType
 203
 204     context['job_name'] = args.jobname
 205     context['test_type'] = test_type
 206     context['random_seed'] = random.randint(0, 1000000)
 207     context['device_type'] = device_type
 208
 209     context['vlttng_cmd'] = vlttng_cmd
 210     context['vlttng_path'] = vlttng_path
 211
 212     context['kernel_url'] = args.kernel
 213     context['nfsrootfs_url'] = nfsrootfs
 214     context['lttng_modules_url'] = args.lmodule
 215     context['jenkins_build_id'] = args.build_id
 216
 217     context['kprobe_round_nb'] = 10
 218
 219     render = jinja_template.render(context)
 220
 221     print('Job to be submitted:')
 222
 223     print(render)
 224
 225     if args.debug:
 226         return 0
 227
 228     server = xmlrpc.client.ServerProxy('http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME))
 229
 230     for attempt in range(10):
 231         try:
 232             jobid = server.scheduler.submit_job(render)
 233         except xmlrpc.client.ProtocolError as error:
 234             print('Protocol error on submit, sleeping and retrying. Attempt #{}'
 235                   .format(attempt))
 236             time.sleep(5)
 237             continue
 238         else:
 239             break
 240
 241     print('Lava jobid:{}'.format(jobid))
 242     print('Lava job URL: http://lava-master-02.internal.efficios.com/scheduler/job/{}'.format(jobid))
 243
 244     #Check the status of the job every 30 seconds
 245     jobstatus = server.scheduler.job_state(jobid)['job_state']
 246     running = False
 247     while jobstatus in ['Submitted', 'Scheduling', 'Scheduled', 'Running']:
 248         if not running and jobstatus == 'Running':
 249             print('Job started running')
 250             running = True
 251         time.sleep(30)
 252         try:
 253             jobstatus = server.scheduler.job_state(jobid)['job_state']
 254         except xmlrpc.client.ProtocolError as error:
 255             print('Protocol error, retrying')
 256             continue
 257     print('Job ended with {} status.'.format(jobstatus))
 258
 259     if jobstatus != 'Finished':
 260         return -1
 261
 262     if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests:
 263         print_test_output(server, jobid)
 264     elif test_type is TestType.baremetal_benchmarks:
 265         fetch_benchmark_results(args.build_id)
 266
 267     passed, failed = check_job_all_test_cases_state_count(server, jobid)
 268     print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
 269
 270     if failed != 0:
 271         return -1
 272
 273     return 0
 274
 275 if __name__ == "__main__":
 276     sys.exit(main())