scripts/system-tests/lava2-submit.py

   1 #!/usr/bin/python3
   2 # Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import argparse
  18 import json
  19 import os
  20 import random
  21 import sys
  22 import time
  23 import xmlrpc.client
  24 from urllib.parse import urljoin
  25 from urllib.request import urlretrieve
  26 import yaml
  27 from jinja2 import Environment, FileSystemLoader
  28
  29 USERNAME = 'lava-jenkins'
  30 HOSTNAME = 'lava-master-02.internal.efficios.com'
  31 OBJSTORE_URL = "https://obj.internal.efficios.com/lava/results/"
  32
  33 class TestType():
  34     """ Enum like for test type """
  35     baremetal_benchmarks = 1
  36     baremetal_tests = 2
  37     kvm_tests = 3
  38     kvm_fuzzing_tests = 4
  39     values = {
  40         'baremetal-benchmarks' : baremetal_benchmarks,
  41         'baremetal-tests' : baremetal_tests,
  42         'kvm-tests' : kvm_tests,
  43         'kvm-fuzzing-tests' : kvm_fuzzing_tests,
  44     }
  45
  46 class DeviceType():
  47     """ Enum like for device type """
  48     x86 = 'x86'
  49     kvm = 'qemu'
  50     values = {
  51         'kvm' : kvm,
  52         'x86' : x86,
  53     }
  54
  55 def get_job_bundle_content(server, job):
  56     try:
  57         bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
  58         bundle = server.dashboard.get(bundle_sha)
  59     except xmlrpc.client.Fault as error:
  60         print('Error while fetching results bundle', error.faultString)
  61         raise error
  62
  63     return json.loads(bundle['content'])
  64
  65 def check_job_all_test_cases_state_count(server, job):
  66     """
  67     Parse the results bundle to see the run-tests testcase
  68     of the lttng-kernel-tests passed successfully
  69     """
  70     print("Testcase result:")
  71     content = server.results.get_testjob_results_yaml(str(job))
  72     testcases = yaml.load(content)
  73
  74     passed_tests = 0
  75     failed_tests = 0
  76     for testcase in testcases:
  77         if testcase['result'] != 'pass':
  78             print("\tFAILED {}\n\t\t See http://{}{}".format(
  79                 testcase['name'],
  80                 HOSTNAME,
  81                 testcase['url']
  82             ))
  83             failed_tests += 1
  84         else:
  85             passed_tests += 1
  86     return (passed_tests, failed_tests)
  87
  88 def fetch_benchmark_results(build_id):
  89     """
  90     Get the benchmark results from the objstore
  91     save them as CSV files localy
  92     """
  93     testcases = ['processed_results_close.csv',
  94                  'processed_results_ioctl.csv',
  95                  'processed_results_open_efault.csv',
  96                  'processed_results_open_enoent.csv',
  97                  'processed_results_dup_close.csv',
  98                  'processed_results_raw_syscall_getpid.csv',
  99                  'processed_results_lttng_test_filter.csv']
 100     for testcase in testcases:
 101         url = urljoin(OBJSTORE_URL, "{:s}/{:s}".format(build_id, testcase))
 102         print('Fetching {}'.format(url))
 103         urlretrieve(url, testcase)
 104
 105 def print_test_output(server, job):
 106     """
 107     Parse the attachment of the testcase to fetch the stdout of the test suite
 108     """
 109     job_finished, log = server.scheduler.jobs.logs(str(job))
 110     logs = yaml.load(log.data.decode('ascii'))
 111     print_line = False
 112     for line in logs:
 113         if line['lvl'] != 'target':
 114             continue
 115         if line['msg'] == '<LAVA_SIGNAL_STARTTC run-tests>':
 116             print('---- TEST SUITE OUTPUT BEGIN ----')
 117             print_line = True
 118             continue
 119         if line['msg'] == '<LAVA_SIGNAL_ENDTC run-tests>':
 120             print('----- TEST SUITE OUTPUT END -----')
 121             break
 122         if print_line:
 123             print("{} {}".format(line['dt'], line['msg']))
 124
 125 def get_vlttng_cmd(lttng_tools_commit, lttng_ust_commit=None):
 126     """
 127     Return vlttng cmd to be used in the job template for setup.
 128     """
 129
 130     vlttng_cmd = 'vlttng --jobs=$(nproc) --profile urcu-master' \
 131                     ' --override projects.babeltrace.build-env.PYTHON=python3' \
 132                     ' --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config' \
 133                     ' --profile babeltrace-stable-1.4' \
 134                     ' --profile babeltrace-python' \
 135                     ' --profile lttng-tools-master' \
 136                     ' --override projects.lttng-tools.checkout='+lttng_tools_commit + \
 137                     ' --profile lttng-tools-no-man-pages'
 138
 139     if lttng_ust_commit is not None:
 140         vlttng_cmd += ' --profile lttng-ust-master ' \
 141                     ' --override projects.lttng-ust.checkout='+lttng_ust_commit+ \
 142                     ' --profile lttng-ust-no-man-pages'
 143
 144     vlttng_path = '/tmp/virtenv'
 145
 146     vlttng_cmd += ' ' + vlttng_path
 147
 148     return vlttng_cmd
 149
 150 def main():
 151     nfsrootfs = "https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_xenial_2018-12-05.tar.gz"
 152     test_type = None
 153     parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
 154     parser.add_argument('-t', '--type', required=True)
 155     parser.add_argument('-j', '--jobname', required=True)
 156     parser.add_argument('-k', '--kernel', required=True)
 157     parser.add_argument('-lm', '--lmodule', required=True)
 158     parser.add_argument('-tc', '--tools-commit', required=True)
 159     parser.add_argument('-id', '--build-id', required=True)
 160     parser.add_argument('-uc', '--ust-commit', required=False)
 161     parser.add_argument('-d', '--debug', required=False, action='store_true')
 162     args = parser.parse_args()
 163
 164     if args.type not in TestType.values:
 165         print('argument -t/--type {} unrecognized.'.format(args.type))
 166         print('Possible values are:')
 167         for k in TestType.values:
 168             print('\t {}'.format(k))
 169         return -1
 170
 171     lava_api_key = None
 172     if not args.debug:
 173         try:
 174             lava_api_key = os.environ['LAVA2_JENKINS_TOKEN']
 175         except Exception as error:
 176             print('LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...',
 177                   error)
 178             return -1
 179
 180     jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__)))
 181     jinja_env = Environment(loader=jinja_loader, trim_blocks=True,
 182                             lstrip_blocks=True)
 183     jinja_template = jinja_env.get_template('template_lava_job.jinja2')
 184
 185     test_type = TestType.values[args.type]
 186
 187     if test_type in [TestType.baremetal_benchmarks, TestType.baremetal_tests]:
 188         device_type = DeviceType.x86
 189     else:
 190         device_type = DeviceType.kvm
 191
 192     vlttng_path = '/tmp/virtenv'
 193
 194     vlttng_cmd = get_vlttng_cmd(args.tools_commit, args.ust_commit)
 195
 196     context = dict()
 197     context['DeviceType'] = DeviceType
 198     context['TestType'] = TestType
 199
 200     context['job_name'] = args.jobname
 201     context['test_type'] = test_type
 202     context['random_seed'] = random.randint(0, 1000000)
 203     context['device_type'] = device_type
 204
 205     context['vlttng_cmd'] = vlttng_cmd
 206     context['vlttng_path'] = vlttng_path
 207
 208     context['kernel_url'] = args.kernel
 209     context['nfsrootfs_url'] = nfsrootfs
 210     context['lttng_modules_url'] = args.lmodule
 211     context['jenkins_build_id'] = args.build_id
 212
 213     context['kprobe_round_nb'] = 10
 214
 215     render = jinja_template.render(context)
 216
 217     print('Job to be submitted:')
 218
 219     print(render)
 220
 221     if args.debug:
 222         return 0
 223
 224     server = xmlrpc.client.ServerProxy('http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME))
 225
 226     for attempt in range(10):
 227         try:
 228             jobid = server.scheduler.submit_job(render)
 229         except xmlrpc.client.ProtocolError as error:
 230             print('Protocol error on submit, sleeping and retrying. Attempt #{}'
 231                   .format(attempt))
 232             time.sleep(5)
 233             continue
 234         else:
 235             break
 236
 237     print('Lava jobid:{}'.format(jobid))
 238     print('Lava job URL: http://lava-master-02.internal.efficios.com/scheduler/job/{}'.format(jobid))
 239
 240     #Check the status of the job every 30 seconds
 241     jobstatus = server.scheduler.job_state(jobid)['job_state']
 242     running = False
 243     while jobstatus in ['Submitted', 'Scheduling', 'Scheduled', 'Running']:
 244         if not running and jobstatus == 'Running':
 245             print('Job started running')
 246             running = True
 247         time.sleep(30)
 248         try:
 249             jobstatus = server.scheduler.job_state(jobid)['job_state']
 250         except xmlrpc.client.ProtocolError as error:
 251             print('Protocol error, retrying')
 252             continue
 253     print('Job ended with {} status.'.format(jobstatus))
 254
 255     if jobstatus != 'Finished':
 256         return -1
 257
 258     if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests:
 259         print_test_output(server, jobid)
 260     elif test_type is TestType.baremetal_benchmarks:
 261         fetch_benchmark_results(args.build_id)
 262
 263     passed, failed = check_job_all_test_cases_state_count(server, jobid)
 264     print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
 265
 266     if failed != 0:
 267         return -1
 268
 269     return 0
 270
 271 if __name__ == "__main__":
 272     sys.exit(main())