scripts/system-tests/lava2-submit.py

   1 #!/usr/bin/python3
   2 # Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import argparse
  18 import json
  19 import os
  20 import random
  21 import sys
  22 import time
  23 import xmlrpc.client
  24 from urllib.parse import urljoin
  25 from urllib.request import urlretrieve
  26 import yaml
  27 from jinja2 import Environment, FileSystemLoader
  28
  29 USERNAME = 'lava-jenkins'
  30 HOSTNAME = 'lava-master-02.internal.efficios.com'
  31 OBJSTORE_URL = "https://obj.internal.efficios.com/lava/results/"
  32
  33 class TestType():
  34     """ Enum like for test type """
  35     baremetal_benchmarks = 1
  36     baremetal_tests = 2
  37     kvm_tests = 3
  38     kvm_fuzzing_tests = 4
  39     values = {
  40         'baremetal-benchmarks' : baremetal_benchmarks,
  41         'baremetal-tests' : baremetal_tests,
  42         'kvm-tests' : kvm_tests,
  43         'kvm-fuzzing-tests' : kvm_fuzzing_tests,
  44     }
  45
  46 class DeviceType():
  47     """ Enum like for device type """
  48     x86 = 'x86'
  49     kvm = 'qemu'
  50     values = {
  51         'kvm' : kvm,
  52         'x86' : x86,
  53     }
  54
  55 def get_job_bundle_content(server, job):
  56     try:
  57         bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
  58         bundle = server.dashboard.get(bundle_sha)
  59     except xmlrpc.client.Fault as error:
  60         print('Error while fetching results bundle', error.faultString)
  61         raise error
  62
  63     return json.loads(bundle['content'])
  64
  65 def check_job_all_test_cases_state_count(server, job):
  66     """
  67     Parse the results bundle to see the run-tests testcase
  68     of the lttng-kernel-tests passed successfully
  69     """
  70     print("Testcase result:")
  71     content = server.results.get_testjob_results_yaml(str(job))
  72     testcases = yaml.load(content)
  73
  74     passed_tests = 0
  75     failed_tests = 0
  76     for testcase in testcases:
  77         if testcase['result'] != 'pass':
  78             print("\tFAILED {}\n\t\t See http://{}{}".format(
  79                 testcase['name'],
  80                 HOSTNAME,
  81                 testcase['url']
  82             ))
  83             failed_tests += 1
  84         else:
  85             passed_tests += 1
  86     return (passed_tests, failed_tests)
  87
  88 def fetch_benchmark_results(build_id):
  89     """
  90     Get the benchmark results from the objstore
  91     save them as CSV files localy
  92     """
  93     testcases = ['processed_results_close.csv',
  94                  'processed_results_ioctl.csv',
  95                  'processed_results_open_efault.csv',
  96                  'processed_results_open_enoent.csv',
  97                  'processed_results_dup_close.csv',
  98                  'processed_results_raw_syscall_getpid.csv',
  99                  'processed_results_lttng_test_filter.csv']
 100     for testcase in testcases:
 101         url = urljoin(OBJSTORE_URL, "{:s}/{:s}".format(build_id, testcase))
 102         urlretrieve(url, testcase)
 103
 104 def print_test_output(server, job):
 105     """
 106     Parse the attachment of the testcase to fetch the stdout of the test suite
 107     """
 108     job_finished, log = server.scheduler.jobs.logs(str(job))
 109     logs = yaml.load(log.data.decode('ascii'))
 110     print_line = False
 111     for line in logs:
 112         if line['lvl'] != 'target':
 113             continue
 114         if line['msg'] == '<LAVA_SIGNAL_STARTTC run-tests>':
 115             print('---- TEST SUITE OUTPUT BEGIN ----')
 116             print_line = True
 117             continue
 118         if line['msg'] == '<LAVA_SIGNAL_ENDTC run-tests>':
 119             print('----- TEST SUITE OUTPUT END -----')
 120             break
 121         if print_line:
 122             print("{} {}".format(line['dt'], line['msg']))
 123
 124 def get_vlttng_cmd(lttng_tools_commit, lttng_ust_commit=None):
 125     """
 126     Return vlttng cmd to be used in the job template for setup.
 127     """
 128
 129     vlttng_cmd = 'vlttng --jobs=$(nproc) --profile urcu-master' \
 130                     ' --override projects.babeltrace.build-env.PYTHON=python3' \
 131                     ' --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config' \
 132                     ' --profile babeltrace-stable-1.4' \
 133                     ' --profile babeltrace-python' \
 134                     ' --profile lttng-tools-master' \
 135                     ' --override projects.lttng-tools.checkout='+lttng_tools_commit + \
 136                     ' --profile lttng-tools-no-man-pages'
 137
 138     if lttng_ust_commit is not None:
 139         vlttng_cmd += ' --profile lttng-ust-master ' \
 140                     ' --override projects.lttng-ust.checkout='+lttng_ust_commit+ \
 141                     ' --profile lttng-ust-no-man-pages'
 142
 143     vlttng_path = '/tmp/virtenv'
 144
 145     vlttng_cmd += ' ' + vlttng_path
 146
 147     return vlttng_cmd
 148
 149 def main():
 150     nfsrootfs = "https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_xenial_2018-12-05.tar.gz"
 151     test_type = None
 152     parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
 153     parser.add_argument('-t', '--type', required=True)
 154     parser.add_argument('-j', '--jobname', required=True)
 155     parser.add_argument('-k', '--kernel', required=True)
 156     parser.add_argument('-lm', '--lmodule', required=True)
 157     parser.add_argument('-tc', '--tools-commit', required=True)
 158     parser.add_argument('-id', '--build-id', required=True)
 159     parser.add_argument('-uc', '--ust-commit', required=False)
 160     parser.add_argument('-d', '--debug', required=False, action='store_true')
 161     args = parser.parse_args()
 162
 163     if args.type not in TestType.values:
 164         print('argument -t/--type {} unrecognized.'.format(args.type))
 165         print('Possible values are:')
 166         for k in TestType.values:
 167             print('\t {}'.format(k))
 168         return -1
 169
 170     lava_api_key = None
 171     if not args.debug:
 172         try:
 173             lava_api_key = os.environ['LAVA2_JENKINS_TOKEN']
 174         except Exception as error:
 175             print('LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...',
 176                   error)
 177             return -1
 178
 179     jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__)))
 180     jinja_env = Environment(loader=jinja_loader, trim_blocks=True,
 181                             lstrip_blocks=True)
 182     jinja_template = jinja_env.get_template('template_lava_job.jinja2')
 183
 184     test_type = TestType.values[args.type]
 185
 186     if test_type in [TestType.baremetal_benchmarks, TestType.baremetal_tests]:
 187         device_type = DeviceType.x86
 188     else:
 189         device_type = DeviceType.kvm
 190
 191     vlttng_path = '/tmp/virtenv'
 192
 193     vlttng_cmd = get_vlttng_cmd(args.tools_commit, args.ust_commit)
 194
 195     context = dict()
 196     context['DeviceType'] = DeviceType
 197     context['TestType'] = TestType
 198
 199     context['job_name'] = args.jobname
 200     context['test_type'] = test_type
 201     context['random_seed'] = random.randint(0, 1000000)
 202     context['device_type'] = device_type
 203
 204     context['vlttng_cmd'] = vlttng_cmd
 205     context['vlttng_path'] = vlttng_path
 206
 207     context['kernel_url'] = args.kernel
 208     context['nfsrootfs_url'] = nfsrootfs
 209     context['lttng_modules_url'] = args.lmodule
 210     context['jenkins_build_id'] = args.build_id
 211
 212     context['kprobe_round_nb'] = 10
 213
 214     render = jinja_template.render(context)
 215
 216     print('Job to be submitted:')
 217
 218     print(render)
 219
 220     if args.debug:
 221         return 0
 222
 223     server = xmlrpc.client.ServerProxy('http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME))
 224
 225     for attempt in range(10):
 226         try:
 227             jobid = server.scheduler.submit_job(render)
 228         except xmlrpc.client.ProtocolError as error:
 229             print('Protocol error on submit, sleeping and retrying. Attempt #{}'
 230                   .format(attempt))
 231             time.sleep(5)
 232             continue
 233         else:
 234             break
 235
 236     print('Lava jobid:{}'.format(jobid))
 237     print('Lava job URL: http://lava-master-02.internal.efficios.com/scheduler/job/{}'.format(jobid))
 238
 239     #Check the status of the job every 30 seconds
 240     jobstatus = server.scheduler.job_state(jobid)['job_state']
 241     running = False
 242     while jobstatus in ['Submitted', 'Scheduling', 'Scheduled', 'Running']:
 243         if not running and jobstatus == 'Running':
 244             print('Job started running')
 245             running = True
 246         time.sleep(30)
 247         try:
 248             jobstatus = server.scheduler.job_state(jobid)['job_state']
 249         except xmlrpc.client.ProtocolError as error:
 250             print('Protocol error, retrying')
 251             continue
 252     print('Job ended with {} status.'.format(jobstatus))
 253
 254     if jobstatus != 'Finished':
 255         return -1
 256
 257     if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests:
 258         print_test_output(server, jobid)
 259     elif test_type is TestType.baremetal_benchmarks:
 260         fetch_benchmark_results(args.build_id)
 261
 262     passed, failed = check_job_all_test_cases_state_count(server, jobid)
 263     print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
 264
 265     if failed != 0:
 266         return -1
 267
 268     return 0
 269
 270 if __name__ == "__main__":
 271     sys.exit(main())