scripts/system-tests/lava2-submit.py

   1 #!/usr/bin/python3
   2 # Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import argparse
  18 import json
  19 import os
  20 import random
  21 import re
  22 import sys
  23 import time
  24 import xmlrpc.client
  25 from urllib.parse import urljoin
  26 from urllib.request import urlretrieve
  27 import yaml
  28 from jinja2 import Environment, FileSystemLoader
  29
  30 USERNAME = 'lava-jenkins'
  31 HOSTNAME = 'lava-master-02.internal.efficios.com'
  32 OBJSTORE_URL = "https://obj.internal.efficios.com/lava/results/"
  33
  34 def parse_stable_version(stable_version_string):
  35     # Get the major and minor version numbers from the lttng version string.
  36     version_match = re.search('stable-(\d).(\d\d)', stable_version_string)
  37
  38     if version_match is not None:
  39         major_version = int(version_match.group(1))
  40         minor_version = int(version_match.group(2))
  41     else:
  42         # Setting to zero to make the comparison below easier.
  43         major_version = 0
  44         minor_version = 0
  45     return major_version, minor_version
  46
  47
  48 class TestType:
  49     """ Enum like for test type """
  50
  51     baremetal_tests = 1
  52     kvm_tests = 2
  53     values = {
  54         'baremetal-tests': baremetal_tests,
  55         'kvm-tests': kvm_tests,
  56     }
  57
  58
  59 class DeviceType:
  60     """ Enum like for device type """
  61
  62     x86 = 'x86'
  63     kvm = 'qemu'
  64     values = {'kvm': kvm, 'x86': x86}
  65
  66
  67 def get_job_bundle_content(server, job):
  68     try:
  69         bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
  70         bundle = server.dashboard.get(bundle_sha)
  71     except xmlrpc.client.Fault as error:
  72         print('Error while fetching results bundle', error.faultString)
  73         raise error
  74
  75     return json.loads(bundle['content'])
  76
  77
  78 def check_job_all_test_cases_state_count(server, job):
  79     """
  80     Parse the results bundle to see the run-tests testcase
  81     of the lttng-kernel-tests passed successfully
  82     """
  83     print("Testcase result:")
  84     content = server.results.get_testjob_results_yaml(str(job))
  85     testcases = yaml.unsafe_load(content)
  86
  87     passed_tests = 0
  88     failed_tests = 0
  89     for testcase in testcases:
  90         if testcase['result'] != 'pass':
  91             print(
  92                 "\tFAILED {}\n\t\t See http://{}{}".format(
  93                     testcase['name'], HOSTNAME, testcase['url']
  94                 )
  95             )
  96             failed_tests += 1
  97         else:
  98             passed_tests += 1
  99     return (passed_tests, failed_tests)
 100
 101
 102 def print_test_output(server, job):
 103     """
 104     Parse the attachment of the testcase to fetch the stdout of the test suite
 105     """
 106     job_finished, log = server.scheduler.jobs.logs(str(job))
 107     logs = yaml.unsafe_load(log.data.decode('ascii'))
 108     print_line = False
 109     for line in logs:
 110         if line['lvl'] != 'target':
 111             continue
 112         if line['msg'] == '<LAVA_SIGNAL_STARTTC run-tests>':
 113             print('---- TEST SUITE OUTPUT BEGIN ----')
 114             print_line = True
 115             continue
 116         if line['msg'] == '<LAVA_SIGNAL_ENDTC run-tests>':
 117             print('----- TEST SUITE OUTPUT END -----')
 118             print_line = False
 119             continue
 120         if print_line:
 121             print("{} {}".format(line['dt'], line['msg']))
 122
 123
 124 def get_vlttng_cmd(
 125     lttng_version, lttng_tools_url, lttng_tools_commit, lttng_ust_url=None, lttng_ust_commit=None
 126 ):
 127     """
 128     Return vlttng cmd to be used in the job template for setup.
 129     """
 130     urcu_profile = ""
 131     if lttng_version == 'master' or (major_version >= 2 and minor_version >= 11):
 132         urcu_profile = "urcu-master"
 133     else:
 134         urcu_profile = "urcu-stable-0.12"
 135
 136     vlttng_cmd = (
 137         'vlttng --jobs=$(nproc) --profile ' + urcu_profile
 138         + ' --override projects.babeltrace.build-env.PYTHON=python3'
 139         ' --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config'
 140         ' --profile babeltrace-stable-1.5'
 141         ' --profile babeltrace-python'
 142         ' --profile lttng-tools-master'
 143         ' --override projects.lttng-tools.source='
 144         + lttng_tools_url
 145         + ' --override projects.lttng-tools.checkout='
 146         + lttng_tools_commit
 147         + ' --profile lttng-tools-no-man-pages'
 148     )
 149
 150     if lttng_ust_commit is not None:
 151         vlttng_cmd += (
 152             ' --profile lttng-ust-master '
 153             ' --override projects.lttng-ust.source='
 154             + lttng_ust_url
 155             + ' --override projects.lttng-ust.checkout='
 156             + lttng_ust_commit
 157             + ' --profile lttng-ust-no-man-pages'
 158         )
 159
 160     major_version, minor_version = parse_stable_version(lttng_version)
 161
 162     if lttng_version == 'master' or (major_version >= 2 and minor_version >= 11):
 163         vlttng_cmd += (
 164             ' --override projects.lttng-tools.configure+=--enable-test-sdt-uprobe'
 165         )
 166
 167     vlttng_path = '/tmp/virtenv'
 168
 169     vlttng_cmd += ' ' + vlttng_path
 170
 171     return vlttng_cmd
 172
 173
 174 def main():
 175     nfsrootfs = "https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_xenial_2018-12-05.tar.gz"
 176     test_type = None
 177     parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
 178     parser.add_argument('-t', '--type', required=True)
 179     parser.add_argument('-lv', '--lttng-version', required=True)
 180     parser.add_argument('-j', '--jobname', required=True)
 181     parser.add_argument('-k', '--kernel', required=True)
 182     parser.add_argument('-lm', '--lmodule', required=True)
 183     parser.add_argument('-tu', '--tools-url', required=True)
 184     parser.add_argument('-tc', '--tools-commit', required=True)
 185     parser.add_argument('-id', '--build-id', required=True)
 186     parser.add_argument('-uu', '--ust-url', required=False)
 187     parser.add_argument('-uc', '--ust-commit', required=False)
 188     parser.add_argument('-d', '--debug', required=False, action='store_true')
 189     args = parser.parse_args()
 190
 191     if args.type not in TestType.values:
 192         print('argument -t/--type {} unrecognized.'.format(args.type))
 193         print('Possible values are:')
 194         for k in TestType.values:
 195             print('\t {}'.format(k))
 196         return -1
 197
 198     lava_api_key = None
 199     if not args.debug:
 200         try:
 201             lava_api_key = os.environ['LAVA2_JENKINS_TOKEN']
 202         except Exception as error:
 203             print(
 204                 'LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...',
 205                 error,
 206             )
 207             return -1
 208
 209     jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__)))
 210     jinja_env = Environment(loader=jinja_loader, trim_blocks=True, lstrip_blocks=True)
 211     jinja_template = jinja_env.get_template('template_lava_job.jinja2')
 212
 213     test_type = TestType.values[args.type]
 214
 215     if test_type is TestType.baremetal_tests:
 216         device_type = DeviceType.x86
 217     else:
 218         device_type = DeviceType.kvm
 219
 220     vlttng_path = '/tmp/virtenv'
 221
 222     vlttng_cmd = get_vlttng_cmd(
 223         args.lttng_version, args.tools_url, args.tools_commit, args.ust_url, args.ust_commit
 224     )
 225
 226     if args.lttng_version == "master":
 227         lttng_version_string = "master"
 228     elif args.lttng_version == "canary":
 229         lttng_version_string = "2.10"
 230     else:
 231         major, minor = parse_stable_version(args.lttng_version)
 232         lttng_version_string = str(major) + "." + str(minor)
 233
 234
 235     context = dict()
 236     context['DeviceType'] = DeviceType
 237     context['TestType'] = TestType
 238
 239     context['job_name'] = args.jobname
 240     context['test_type'] = test_type
 241     context['random_seed'] = random.randint(0, 1000000)
 242     context['device_type'] = device_type
 243
 244     context['vlttng_cmd'] = vlttng_cmd
 245     context['vlttng_path'] = vlttng_path
 246     context['lttng_version_string'] = lttng_version_string
 247
 248     context['kernel_url'] = args.kernel
 249     context['nfsrootfs_url'] = nfsrootfs
 250     context['lttng_modules_url'] = args.lmodule
 251     context['jenkins_build_id'] = args.build_id
 252
 253     context['kprobe_round_nb'] = 10
 254
 255     render = jinja_template.render(context)
 256
 257     print('Job to be submitted:')
 258
 259     print(render)
 260
 261     if args.debug:
 262         return 0
 263
 264     server = xmlrpc.client.ServerProxy(
 265         'http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME)
 266     )
 267
 268     for attempt in range(10):
 269         try:
 270             jobid = server.scheduler.submit_job(render)
 271         except xmlrpc.client.ProtocolError as error:
 272             print(
 273                 'Protocol error on submit, sleeping and retrying. Attempt #{}'.format(
 274                     attempt
 275                 )
 276             )
 277             time.sleep(5)
 278             continue
 279         else:
 280             break
 281
 282     print('Lava jobid:{}'.format(jobid))
 283     print(
 284         'Lava job URL: http://lava-master-02.internal.efficios.com/scheduler/job/{}'.format(
 285             jobid
 286         )
 287     )
 288
 289     # Check the status of the job every 30 seconds
 290     jobstatus = server.scheduler.job_state(jobid)['job_state']
 291     running = False
 292     while jobstatus in ['Submitted', 'Scheduling', 'Scheduled', 'Running']:
 293         if not running and jobstatus == 'Running':
 294             print('Job started running')
 295             running = True
 296         time.sleep(30)
 297         try:
 298             jobstatus = server.scheduler.job_state(jobid)['job_state']
 299         except xmlrpc.client.ProtocolError as error:
 300             print('Protocol error, retrying')
 301             continue
 302     print('Job ended with {} status.'.format(jobstatus))
 303
 304     if jobstatus != 'Finished':
 305         return -1
 306
 307     if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests:
 308         print_test_output(server, jobid)
 309
 310     passed, failed = check_job_all_test_cases_state_count(server, jobid)
 311     print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
 312
 313     if failed != 0:
 314         return -1
 315
 316     return 0
 317
 318
 319 if __name__ == "__main__":
 320     sys.exit(main())