scripts/system-tests/lava2-submit.py

   1 #!/usr/bin/python3
   2 # Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import argparse
  18 import json
  19 import os
  20 import random
  21 import re
  22 import sys
  23 import time
  24 import xmlrpc.client
  25 from urllib.parse import urljoin
  26 from urllib.request import urlretrieve
  27 import yaml
  28 from jinja2 import Environment, FileSystemLoader
  29
  30 USERNAME = 'lava-jenkins'
  31 HOSTNAME = 'lava-master-02.internal.efficios.com'
  32 OBJSTORE_URL = "https://obj.internal.efficios.com/lava/results/"
  33
  34 def parse_stable_version(stable_version_string):
  35     # Get the major and minor version numbers from the lttng version string.
  36     version_match = re.search('stable-(\d).(\d\d)', stable_version_string)
  37
  38     if version_match is not None:
  39         major_version = int(version_match.group(1))
  40         minor_version = int(version_match.group(2))
  41     else:
  42         # Setting to zero to make the comparison below easier.
  43         major_version = 0
  44         minor_version = 0
  45     return major_version, minor_version
  46
  47
  48 class TestType:
  49     """ Enum like for test type """
  50
  51     baremetal_tests = 1
  52     kvm_tests = 2
  53     values = {
  54         'baremetal-tests': baremetal_tests,
  55         'kvm-tests': kvm_tests,
  56     }
  57
  58
  59 class DeviceType:
  60     """ Enum like for device type """
  61
  62     x86 = 'x86'
  63     kvm = 'qemu'
  64     values = {'kvm': kvm, 'x86': x86}
  65
  66
  67 def get_job_bundle_content(server, job):
  68     try:
  69         bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
  70         bundle = server.dashboard.get(bundle_sha)
  71     except xmlrpc.client.Fault as error:
  72         print('Error while fetching results bundle', error.faultString)
  73         raise error
  74
  75     return json.loads(bundle['content'])
  76
  77
  78 def check_job_all_test_cases_state_count(server, job):
  79     """
  80     Parse the results bundle to see the run-tests testcase
  81     of the lttng-kernel-tests passed successfully
  82     """
  83     print("Testcase result:")
  84     content = server.results.get_testjob_results_yaml(str(job))
  85     testcases = yaml.unsafe_load(content)
  86
  87     passed_tests = 0
  88     failed_tests = 0
  89     for testcase in testcases:
  90         if testcase['result'] != 'pass':
  91             print(
  92                 "\tFAILED {}\n\t\t See http://{}{}".format(
  93                     testcase['name'], HOSTNAME, testcase['url']
  94                 )
  95             )
  96             failed_tests += 1
  97         else:
  98             passed_tests += 1
  99     return (passed_tests, failed_tests)
 100
 101
 102 def print_test_output(server, job):
 103     """
 104     Parse the attachment of the testcase to fetch the stdout of the test suite
 105     """
 106     job_finished, log = server.scheduler.jobs.logs(str(job))
 107     logs = yaml.unsafe_load(log.data.decode('ascii'))
 108     print_line = False
 109     for line in logs:
 110         if line['lvl'] != 'target':
 111             continue
 112         if line['msg'] == '<LAVA_SIGNAL_STARTTC run-tests>':
 113             print('---- TEST SUITE OUTPUT BEGIN ----')
 114             print_line = True
 115             continue
 116         if line['msg'] == '<LAVA_SIGNAL_ENDTC run-tests>':
 117             print('----- TEST SUITE OUTPUT END -----')
 118             print_line = False
 119             continue
 120         if print_line:
 121             print("{} {}".format(line['dt'], line['msg']))
 122
 123
 124 def get_vlttng_cmd(
 125     lttng_version, lttng_tools_url, lttng_tools_commit, lttng_ust_url=None, lttng_ust_commit=None
 126 ):
 127     """
 128     Return vlttng cmd to be used in the job template for setup.
 129     """
 130
 131     major_version, minor_version = parse_stable_version(lttng_version)
 132
 133     urcu_profile = ""
 134     if lttng_version == 'master' or (major_version >= 2 and minor_version >= 11):
 135         urcu_profile = "urcu-master"
 136     else:
 137         urcu_profile = "urcu-stable-0.12"
 138
 139     # Starting with 2.14, babeltrace2 is the reader for testing.
 140     if lttng_version == 'master' or (major_version >= 2 and minor_version >= 14):
 141         babeltrace_profile = " --profile babeltrace2-stable-2.0 --profile babeltrace2-python"
 142     else:
 143         babeltrace_profile = " --profile babeltrace-stable-1.5 --profile babeltrace-python"
 144
 145     vlttng_cmd = (
 146         'vlttng --jobs=$(nproc) --profile ' + urcu_profile
 147         + ' --override projects.babeltrace.build-env.PYTHON=python3'
 148         ' --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config'
 149         + babeltrace_profile
 150         + ' --profile lttng-tools-master'
 151         ' --override projects.lttng-tools.source='
 152         + lttng_tools_url
 153         + ' --override projects.lttng-tools.checkout='
 154         + lttng_tools_commit
 155         + ' --profile lttng-tools-no-man-pages'
 156     )
 157
 158     if lttng_ust_commit is not None:
 159         vlttng_cmd += (
 160             ' --profile lttng-ust-master '
 161             ' --override projects.lttng-ust.source='
 162             + lttng_ust_url
 163             + ' --override projects.lttng-ust.checkout='
 164             + lttng_ust_commit
 165             + ' --profile lttng-ust-no-man-pages'
 166         )
 167
 168
 169     if lttng_version == 'master' or (major_version >= 2 and minor_version >= 11):
 170         vlttng_cmd += (
 171             ' --override projects.lttng-tools.configure+=--enable-test-sdt-uprobe'
 172         )
 173
 174     vlttng_path = '/tmp/virtenv'
 175
 176     vlttng_cmd += ' ' + vlttng_path
 177
 178     return vlttng_cmd
 179
 180
 181 def main():
 182     send_retry_limit = 10
 183     nfsrootfs = "https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_xenial_2018-12-05.tar.gz"
 184     test_type = None
 185     parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
 186     parser.add_argument('-t', '--type', required=True)
 187     parser.add_argument('-lv', '--lttng-version', required=True)
 188     parser.add_argument('-j', '--jobname', required=True)
 189     parser.add_argument('-k', '--kernel', required=True)
 190     parser.add_argument('-lm', '--lmodule', required=True)
 191     parser.add_argument('-tu', '--tools-url', required=True)
 192     parser.add_argument('-tc', '--tools-commit', required=True)
 193     parser.add_argument('-id', '--build-id', required=True)
 194     parser.add_argument('-uu', '--ust-url', required=False)
 195     parser.add_argument('-uc', '--ust-commit', required=False)
 196     parser.add_argument('-d', '--debug', required=False, action='store_true')
 197     args = parser.parse_args()
 198
 199     if args.type not in TestType.values:
 200         print('argument -t/--type {} unrecognized.'.format(args.type))
 201         print('Possible values are:')
 202         for k in TestType.values:
 203             print('\t {}'.format(k))
 204         return -1
 205
 206     lava_api_key = None
 207     if not args.debug:
 208         try:
 209             lava_api_key = os.environ['LAVA2_JENKINS_TOKEN']
 210         except Exception as error:
 211             print(
 212                 'LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...',
 213                 error,
 214             )
 215             return -1
 216
 217     jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__)))
 218     jinja_env = Environment(loader=jinja_loader, trim_blocks=True, lstrip_blocks=True)
 219     jinja_template = jinja_env.get_template('template_lava_job.jinja2')
 220
 221     test_type = TestType.values[args.type]
 222
 223     if test_type is TestType.baremetal_tests:
 224         device_type = DeviceType.x86
 225     else:
 226         device_type = DeviceType.kvm
 227
 228     vlttng_path = '/tmp/virtenv'
 229
 230     vlttng_cmd = get_vlttng_cmd(
 231         args.lttng_version, args.tools_url, args.tools_commit, args.ust_url, args.ust_commit
 232     )
 233
 234     if args.lttng_version == "master":
 235         lttng_version_string = "master"
 236     elif args.lttng_version == "canary":
 237         lttng_version_string = "2.10"
 238     else:
 239         major, minor = parse_stable_version(args.lttng_version)
 240         lttng_version_string = str(major) + "." + str(minor)
 241
 242
 243     context = dict()
 244     context['DeviceType'] = DeviceType
 245     context['TestType'] = TestType
 246
 247     context['job_name'] = args.jobname
 248     context['test_type'] = test_type
 249     context['random_seed'] = random.randint(0, 1000000)
 250     context['device_type'] = device_type
 251
 252     context['vlttng_cmd'] = vlttng_cmd
 253     context['vlttng_path'] = vlttng_path
 254     context['lttng_version_string'] = lttng_version_string
 255
 256     context['kernel_url'] = args.kernel
 257     context['nfsrootfs_url'] = nfsrootfs
 258     context['lttng_modules_url'] = args.lmodule
 259     context['jenkins_build_id'] = args.build_id
 260
 261     context['kprobe_round_nb'] = 10
 262
 263     render = jinja_template.render(context)
 264
 265     print('Job to be submitted:')
 266
 267     print(render)
 268
 269     if args.debug:
 270         return 0
 271
 272     server = xmlrpc.client.ServerProxy(
 273         'http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME)
 274     )
 275
 276     for attempt in range(1, send_retry_limit + 1):
 277         try:
 278             jobid = server.scheduler.submit_job(render)
 279         except xmlrpc.client.ProtocolError as error:
 280             print(
 281                 'Protocol error on submit, sleeping and retrying. Attempt #{}'.format(
 282                     attempt
 283                 )
 284             )
 285             time.sleep(5)
 286             continue
 287         else:
 288             break
 289     # Early exit when the maximum number of retry is reached.
 290     if attempt == send_retry_limit:
 291             print(
 292                 'Protocol error on submit, maximum number of retry reached ({})'.format(
 293                     attempt
 294                 )
 295             )
 296             return -1
 297
 298     print('Lava jobid:{}'.format(jobid))
 299     print(
 300         'Lava job URL: http://lava-master-02.internal.efficios.com/scheduler/job/{}'.format(
 301             jobid
 302         )
 303     )
 304
 305     # Check the status of the job every 30 seconds
 306     jobstatus = server.scheduler.job_state(jobid)['job_state']
 307     running = False
 308     while jobstatus in ['Submitted', 'Scheduling', 'Scheduled', 'Running']:
 309         if not running and jobstatus == 'Running':
 310             print('Job started running')
 311             running = True
 312         time.sleep(30)
 313         try:
 314             jobstatus = server.scheduler.job_state(jobid)['job_state']
 315         except xmlrpc.client.ProtocolError as error:
 316             print('Protocol error, retrying')
 317             continue
 318     print('Job ended with {} status.'.format(jobstatus))
 319
 320     if jobstatus != 'Finished':
 321         return -1
 322
 323     if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests:
 324         print_test_output(server, jobid)
 325
 326     passed, failed = check_job_all_test_cases_state_count(server, jobid)
 327     print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
 328
 329     if failed != 0:
 330         return -1
 331
 332     return 0
 333
 334
 335 if __name__ == "__main__":
 336     sys.exit(main())