[lttng-ci.git] / scripts / system-tests / lava2-submit.py

#!/usr/bin/python
# Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import argparse
import base64
import json
import os
import random
import sys
import time
import yaml
import xmlrpc.client
import pprint

from jinja2 import Environment, FileSystemLoader, meta

USERNAME = 'lava-jenkins'
HOSTNAME = 'lava-master-02.internal.efficios.com'
OBJSTORE_URL = "https://obj.internal.efficios.com/lava/results/"

class TestType():
    baremetal_benchmarks=1
    baremetal_tests=2
    kvm_tests=3
    kvm_fuzzing_tests=4
    values = {
        'baremetal-benchmarks' : baremetal_benchmarks,
        'baremetal-tests' : baremetal_tests,
        'kvm-tests' : kvm_tests,
        'kvm-fuzzin-tests' : kvm_fuzzing_tests,
    }

class DeviceType():
    x86 = 'x86'
    kvm = 'qemu'
    values = {
        'kvm' : kvm,
        'x86' : x86,
    }

def get_job_bundle_content(server, job):
    try:
        bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
        bundle = server.dashboard.get(bundle_sha)
    except xmlrpc.client.Fault as f:
        print('Error while fetching results bundle', f.faultString)
        raise f

    return json.loads(bundle['content'])

# Parse the results bundle to see the run-tests testcase
# of the lttng-kernel-tests passed successfully
def check_job_all_test_cases_state_count(server, job):
    print("Testcase result:")
    content = server.results.get_testjob_results_yaml(str(job))
    testcases = yaml.load(content)

    passed_tests=0
    failed_tests=0
    for testcase in testcases:
        if testcase['result'] != 'pass':
            print("\tFAILED {}\n\t\t See http://{}{}".format(
                testcase['name'],
                HOSTNAME,
                testcase['url']
            ))
            failed_tests+=1
        else:
            passed_tests+=1
    return (passed_tests, failed_tests)

# Get the benchmark results from the objstore
# save them as CSV files localy
def fetch_benchmark_results(build_id):
    testcases = ['processed_results_close.csv',
            'processed_results_ioctl.csv',
            'processed_results_open_efault.csv',
            'processed_results_open_enoent.csv',
            'processed_results_dup_close.csv',
            'processed_results_raw_syscall_getpid.csv',
            'processed_results_lttng_test_filter.csv']
    for testcase in testcases:
        url = urljoin(OBJSTORE_URL, "{:s}/{:s}".format(build_id, testcase))
        urlretrieve(url, testcase)

# Parse the attachment of the testcase to fetch the stdout of the test suite
def print_test_output(server, job):
    job_finished, log = server.scheduler.jobs.logs(str(job))
    logs = yaml.load(log.data.decode('ascii'))
    print_line = False
    for line in logs:
        if line['lvl'] != 'target':
            continue
        if line['msg'] == '<LAVA_SIGNAL_STARTTC run-tests>':
            print('---- TEST SUITE OUTPUT BEGIN ----')
            print_line = True
            continue
        if line['msg'] == '<LAVA_SIGNAL_ENDTC run-tests>':
            print('----- TEST SUITE OUTPUT END -----')
            break
        if print_line:
            print("{} {}".format(line['dt'], line['msg']))

def get_vlttng_cmd(device, lttng_tools_commit, lttng_ust_commit=None):

    vlttng_cmd = 'vlttng --jobs=$(nproc) --profile urcu-master' \
                    ' --override projects.babeltrace.build-env.PYTHON=python3' \
                    ' --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config' \
                    ' --profile babeltrace-stable-1.4' \
                    ' --profile babeltrace-python' \
                    ' --profile lttng-tools-master' \
                    ' --override projects.lttng-tools.checkout='+lttng_tools_commit + \
                    ' --profile lttng-tools-no-man-pages'

    if lttng_ust_commit is not None:
        vlttng_cmd += ' --profile lttng-ust-master ' \
                    ' --override projects.lttng-ust.checkout='+lttng_ust_commit+ \
                    ' --profile lttng-ust-no-man-pages'

    vlttng_path = '/tmp/virtenv'

    vlttng_cmd += ' ' + vlttng_path

    return vlttng_cmd

def main():
    nfsrootfs = "https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_xenial_2018-12-05.tar.gz"
    test_type = None
    parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
    parser.add_argument('-t', '--type', required=True)
    parser.add_argument('-j', '--jobname', required=True)
    parser.add_argument('-k', '--kernel', required=True)
    parser.add_argument('-lm', '--lmodule', required=True)
    parser.add_argument('-tc', '--tools-commit', required=True)
    parser.add_argument('-id', '--build-id', required=True)
    parser.add_argument('-uc', '--ust-commit', required=False)
    parser.add_argument('-d', '--debug', required=False, action='store_true')
    args = parser.parse_args()

    if args.type not in TestType.values:
        print('argument -t/--type {} unrecognized.'.format(args.type))
        print('Possible values are:')
        for k in TestType.values:
            print('\t {}'.format(k))
        return -1

    lava_api_key = None
    if not args.debug:
        try:
            lava_api_key = os.environ['LAVA2_JENKINS_TOKEN']
        except Exception as e:
            print('LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...', e )
            return -1

    jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__)))
    jinja_env = Environment(loader=jinja_loader, trim_blocks=True,
            lstrip_blocks= True)
    jinja_template = jinja_env.get_template('template_lava_job.jinja2')
    template_source = jinja_env.loader.get_source(jinja_env, 'template_lava_job.jinja2')
    parsed_content = jinja_env.parse(template_source)
    undef = meta.find_undeclared_variables(parsed_content)

    test_type = TestType.values[args.type]

    if test_type in [TestType.baremetal_benchmarks, TestType.baremetal_tests]:
        device_type = DeviceType.x86
    else:
        device_type = DeviceType.kvm

    vlttng_path = '/tmp/virtenv'

    vlttng_cmd = get_vlttng_cmd(device_type, args.tools_commit, args.ust_commit)

    context = dict()
    context['DeviceType'] = DeviceType
    context['TestType'] = TestType

    context['job_name'] = args.jobname
    context['test_type'] = test_type
    context['random_seed'] = random.randint(0, 1000000)
    context['device_type'] = device_type

    context['vlttng_cmd'] = vlttng_cmd
    context['vlttng_path'] = vlttng_path

    context['kernel_url'] = args.kernel
    context['nfsrootfs_url'] = nfsrootfs
    context['lttng_modules_url'] = args.lmodule
    context['jenkins_build_id'] = args.build_id

    context['kprobe_round_nb'] = 10

    render = jinja_template.render(context)

    print('Job to be submitted:')

    print(render)

    if args.debug:
        return 0

    server = xmlrpc.client.ServerProxy('http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME))

    jobid = server.scheduler.submit_job(render)

    print('Lava jobid:{}'.format(jobid))
    print('Lava job URL: http://lava-master-02.internal.efficios.com/scheduler/job/{}'.format(jobid))

    #Check the status of the job every 30 seconds
    jobstatus = server.scheduler.job_state(jobid)['job_state']
    running = False
    while jobstatus in ['Submitted','Scheduling','Scheduled','Running']:
        if not running and jobstatus == 'Running':
            print('Job started running')
            running = True
        time.sleep(30)
        jobstatus = server.scheduler.job_state(jobid)['job_state']
    print('Job ended with {} status.'.format(jobstatus))

    if jobstatus != 'Finished':
        return -1

    if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests:
        print_test_output(server, jobid)
    elif test_type is TestType.baremetal_benchmarks:
        fetch_benchmark_results(args.build_id)

    passed, failed=check_job_all_test_cases_state_count(server, jobid)
    print('With {} passed and {} failed Lava test cases.'.format(passed, failed))

    if failed == 0:
        return 0
    else:
        return -1

if __name__ == "__main__":
    sys.exit(main())
Commit	Line	Data
	1	#!/usr/bin/python
	2	# Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
	3	#
	4	# This program is free software: you can redistribute it and/or modify
	5	# it under the terms of the GNU General Public License as published by
	6	# the Free Software Foundation, either version 3 of the License, or
	7	# (at your option) any later version.
	8	#
	9	# This program is distributed in the hope that it will be useful,
	10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	# GNU General Public License for more details.
	13	#
	14	# You should have received a copy of the GNU General Public License
	15	# along with this program. If not, see <http://www.gnu.org/licenses/>.
	16
	17	import argparse
	18	import base64
	19	import json
	20	import os
	21	import random
	22	import sys
	23	import time
	24	import yaml
	25	import xmlrpc.client
	26	import pprint
	27
	28	from jinja2 import Environment, FileSystemLoader, meta
	29
	30	USERNAME = 'lava-jenkins'
	31	HOSTNAME = 'lava-master-02.internal.efficios.com'
	32	OBJSTORE_URL = "https://obj.internal.efficios.com/lava/results/"
	33
	34	class TestType():
	35	baremetal_benchmarks=1
	36	baremetal_tests=2
	37	kvm_tests=3
	38	kvm_fuzzing_tests=4
	39	values = {
	40	'baremetal-benchmarks' : baremetal_benchmarks,
	41	'baremetal-tests' : baremetal_tests,
	42	'kvm-tests' : kvm_tests,
	43	'kvm-fuzzin-tests' : kvm_fuzzing_tests,
	44	}
	45
	46	class DeviceType():
	47	x86 = 'x86'
	48	kvm = 'qemu'
	49	values = {
	50	'kvm' : kvm,
	51	'x86' : x86,
	52	}
	53
	54	def get_job_bundle_content(server, job):
	55	try:
	56	bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
	57	bundle = server.dashboard.get(bundle_sha)
	58	except xmlrpc.client.Fault as f:
	59	print('Error while fetching results bundle', f.faultString)
	60	raise f
	61
	62	return json.loads(bundle['content'])
	63
	64	# Parse the results bundle to see the run-tests testcase
	65	# of the lttng-kernel-tests passed successfully
	66	def check_job_all_test_cases_state_count(server, job):
	67	print("Testcase result:")
	68	content = server.results.get_testjob_results_yaml(str(job))
	69	testcases = yaml.load(content)
	70
	71	passed_tests=0
	72	failed_tests=0
	73	for testcase in testcases:
	74	if testcase['result'] != 'pass':
	75	print("\tFAILED {}\n\t\t See http://{}{}".format(
	76	testcase['name'],
	77	HOSTNAME,
	78	testcase['url']
	79	))
	80	failed_tests+=1
	81	else:
	82	passed_tests+=1
	83	return (passed_tests, failed_tests)
	84
	85	# Get the benchmark results from the objstore
	86	# save them as CSV files localy
	87	def fetch_benchmark_results(build_id):
	88	testcases = ['processed_results_close.csv',
	89	'processed_results_ioctl.csv',
	90	'processed_results_open_efault.csv',
	91	'processed_results_open_enoent.csv',
	92	'processed_results_dup_close.csv',
	93	'processed_results_raw_syscall_getpid.csv',
	94	'processed_results_lttng_test_filter.csv']
	95	for testcase in testcases:
	96	url = urljoin(OBJSTORE_URL, "{:s}/{:s}".format(build_id, testcase))
	97	urlretrieve(url, testcase)
	98
	99	# Parse the attachment of the testcase to fetch the stdout of the test suite
	100	def print_test_output(server, job):
	101	job_finished, log = server.scheduler.jobs.logs(str(job))
	102	logs = yaml.load(log.data.decode('ascii'))
	103	print_line = False
	104	for line in logs:
	105	if line['lvl'] != 'target':
	106	continue
	107	if line['msg'] == '<LAVA_SIGNAL_STARTTC run-tests>':
	108	print('---- TEST SUITE OUTPUT BEGIN ----')
	109	print_line = True
	110	continue
	111	if line['msg'] == '<LAVA_SIGNAL_ENDTC run-tests>':
	112	print('----- TEST SUITE OUTPUT END -----')
	113	break
	114	if print_line:
	115	print("{} {}".format(line['dt'], line['msg']))
	116
	117	def get_vlttng_cmd(device, lttng_tools_commit, lttng_ust_commit=None):
	118
	119	vlttng_cmd = 'vlttng --jobs=$(nproc) --profile urcu-master' \
	120	' --override projects.babeltrace.build-env.PYTHON=python3' \
	121	' --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config' \
	122	' --profile babeltrace-stable-1.4' \
	123	' --profile babeltrace-python' \
	124	' --profile lttng-tools-master' \
	125	' --override projects.lttng-tools.checkout='+lttng_tools_commit + \
	126	' --profile lttng-tools-no-man-pages'
	127
	128	if lttng_ust_commit is not None:
	129	vlttng_cmd += ' --profile lttng-ust-master ' \
	130	' --override projects.lttng-ust.checkout='+lttng_ust_commit+ \
	131	' --profile lttng-ust-no-man-pages'
	132
	133	vlttng_path = '/tmp/virtenv'
	134
	135	vlttng_cmd += ' ' + vlttng_path
	136
	137	return vlttng_cmd
	138
	139	def main():
	140	nfsrootfs = "https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_xenial_2018-12-05.tar.gz"
	141	test_type = None
	142	parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
	143	parser.add_argument('-t', '--type', required=True)
	144	parser.add_argument('-j', '--jobname', required=True)
	145	parser.add_argument('-k', '--kernel', required=True)
	146	parser.add_argument('-lm', '--lmodule', required=True)
	147	parser.add_argument('-tc', '--tools-commit', required=True)
	148	parser.add_argument('-id', '--build-id', required=True)
	149	parser.add_argument('-uc', '--ust-commit', required=False)
	150	parser.add_argument('-d', '--debug', required=False, action='store_true')
	151	args = parser.parse_args()
	152
	153	if args.type not in TestType.values:
	154	print('argument -t/--type {} unrecognized.'.format(args.type))
	155	print('Possible values are:')
	156	for k in TestType.values:
	157	print('\t {}'.format(k))
	158	return -1
	159
	160	lava_api_key = None
	161	if not args.debug:
	162	try:
	163	lava_api_key = os.environ['LAVA2_JENKINS_TOKEN']
	164	except Exception as e:
	165	print('LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...', e )
	166	return -1
	167
	168	jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__)))
	169	jinja_env = Environment(loader=jinja_loader, trim_blocks=True,
	170	lstrip_blocks= True)
	171	jinja_template = jinja_env.get_template('template_lava_job.jinja2')
	172	template_source = jinja_env.loader.get_source(jinja_env, 'template_lava_job.jinja2')
	173	parsed_content = jinja_env.parse(template_source)
	174	undef = meta.find_undeclared_variables(parsed_content)
	175
	176	test_type = TestType.values[args.type]
	177
	178	if test_type in [TestType.baremetal_benchmarks, TestType.baremetal_tests]:
	179	device_type = DeviceType.x86
	180	else:
	181	device_type = DeviceType.kvm
	182
	183	vlttng_path = '/tmp/virtenv'
	184
	185	vlttng_cmd = get_vlttng_cmd(device_type, args.tools_commit, args.ust_commit)
	186
	187	context = dict()
	188	context['DeviceType'] = DeviceType
	189	context['TestType'] = TestType
	190
	191	context['job_name'] = args.jobname
	192	context['test_type'] = test_type
	193	context['random_seed'] = random.randint(0, 1000000)
	194	context['device_type'] = device_type
	195
	196	context['vlttng_cmd'] = vlttng_cmd
	197	context['vlttng_path'] = vlttng_path
	198
	199	context['kernel_url'] = args.kernel
	200	context['nfsrootfs_url'] = nfsrootfs
	201	context['lttng_modules_url'] = args.lmodule
	202	context['jenkins_build_id'] = args.build_id
	203
	204	context['kprobe_round_nb'] = 10
	205
	206	render = jinja_template.render(context)
	207
	208	print('Job to be submitted:')
	209
	210	print(render)
	211
	212	if args.debug:
	213	return 0
	214
	215	server = xmlrpc.client.ServerProxy('http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME))
	216
	217	jobid = server.scheduler.submit_job(render)
	218
	219	print('Lava jobid:{}'.format(jobid))
	220	print('Lava job URL: http://lava-master-02.internal.efficios.com/scheduler/job/{}'.format(jobid))
	221
	222	#Check the status of the job every 30 seconds
	223	jobstatus = server.scheduler.job_state(jobid)['job_state']
	224	running = False
	225	while jobstatus in ['Submitted','Scheduling','Scheduled','Running']:
	226	if not running and jobstatus == 'Running':
	227	print('Job started running')
	228	running = True
	229	time.sleep(30)
	230	jobstatus = server.scheduler.job_state(jobid)['job_state']
	231	print('Job ended with {} status.'.format(jobstatus))
	232
	233	if jobstatus != 'Finished':
	234	return -1
	235
	236	if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests:
	237	print_test_output(server, jobid)
	238	elif test_type is TestType.baremetal_benchmarks:
	239	fetch_benchmark_results(args.build_id)
	240
	241	passed, failed=check_job_all_test_cases_state_count(server, jobid)
	242	print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
	243
	244	if failed == 0:
	245	return 0
	246	else:
	247	return -1
	248
	249	if __name__ == "__main__":
	250	sys.exit(main())