[lttng-ci.git] / scripts / system-tests / lava2-submit.py

#!/usr/bin/python
# Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import argparse
import base64
import json
import os
import random
import sys
import time
import yaml
import xmlrpc.client
import pprint

from jinja2 import Environment, FileSystemLoader, meta

USERNAME = 'lava-jenkins'
HOSTNAME = 'lava-master-02.internal.efficios.com'
OBJSTORE_URL = "https://obj.internal.efficios.com/lava/results/"

class TestType():
    baremetal_benchmarks=1
    baremetal_tests=2
    kvm_tests=3
    kvm_fuzzing_tests=4
    values = {
        'baremetal-benchmarks' : baremetal_benchmarks,
        'baremetal-tests' : baremetal_tests,
        'kvm-tests' : kvm_tests,
        'kvm-fuzzin-tests' : kvm_fuzzing_tests,
    }

class DeviceType():
    x86 = 'x86'
    kvm = 'qemu'
    values = {
        'kvm' : kvm,
        'x86' : x86,
    }

def get_job_bundle_content(server, job):
    try:
        bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
        bundle = server.dashboard.get(bundle_sha)
    except xmlrpc.client.Fault as f:
        print('Error while fetching results bundle', f.faultString)
        raise f

    return json.loads(bundle['content'])

# Parse the results bundle to see the run-tests testcase
# of the lttng-kernel-tests passed successfully
def check_job_all_test_cases_state_count(server, job):
    print("Testcase result:")
    content = server.results.get_testjob_results_yaml(str(job))
    testcases = yaml.load(content)

    passed_tests=0
    failed_tests=0
    for testcase in testcases:
        if testcase['result'] != 'pass':
            print("\tFAILED {}\n\t\t See http://{}{}".format(
                testcase['name'],
                HOSTNAME,
                testcase['url']
            ))
            failed_tests+=1
        else:
            passed_tests+=1
    return (passed_tests, failed_tests)

# Get the benchmark results from the objstore
# save them as CSV files localy
def fetch_benchmark_results(build_id):
    testcases = ['processed_results_close.csv',
            'processed_results_ioctl.csv',
            'processed_results_open_efault.csv',
            'processed_results_open_enoent.csv',
            'processed_results_dup_close.csv',
            'processed_results_raw_syscall_getpid.csv',
            'processed_results_lttng_test_filter.csv']
    for testcase in testcases:
        url = urljoin(OBJSTORE_URL, "{:s}/{:s}".format(build_id, testcase))
        urlretrieve(url, testcase)

# Parse the attachment of the testcase to fetch the stdout of the test suite
def print_test_output(server, job):
    job_finished, log = server.scheduler.jobs.logs(str(job))
    logs = yaml.load(log.data.decode('ascii'))
    print_line = False
    for line in logs:
        if line['lvl'] != 'target':
            continue
        if line['msg'] == '<LAVA_SIGNAL_STARTTC run-tests>':
            print('---- TEST SUITE OUTPUT BEGIN ----')
            print_line = True
            continue
        if line['msg'] == '<LAVA_SIGNAL_ENDTC run-tests>':
            print('----- TEST SUITE OUTPUT END -----')
            break
        if print_line:
            print("{} {}".format(line['dt'], line['msg']))

def get_vlttng_cmd(device, lttng_tools_commit, lttng_ust_commit=None):

    vlttng_cmd = 'vlttng --jobs=$(nproc) --profile urcu-master' \
                    ' --override projects.babeltrace.build-env.PYTHON=python3' \
                    ' --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config' \
                    ' --profile babeltrace-stable-1.4' \
                    ' --profile babeltrace-python' \
                    ' --profile lttng-tools-master' \
                    ' --override projects.lttng-tools.checkout='+lttng_tools_commit + \
                    ' --profile lttng-tools-no-man-pages' \
                    ' --verbose'

    if lttng_ust_commit is not None:
        vlttng_cmd += ' --profile lttng-ust-master ' \
                    ' --override projects.lttng-ust.checkout='+lttng_ust_commit+ \
                    ' --profile lttng-ust-no-man-pages'

    vlttng_path = '/tmp/virtenv'

    vlttng_cmd += ' ' + vlttng_path

    return vlttng_cmd

def main():
    nfsrootfs = "https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_xenial_2018-12-05.tar.gz"
    test_type = None
    parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
    parser.add_argument('-t', '--type', required=True)
    parser.add_argument('-j', '--jobname', required=True)
    parser.add_argument('-k', '--kernel', required=True)
    parser.add_argument('-lm', '--lmodule', required=True)
    parser.add_argument('-tc', '--tools-commit', required=True)
    parser.add_argument('-id', '--build-id', required=True)
    parser.add_argument('-uc', '--ust-commit', required=False)
    parser.add_argument('-d', '--debug', required=False, action='store_true')
    args = parser.parse_args()

    if args.type not in TestType.values:
        print('argument -t/--type {} unrecognized.'.format(args.type))
        print('Possible values are:')
        for k in TestType.values:
            print('\t {}'.format(k))
        return -1

    lava_api_key = None
    if not args.debug:
        try:
            lava_api_key = os.environ['LAVA2_JENKINS_TOKEN']
        except Exception as e:
            print('LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...', e )
            return -1

    jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__)))
    jinja_env = Environment(loader=jinja_loader, trim_blocks=True,
            lstrip_blocks= True)
    jinja_template = jinja_env.get_template('template_lava_job.jinja2')
    template_source = jinja_env.loader.get_source(jinja_env, 'template_lava_job.jinja2')
    parsed_content = jinja_env.parse(template_source)
    undef = meta.find_undeclared_variables(parsed_content)

    test_type = TestType.values[args.type]

    if test_type in [TestType.baremetal_benchmarks, TestType.baremetal_tests]:
        device_type = DeviceType.x86
    else:
        device_type = DeviceType.kvm

    vlttng_path = '/tmp/virtenv'

    vlttng_cmd = get_vlttng_cmd(device_type, args.tools_commit, args.ust_commit)

    context = dict()
    context['DeviceType'] = DeviceType
    context['TestType'] = TestType

    context['job_name'] = args.jobname
    context['test_type'] = test_type
    context['random_seed'] = random.randint(0, 1000000)
    context['device_type'] = device_type

    context['vlttng_cmd'] = vlttng_cmd
    context['vlttng_path'] = vlttng_path

    context['kernel_url'] = args.kernel
    context['nfsrootfs_url'] = nfsrootfs
    context['lttng_modules_url'] = args.lmodule
    context['jenkins_build_id'] = args.build_id

    context['kprobe_round_nb'] = 10

    render = jinja_template.render(context)

    print('Job to be submitted:')

    print(render)

    if args.debug:
        return 0

    server = xmlrpc.client.ServerProxy('http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME))

    jobid = server.scheduler.submit_job(render)

    print('Lava jobid:{}'.format(jobid))
    print('Lava job URL: http://lava-master-02.internal.efficios.com/scheduler/job/{}'.format(jobid))

    #Check the status of the job every 30 seconds
    jobstatus = server.scheduler.job_state(jobid)['job_state']
    running = False
    while jobstatus in ['Submitted','Scheduling','Scheduled','Running']:
        if not running and jobstatus == 'Running':
            print('Job started running')
            running = True
        time.sleep(30)
        jobstatus = server.scheduler.job_state(jobid)['job_state']
    print('Job ended with {} status.'.format(jobstatus))

    if jobstatus != 'Finished':
        return -1

    if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests:
        print_test_output(server, jobid)
    elif test_type is TestType.baremetal_benchmarks:
        fetch_benchmark_results(args.build_id)

    passed, failed=check_job_all_test_cases_state_count(server, jobid)
    print('With {} passed and {} failed Lava test cases.'.format(passed, failed))

    if failed == 0:
        return 0
    else:
        return -1

if __name__ == "__main__":
    sys.exit(main())
Commit	Line	Data
	1	#!/usr/bin/python
	2	# Copyright (C) 2016 - Francis Deslauriers <francis.deslauriers@efficios.com>
	3	#
	4	# This program is free software: you can redistribute it and/or modify
	5	# it under the terms of the GNU General Public License as published by
	6	# the Free Software Foundation, either version 3 of the License, or
	7	# (at your option) any later version.
	8	#
	9	# This program is distributed in the hope that it will be useful,
	10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	11	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	12	# GNU General Public License for more details.
	13	#
	14	# You should have received a copy of the GNU General Public License
	15	# along with this program. If not, see <http://www.gnu.org/licenses/>.
	16
	17	import argparse
	18	import base64
	19	import json
	20	import os
	21	import random
	22	import sys
	23	import time
	24	import yaml
	25	import xmlrpc.client
	26	import pprint
	27
	28	from jinja2 import Environment, FileSystemLoader, meta
	29
	30	USERNAME = 'lava-jenkins'
	31	HOSTNAME = 'lava-master-02.internal.efficios.com'
	32	OBJSTORE_URL = "https://obj.internal.efficios.com/lava/results/"
	33
	34	class TestType():
	35	baremetal_benchmarks=1
	36	baremetal_tests=2
	37	kvm_tests=3
	38	kvm_fuzzing_tests=4
	39	values = {
	40	'baremetal-benchmarks' : baremetal_benchmarks,
	41	'baremetal-tests' : baremetal_tests,
	42	'kvm-tests' : kvm_tests,
	43	'kvm-fuzzin-tests' : kvm_fuzzing_tests,
	44	}
	45
	46	class DeviceType():
	47	x86 = 'x86'
	48	kvm = 'qemu'
	49	values = {
	50	'kvm' : kvm,
	51	'x86' : x86,
	52	}
	53
	54	def get_job_bundle_content(server, job):
	55	try:
	56	bundle_sha = server.scheduler.job_status(str(job))['bundle_sha1']
	57	bundle = server.dashboard.get(bundle_sha)
	58	except xmlrpc.client.Fault as f:
	59	print('Error while fetching results bundle', f.faultString)
	60	raise f
	61
	62	return json.loads(bundle['content'])
	63
	64	# Parse the results bundle to see the run-tests testcase
	65	# of the lttng-kernel-tests passed successfully
	66	def check_job_all_test_cases_state_count(server, job):
	67	print("Testcase result:")
	68	content = server.results.get_testjob_results_yaml(str(job))
	69	testcases = yaml.load(content)
	70
	71	passed_tests=0
	72	failed_tests=0
	73	for testcase in testcases:
	74	if testcase['result'] != 'pass':
	75	print("\tFAILED {}\n\t\t See http://{}{}".format(
	76	testcase['name'],
	77	HOSTNAME,
	78	testcase['url']
	79	))
	80	failed_tests+=1
	81	else:
	82	passed_tests+=1
	83	return (passed_tests, failed_tests)
	84
	85	# Get the benchmark results from the objstore
	86	# save them as CSV files localy
	87	def fetch_benchmark_results(build_id):
	88	testcases = ['processed_results_close.csv',
	89	'processed_results_ioctl.csv',
	90	'processed_results_open_efault.csv',
	91	'processed_results_open_enoent.csv',
	92	'processed_results_dup_close.csv',
	93	'processed_results_raw_syscall_getpid.csv',
	94	'processed_results_lttng_test_filter.csv']
	95	for testcase in testcases:
	96	url = urljoin(OBJSTORE_URL, "{:s}/{:s}".format(build_id, testcase))
	97	urlretrieve(url, testcase)
	98
	99	# Parse the attachment of the testcase to fetch the stdout of the test suite
	100	def print_test_output(server, job):
	101	job_finished, log = server.scheduler.jobs.logs(str(job))
	102	logs = yaml.load(log.data.decode('ascii'))
	103	print_line = False
	104	for line in logs:
	105	if line['lvl'] != 'target':
	106	continue
	107	if line['msg'] == '<LAVA_SIGNAL_STARTTC run-tests>':
	108	print('---- TEST SUITE OUTPUT BEGIN ----')
	109	print_line = True
	110	continue
	111	if line['msg'] == '<LAVA_SIGNAL_ENDTC run-tests>':
	112	print('----- TEST SUITE OUTPUT END -----')
	113	break
	114	if print_line:
	115	print("{} {}".format(line['dt'], line['msg']))
	116
	117	def get_vlttng_cmd(device, lttng_tools_commit, lttng_ust_commit=None):
	118
	119	vlttng_cmd = 'vlttng --jobs=$(nproc) --profile urcu-master' \
	120	' --override projects.babeltrace.build-env.PYTHON=python3' \
	121	' --override projects.babeltrace.build-env.PYTHON_CONFIG=python3-config' \
	122	' --profile babeltrace-stable-1.4' \
	123	' --profile babeltrace-python' \
	124	' --profile lttng-tools-master' \
	125	' --override projects.lttng-tools.checkout='+lttng_tools_commit + \
	126	' --profile lttng-tools-no-man-pages' \
	127	' --verbose'
	128
	129	if lttng_ust_commit is not None:
	130	vlttng_cmd += ' --profile lttng-ust-master ' \
	131	' --override projects.lttng-ust.checkout='+lttng_ust_commit+ \
	132	' --profile lttng-ust-no-man-pages'
	133
	134	vlttng_path = '/tmp/virtenv'
	135
	136	vlttng_cmd += ' ' + vlttng_path
	137
	138	return vlttng_cmd
	139
	140	def main():
	141	nfsrootfs = "https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_xenial_2018-12-05.tar.gz"
	142	test_type = None
	143	parser = argparse.ArgumentParser(description='Launch baremetal test using Lava')
	144	parser.add_argument('-t', '--type', required=True)
	145	parser.add_argument('-j', '--jobname', required=True)
	146	parser.add_argument('-k', '--kernel', required=True)
	147	parser.add_argument('-lm', '--lmodule', required=True)
	148	parser.add_argument('-tc', '--tools-commit', required=True)
	149	parser.add_argument('-id', '--build-id', required=True)
	150	parser.add_argument('-uc', '--ust-commit', required=False)
	151	parser.add_argument('-d', '--debug', required=False, action='store_true')
	152	args = parser.parse_args()
	153
	154	if args.type not in TestType.values:
	155	print('argument -t/--type {} unrecognized.'.format(args.type))
	156	print('Possible values are:')
	157	for k in TestType.values:
	158	print('\t {}'.format(k))
	159	return -1
	160
	161	lava_api_key = None
	162	if not args.debug:
	163	try:
	164	lava_api_key = os.environ['LAVA2_JENKINS_TOKEN']
	165	except Exception as e:
	166	print('LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...', e )
	167	return -1
	168
	169	jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__)))
	170	jinja_env = Environment(loader=jinja_loader, trim_blocks=True,
	171	lstrip_blocks= True)
	172	jinja_template = jinja_env.get_template('template_lava_job.jinja2')
	173	template_source = jinja_env.loader.get_source(jinja_env, 'template_lava_job.jinja2')
	174	parsed_content = jinja_env.parse(template_source)
	175	undef = meta.find_undeclared_variables(parsed_content)
	176
	177	test_type = TestType.values[args.type]
	178
	179	if test_type in [TestType.baremetal_benchmarks, TestType.baremetal_tests]:
	180	device_type = DeviceType.x86
	181	else:
	182	device_type = DeviceType.kvm
	183
	184	vlttng_path = '/tmp/virtenv'
	185
	186	vlttng_cmd = get_vlttng_cmd(device_type, args.tools_commit, args.ust_commit)
	187
	188	context = dict()
	189	context['DeviceType'] = DeviceType
	190	context['TestType'] = TestType
	191
	192	context['job_name'] = args.jobname
	193	context['test_type'] = test_type
	194	context['random_seed'] = random.randint(0, 1000000)
	195	context['device_type'] = device_type
	196
	197	context['vlttng_cmd'] = vlttng_cmd
	198	context['vlttng_path'] = vlttng_path
	199
	200	context['kernel_url'] = args.kernel
	201	context['nfsrootfs_url'] = nfsrootfs
	202	context['lttng_modules_url'] = args.lmodule
	203	context['jenkins_build_id'] = args.build_id
	204
	205	context['kprobe_round_nb'] = 10
	206
	207	render = jinja_template.render(context)
	208
	209	print('Job to be submitted:')
	210
	211	print(render)
	212
	213	if args.debug:
	214	return 0
	215
	216	server = xmlrpc.client.ServerProxy('http://%s:%s@%s/RPC2' % (USERNAME, lava_api_key, HOSTNAME))
	217
	218	jobid = server.scheduler.submit_job(render)
	219
	220	print('Lava jobid:{}'.format(jobid))
	221	print('Lava job URL: http://lava-master-02.internal.efficios.com/scheduler/job/{}'.format(jobid))
	222
	223	#Check the status of the job every 30 seconds
	224	jobstatus = server.scheduler.job_state(jobid)['job_state']
	225	running = False
	226	while jobstatus in ['Submitted','Scheduling','Scheduled','Running']:
	227	if not running and jobstatus == 'Running':
	228	print('Job started running')
	229	running = True
	230	time.sleep(30)
	231	jobstatus = server.scheduler.job_state(jobid)['job_state']
	232	print('Job ended with {} status.'.format(jobstatus))
	233
	234	if jobstatus != 'Finished':
	235	return -1
	236
	237	if test_type is TestType.kvm_tests or test_type is TestType.baremetal_tests:
	238	print_test_output(server, jobid)
	239	elif test_type is TestType.baremetal_benchmarks:
	240	fetch_benchmark_results(args.build_id)
	241
	242	passed, failed=check_job_all_test_cases_state_count(server, jobid)
	243	print('With {} passed and {} failed Lava test cases.'.format(passed, failed))
	244
	245	if failed == 0:
	246	return 0
	247	else:
	248	return -1
	249
	250	if __name__ == "__main__":
	251	sys.exit(main())