From 5c65bbc2ea87bdcecc0c02cc7c19c50c94d6d6cb Mon Sep 17 00:00:00 2001 From: Jonathan Rajotte Date: Wed, 15 May 2019 16:20:41 -0400 Subject: [PATCH] Introduce babeltrace benchmark pipeline This benchmark pipeline leverage jenkins, lava and obj.internal.efficios.com. The babeltrace_benchmark (jobs/babeltrace-benchmark.yaml) jobs is responsible for queueing lava jobs ( template_lava_job_bt_benchmark.jinja2 and lava/benchmark/babeltrace/benchmark.yml ) and generating the benchmark report graph. The results for each individual commit lava jobs are uploaded to obj.internal.efficios.com in the form of a json file. Signed-off-by: Jonathan Rajotte --- jobs/babeltrace-benchmark.yaml | 62 +++ lava/benchmark/babeltrace/benchmark.yml | 37 ++ scripts/babeltrace-benchmark/benchmark.py | 415 ++++++++++++++++++ scripts/babeltrace-benchmark/benchmark.sh | 55 +++ scripts/babeltrace-benchmark/lava_submit.py | 137 ++++++ scripts/babeltrace-benchmark/requirement.txt | 33 ++ .../template_lava_job_bt_benchmark.jinja2 | 84 ++++ scripts/babeltrace-benchmark/time.py | 185 ++++++++ 8 files changed, 1008 insertions(+) create mode 100644 jobs/babeltrace-benchmark.yaml create mode 100644 lava/benchmark/babeltrace/benchmark.yml create mode 100644 scripts/babeltrace-benchmark/benchmark.py create mode 100755 scripts/babeltrace-benchmark/benchmark.sh create mode 100644 scripts/babeltrace-benchmark/lava_submit.py create mode 100644 scripts/babeltrace-benchmark/requirement.txt create mode 100644 scripts/babeltrace-benchmark/template_lava_job_bt_benchmark.jinja2 create mode 100644 scripts/babeltrace-benchmark/time.py diff --git a/jobs/babeltrace-benchmark.yaml b/jobs/babeltrace-benchmark.yaml new file mode 100644 index 0000000..a8c90e6 --- /dev/null +++ b/jobs/babeltrace-benchmark.yaml @@ -0,0 +1,62 @@ +- job: + name: babeltrace-benchmark + project-type: freestyle + defaults: global + description: | + This job is responsible for triggering jobs in lava and generate the report + + The Babeltrace project provides trace read and write libraries, as well + as a trace converter. Plugins can be created for any trace format to + allow its conversion to/from another trace format. + + The main format expected to be converted to/from is the Common Trace + Format (CTF). The default input format of the "babeltrace" command is + CTF, and its default output format is a human-readable text log. The + "babeltrace-log" command converts from a text log to a CTF trace. + +

Job is managed by Jenkins Job Builder.

+ + wrappers: + - ansicolor + - timestamps + - workspace-cleanup + - credentials-binding: + - text: + credential-id: jenkins_lava2_key + variable: LAVA2_JENKINS_TOKEN + + triggers: + - timed: '@daily' + builders: + - shell: + !include-raw: scripts/babeltrace-benchmark/benchmark.sh + + publishers: + - archive: + artifacts: 'results/**' + allow-empty: false + scm: + - babeltrace + - lttng-ci + +- scm: + name: babeltrace + scm: + - git: + url: git://github.com/efficios/babeltrace.git + browser: githubweb + browser-url: https://github.com/efficios/babeltrace + basedir: src/babeltrace + skip-tag: true + git-tool: jgit +- scm: + name: lttng-ci + scm: + - git: + url: git://github.com/lttng/lttng-ci.git + browser: githubweb + browser-url: https://github.com/lttng/lttng-ci + basedir: src/lttng-ci + skip-tag: true + git-tool: jgit + diff --git a/lava/benchmark/babeltrace/benchmark.yml b/lava/benchmark/babeltrace/benchmark.yml new file mode 100644 index 0000000..3bd1c33 --- /dev/null +++ b/lava/benchmark/babeltrace/benchmark.yml @@ -0,0 +1,37 @@ +metadata: + format: Lava-Test Test Definition 1.0 + name: babeltrace 2.0 benchmark + description: "Run benchmark for babeltrace" +params: + TRACE_LOCATION: "https://obj.internal.efficios.com/lava/traces/benchmark/babeltrace/babeltrace_benchmark_trace.tar.gz" + COMMIT: "invalid" + GIT_URL: "https://github.com/efficios/babeltrace.git" +run: + steps: + - apt install -y time + - git clone ${GIT_URL} babeltrace + - pushd babeltrace + - git checkout ${COMMIT} + - ./bootstrap + - ./configure --disable-man-pages + - make -j + - make install + - ldconfig + - if [ -a /usr/local/bin/babeltrace ] ; then echo "Running bt1"; else ln -s /usr/local/bin/babeltrace2 /usr/local/bin/babeltrace; fi + - popd + - apt install -y curl python3 + - git clone https://github.com/lttng/lttng-ci ci + - export TMPDIR="/tmp" + - mkdir -p coredump + - echo "$(pwd)/coredump/core.%e.%p.%h.%t" > /proc/sys/kernel/core_pattern + - ulimit -c unlimited + - mkdir /tmp/ram_disk + - mount -t tmpfs -o size=10024m new_ram_disk /tmp/ram_disk + - curl -o /tmp/trace.tar.gz "${TRACE_LOCATION}" + - mkdir /tmp/ram_disk/trace + - tar xvf /tmp/trace.tar.gz --directory /tmp/ram_disk/trace/ + - python3 ./ci/scripts/babeltrace-benchmark/time.py --output=result_dummy_sink --command "babeltrace /tmp/ram_disk/trace/ -o dummy" --iteration 5 + - python3 ./ci/scripts/babeltrace-benchmark/time.py --output=result_text_sink --command "babeltrace /tmp/ram_disk/trace/" --iteration 5 + - ./ci/lava/upload_artifact.sh result_dummy_sink results/benchmarks/babeltrace/dummy/${COMMIT} + - ./ci/lava/upload_artifact.sh result_text_sink results/benchmarks/babeltrace/text/${COMMIT} + diff --git a/scripts/babeltrace-benchmark/benchmark.py b/scripts/babeltrace-benchmark/benchmark.py new file mode 100644 index 0000000..cf58ad1 --- /dev/null +++ b/scripts/babeltrace-benchmark/benchmark.py @@ -0,0 +1,415 @@ +#!/usr/bin/python3 +# Copyright (C) 2019 - Jonathan Rajotte +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import json +import os +import tempfile +from statistics import mean +import argparse +import sys +from operator import add + +import matplotlib.pyplot as plt +from matplotlib.backends.backend_pdf import PdfPages +from matplotlib.ticker import PercentFormatter + +import git +import numpy +import lava_submit + +from minio import Minio +from minio.error import NoSuchKey +from minio.error import ResponseError + + +BENCHMARK_TYPES = ["dummy", "text"] +DEFAULT_BUCKET = "lava" + + +def graph_get_color(branch): + """ + Get the color matching the branch. + """ + color = {"stable-1.5": "red", "stable-2.0": "green", "master": "blue"} + return color[branch] + + +def graph_get_title(branch, benchmark_type): + """ + Get title for graph based on benchmark type. + """ + string = {"dummy": "Dummy output", "text": "Text output"} + return "{} - {}".format(branch, string[benchmark_type]) + + +def get_client(): + """ + Return minio client configured. + """ + return Minio( + "obj.internal.efficios.com", access_key="jenkins", secret_key="echo123456" + ) + + +def get_file(client, prefix, file_name, workdir_name): + """ + Return the path of the downloaded file. + Return None on error + """ + destination = os.path.join(workdir_name, file_name) + object_name = "{}/{}".format(prefix, file_name) + try: + client.fget_object(DEFAULT_BUCKET, object_name, destination) + except NoSuchKey: + return None + + return destination + + +def delete_file(client, prefix, file_name): + """ + Delete the file on remote. + """ + object_name = "{}/{}".format(prefix, file_name) + try: + client.remove_object(DEFAULT_BUCKET, object_name) + except ResponseError as err: + print(err) + except NoSuchKey: + pass + + +def get_git_log(bt_version, cutoff, repo_path): + """ + Return an ordered (older to newer) list of commits for the bt_version and + cutoff. WARNING: This changes the git repo HEAD. + """ + repo = git.Repo(repo_path) + repo.git.fetch() + return repo.git.log( + "{}..origin/{}".format(cutoff, bt_version), "--pretty=format:%H", "--reverse" + ).split("\n") + + +def parse_result(result_path): + """ + Parse the result file. Return a dataset of User time + System time. + """ + with open(result_path) as result: + parsed_result = json.load(result) + return list( + map( + add, + parsed_result["User time (seconds)"], + parsed_result["System time (seconds)"], + ) + ) + + +def get_benchmark_results(client, commit, workdir): + """ + Fetch the benchmark result from a certain commit across all benchmark type. + """ + results = {} + benchmark_valid = True + for b_type in BENCHMARK_TYPES: + prefix = "/results/benchmarks/babeltrace/{}/".format(b_type) + result_file = get_file(client, prefix, commit, workdir) + if not result_file: + """ + Benchmark is either corrupted or not complete. + """ + return None, benchmark_valid + results[b_type] = parse_result(result_file) + if all(i == 0.0 for i in results[b_type]): + benchmark_valid = False + print("Invalid benchmark for {}/{}/{}".format(prefix, b_type, commit)) + # The dataset is valid return immediately. + return results, benchmark_valid + + +def plot_raw_value(branch, benchmark_type, x_data, y_data, labels, latest_values): + """ + Plot the graph using the raw value. + """ + point_x_data = [] + outlier_x_data = [] + point_y_data = [] + outlier_y_data = [] + for pos in range(len(x_data)): + x = x_data[pos] + valid_points, outliers = sanitize_dataset(y_data[pos]) + for y in valid_points: + point_x_data.append(x) + point_y_data.append(y) + for y in outliers: + outlier_x_data.append(x) + outlier_y_data.append(y) + + plt.plot( + point_x_data, point_y_data, "o", label=branch, color=graph_get_color(branch) + ) + plt.plot(outlier_x_data, outlier_y_data, "+", label="outlier", color="black") + + ymin = 0 + ymax = 1 + if y_data: + ymin = 0.8 * min([item for sublist in y_data for item in sublist]) + ymax = 1.2 * max([item for sublist in y_data for item in sublist]) + # Put latest of other branches for reference as horizontal line. + for l_branch, l_result in latest_values.items(): + if not l_result or l_branch == branch: + continue + plt.axhline( + y=l_result, + label="Latest {}".format(l_branch), + color=graph_get_color(l_branch), + ) + if l_result <= ymin: + ymin = 0.8 * l_result + if l_result >= ymax: + ymax = 1.2 * l_result + + plt.ylim(ymin=ymin, ymax=ymax) + plt.xticks(x_data, labels, rotation=90, family="monospace") + plt.title(graph_get_title(branch, benchmark_type), fontweight="bold") + plt.ylabel("User + system time (s)") + plt.xlabel("Latest commits") + plt.legend() + + plt.tight_layout() + return + + +def plot_ratio(branch, benchmark_type, x_data, y_data, labels, latest_values): + """ + Plot the graph using a ratio using first point as reference (0%). + """ + reference = 0.01 + y_abs_max = 100 + + if y_data: + reference = y_data[0] + + # Transform y_data to a list of ratio for which the reference is the first + # element. + local_y_data = list(map(lambda y: ((y / reference) - 1.0) * 100, y_data)) + + plt.plot(x_data, local_y_data, "o", label=branch, color=graph_get_color(branch)) + + # Put latest of other branches for reference as horizontal line. + for l_branch, l_result in latest_values.items(): + if not l_result or l_branch == branch: + continue + ratio_l_result = ((l_result / reference) - 1.0) * 100.0 + print( + "branch {} branch {} value {} l_result {} reference {}".format( + branch, l_branch, ratio_l_result, l_result, reference + ) + ) + plt.axhline( + y=ratio_l_result, + label="Latest {}".format(l_branch), + color=graph_get_color(l_branch), + ) + + # Draw the reference line. + plt.axhline(y=0, label="Reference (leftmost point)", linestyle="-", color="Black") + + # Get max absolute value to align the y axis with zero in the middle. + if local_y_data: + local_abs_max = abs(max(local_y_data, key=abs)) * 1.3 + if y_abs_max > 100: + y_abs_max = local_abs_max + + plt.ylim(ymin=y_abs_max * -1, ymax=y_abs_max) + + ax = plt.gca() + percent_formatter = PercentFormatter() + ax.yaxis.set_major_formatter(percent_formatter) + ax.yaxis.set_minor_formatter(percent_formatter) + plt.xticks(x_data, labels, rotation=90, family="monospace") + plt.title(graph_get_title(branch, benchmark_type), fontweight="bold") + plt.ylabel("Ratio") + plt.xlabel("Latest commits") + plt.legend() + + plt.tight_layout() + return + + +def generate_graph(branches, report_name, git_path): + + # The PDF document + pdf_pages = PdfPages(report_name) + + client = get_client() + branch_results = dict() + + # Fetch the results for each branch. + for branch, cutoff in branches.items(): + commits = get_git_log(branch, cutoff, git_path) + results = [] + with tempfile.TemporaryDirectory() as workdir: + for commit in commits: + b_results, valid = get_benchmark_results(client, commit, workdir) + if not b_results or not valid: + continue + results.append((commit, b_results)) + branch_results[branch] = results + + for b_type in BENCHMARK_TYPES: + latest_values = {} + max_len = 0 + + # Find the maximum size for a series inside our series dataset. + # This is used later to compute the size of the actual plot (pdf). + # While there gather the comparison value used to draw comparison line + # between branches. + for branch, results in branch_results.items(): + max_len = max([max_len, len(results)]) + if results: + latest_values[branch] = mean( + sanitize_dataset(results[-1][1][b_type])[0] + ) + else: + latest_values[branch] = None + + for branch, results in branch_results.items(): + # Create a figure instance + if max_len and max_len > 10: + width = 0.16 * max_len + else: + width = 11.69 + + x_data = list(range(len(results))) + y_data = [c[1][b_type] for c in results] + labels = [c[0][:8] for c in results] + + fig = plt.figure(figsize=(width, 8.27), dpi=100) + plot_raw_value(branch, b_type, x_data, y_data, labels, latest_values) + pdf_pages.savefig(fig) + + fig = plt.figure(figsize=(width, 8.27), dpi=100) + # Use the mean of each sanitize dataset here, we do not care for + # variance for ratio. At least not yet. + y_data = [mean(sanitize_dataset(c[1][b_type])[0]) for c in results] + plot_ratio(branch, b_type, x_data, y_data, labels, latest_values) + pdf_pages.savefig(fig) + + pdf_pages.close() + + +def launch_jobs(branches, git_path, wait_for_completion, debug): + """ + Lauch jobs for all missing results. + """ + client = get_client() + for branch, cutoff in branches.items(): + commits = get_git_log(branch, cutoff, git_path) + + with tempfile.TemporaryDirectory() as workdir: + for commit in commits: + b_results = get_benchmark_results(client, commit, workdir)[0] + if b_results: + continue + lava_submit.submit( + commit, wait_for_completion=wait_for_completion, debug=debug + ) + + +def main(): + """ + Parse arguments and execute as needed. + """ + bt_branches = { + "master": "31976fe2d70a8b6b7f8b31b9e0b3bc004d415575", + "stable-2.0": "07f585356018b4ddfbd0e09c49a14e38977c6973", + "stable-1.5": "49e98b837a5667130e0d1e062a6bd7985c7c4582", + } + + parser = argparse.ArgumentParser(description="Babeltrace benchmark utility") + parser.add_argument( + "--generate-jobs", action="store_true", help="Generate and send jobs" + ) + parser.add_argument( + "--do-not-wait-on-completion", + action="store_true", + default=False, + help="Wait for the completion of each jobs sent. This is useful" + "for the ci. Otherwise we could end up spaming the lava instance.", + ) + parser.add_argument( + "--generate-report", + action="store_true", + help="Generate graphs and save them to pdf", + ) + parser.add_argument( + "--report-name", default="report.pdf", help="The name of the pdf report." + ) + parser.add_argument( + "--debug", action="store_true", default=False, help="Do not send jobs to lava." + ) + parser.add_argument( + "--repo-path", help="The location of the git repo to use.", required=True + ) + + args = parser.parse_args() + + if not os.path.exists(args.repo_path): + print("Repository location does not exists.") + return 1 + + if args.generate_jobs: + print("Launching jobs for:") + for branch, cutoff in bt_branches.items(): + print("\t Branch {} with cutoff {}".format(branch, cutoff)) + launch_jobs( + bt_branches, args.repo_path, not args.do_not_wait_on_completion, args.debug + ) + + if args.generate_report: + print("Generating pdf report ({}) for:".format(args.report_name)) + for branch, cutoff in bt_branches.items(): + print("\t Branch {} with cutoff {}".format(branch, cutoff)) + generate_graph(bt_branches, args.report_name, args.repo_path) + + return 0 + + +def sanitize_dataset(dataset): + """ + Use IRQ 1.5 [1] to remove outlier from the dataset. This is useful to get a + representative mean without outlier in it. + [1] https://en.wikipedia.org/wiki/Interquartile_range#Outliers + """ + sorted_data = sorted(dataset) + q1, q3 = numpy.percentile(sorted_data, [25, 75]) + iqr = q3 - q1 + lower_bound = q1 - (1.5 * iqr) + upper_bound = q3 + (1.5 * iqr) + new_dataset = [] + outliers = [] + for i in dataset: + if lower_bound <= i <= upper_bound: + new_dataset.append(i) + else: + outliers.append(i) + return new_dataset, outliers + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/babeltrace-benchmark/benchmark.sh b/scripts/babeltrace-benchmark/benchmark.sh new file mode 100755 index 0000000..c9f579c --- /dev/null +++ b/scripts/babeltrace-benchmark/benchmark.sh @@ -0,0 +1,55 @@ +#!/bin/bash -exu +# +# Copyright (C) 2019 - Jonathan Rajotte-Julien +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +SRC_DIR="$WORKSPACE/src/babeltrace" +SCRIPT_DIR="$WORKSPACE/src/lttng-ci" +RESULTS_DIR="$WORKSPACE/results" + +REQUIREMENT_PATH="${SCRIPT_DIR}/scripts/babeltrace-benchmark/requirement.txt" +SCRIPT_PATH="${SCRIPT_DIR}/scripts/babeltrace-benchmark/benchmark.py" +VENV="$(mktemp -d)" +TMPDIR="${VENV}/tmp" + +mkdir -p "$TMPDIR" +export TMPDIR + +function setup_env () +{ + mkdir -p "$RESULTS_DIR" + virtualenv --python python3 "$VENV" + set +u + # shellcheck disable=SC1090 + . "${VENV}/bin/activate" + set -u + pip install -r "$REQUIREMENT_PATH" +} + +function run_jobs () +{ + python "$SCRIPT_PATH" --generate-jobs --repo-path "$SRC_DIR" +} + +function generate_report () +{ + python "$SCRIPT_PATH" --generate-report --repo-path "$SRC_DIR" --report-name "${RESULTS_DIR}/babeltrace-benchmark.pdf" +} + +setup_env +run_jobs +generate_report + +rm -rf "$VENV" diff --git a/scripts/babeltrace-benchmark/lava_submit.py b/scripts/babeltrace-benchmark/lava_submit.py new file mode 100644 index 0000000..b8e4a5a --- /dev/null +++ b/scripts/babeltrace-benchmark/lava_submit.py @@ -0,0 +1,137 @@ +#!/usr/bin/python3 +# Copyright (C) 2019 - Jonathan Rajotte Julien +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import argparse +import os +import sys +import time +import xmlrpc.client + +from jinja2 import Environment, FileSystemLoader + +USERNAME = "lava-jenkins" +HOSTNAME = "lava-master-02.internal.efficios.com" +DEFAULT_KERNEL_COMMIT = "1a1a512b983108015ced1e7a7c7775cfeec42d8c" + + +def wait_on(server, jobid): + """ + Wait for the completion of the job. + Do not care for result. This is mostly to prevent flooding of lava with + multiple jobs for the same commit hash. Jenkins is responsible for + running only one job for job submissions. + """ + # Check the status of the job every 30 seconds + jobstatus = server.scheduler.job_state(jobid)["job_state"] + running = False + while jobstatus in ["Submitted", "Scheduling", "Scheduled", "Running"]: + if not running and jobstatus == "Running": + print("Job started running", flush=True) + running = True + time.sleep(30) + try: + jobstatus = server.scheduler.job_state(jobid)["job_state"] + except xmlrpc.client.ProtocolError: + print("Protocol error, retrying", flush=True) + continue + print("Job ended with {} status.".format(jobstatus), flush=True) + + +def submit( + commit, debug=False, kernel_commit=DEFAULT_KERNEL_COMMIT, wait_for_completion=True +): + nfsrootfs = "https://obj.internal.efficios.com/lava/rootfs/rootfs_amd64_xenial_2018-12-05.tar.gz" + kernel_url = "https://obj.internal.efficios.com/lava/kernel/{}.baremetal.bzImage".format( + kernel_commit + ) + modules_url = "https://obj.internal.efficios.com/lava/modules/linux/{}.baremetal.linux.modules.tar.gz".format( + kernel_commit + ) + + lava_api_key = None + if not debug: + try: + lava_api_key = os.environ["LAVA2_JENKINS_TOKEN"] + except Exception as error: + print( + "LAVA2_JENKINS_TOKEN not found in the environment variable. Exiting...", + error, + ) + return -1 + + jinja_loader = FileSystemLoader(os.path.dirname(os.path.realpath(__file__))) + jinja_env = Environment(loader=jinja_loader, trim_blocks=True, lstrip_blocks=True) + jinja_template = jinja_env.get_template("template_lava_job_bt_benchmark.jinja2") + + context = dict() + context["kernel_url"] = kernel_url + context["nfsrootfs_url"] = nfsrootfs + context["modules_url"] = modules_url + context["commit_hash"] = commit + + render = jinja_template.render(context) + + print("Job to be submitted:", flush=True) + + print(render, flush=True) + + if debug: + return 0 + + server = xmlrpc.client.ServerProxy( + "http://%s:%s@%s/RPC2" % (USERNAME, lava_api_key, HOSTNAME) + ) + + for attempt in range(10): + try: + jobid = server.scheduler.submit_job(render) + except xmlrpc.client.ProtocolError as error: + print( + "Protocol error on submit, sleeping and retrying. Attempt #{}".format( + attempt + ), + flush=True, + ) + time.sleep(5) + continue + else: + break + + print("Lava jobid:{}".format(jobid), flush=True) + print( + "Lava job URL: http://lava-master-02.internal.efficios.com/scheduler/job/{}".format( + jobid + ), + flush=True, + ) + + if not wait_for_completion: + return 0 + + wait_on(server, jobid) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Launch baremetal babeltrace test using Lava" + ) + parser.add_argument("-c", "--commit", required=True) + parser.add_argument( + "-k", "--kernel-commit", required=False, default=DEFAULT_KERNEL_COMMIT + ) + parser.add_argument("-d", "--debug", required=False, action="store_true") + args = parser.parse_args() + sys.exit(submit(args.kernel_commit, args.commit, args.debug)) diff --git a/scripts/babeltrace-benchmark/requirement.txt b/scripts/babeltrace-benchmark/requirement.txt new file mode 100644 index 0000000..3404236 --- /dev/null +++ b/scripts/babeltrace-benchmark/requirement.txt @@ -0,0 +1,33 @@ +backcall==0.1.0 +certifi==2019.3.9 +cycler==0.10.0 +decorator==4.4.0 +gitdb2==2.0.5 +GitPython==2.1.11 +ipython==7.5.0 +ipython-genutils==0.2.0 +jedi==0.13.3 +Jinja2==2.10.1 +kiwisolver==1.1.0 +MarkupSafe==1.1.1 +matplotlib==3.1.0 +minio==4.0.17 +mpld3==0.3 +numpy==1.16.4 +parso==0.4.0 +pexpect==4.7.0 +pickleshare==0.7.5 +pkg-resources==0.0.0 +prompt-toolkit==2.0.9 +ptyprocess==0.6.0 +Pygments==2.4.2 +pyparsing==2.4.0 +python-dateutil==2.8.0 +python-magic==0.4.15 +pytz==2019.1 +s3cmd==2.0.2 +six==1.12.0 +smmap2==2.0.5 +traitlets==4.3.2 +urllib3==1.25.3 +wcwidth==0.1.7 diff --git a/scripts/babeltrace-benchmark/template_lava_job_bt_benchmark.jinja2 b/scripts/babeltrace-benchmark/template_lava_job_bt_benchmark.jinja2 new file mode 100644 index 0000000..4b7d724 --- /dev/null +++ b/scripts/babeltrace-benchmark/template_lava_job_bt_benchmark.jinja2 @@ -0,0 +1,84 @@ +device_type: x86 +job_name: babeltrace_benchmark +timeouts: + job: + hours: 3 + action: + hours: 3 + connection: + minutes: 4 + connections: + lava-test-shell: + minutes: 4 +priority: medium +visibility: public +context: + extra_kernel_args: cpuidle.off=1 + extra_nfsroot_args: ",nfsvers=3 nfsrootdebug" + +tags: + - dev-sda1 + + +actions: + - deploy: + timeout: + minutes: 10 + to: tftp + kernel: + url: {{ kernel_url }} + type: zimage + modules: + url: {{ modules_url }} + compression: gz + nfsrootfs: + url: {{ nfsrootfs_url }} + compression: gz + + - boot: + timeout: + minutes: 10 + method: ipxe + commands: nfs + auto_login: + login_prompt: 'login:' + username: root + password_prompt: 'Password:' + password: root + prompts: + - 'root@linaro-server:~#' + + - test: + definitions: + # Base setup of environment # + - repository: + metadata: + format: Lava-Test Test Definition 1.0 + name: x86-env-setup + description: "Basic environment setup for x86 board" + os: + - ubuntu + devices: + - x86 + run: + steps: + - chmod 755 / + - systemctl start systemd-timesyncd + - echo nameserver 172.18.0.12 > /etc/resolv.conf + - ip a + - mount /dev/sda /tmp + - rm -rf /tmp/* + - locale-gen en_US.UTF-8 + - apt-get update + - apt-get upgrade + - hash -r + - sync + from: inline + name: x86-env-setup-inline + path: inline/x86-env-setup.yaml + - repository: https://github.com/lttng/lttng-ci.git + from: git + path: lava/benchmark/babeltrace/benchmark.yml + name: babeltrace-benchmark + params: + COMMIT: {{ commit_hash }} diff --git a/scripts/babeltrace-benchmark/time.py b/scripts/babeltrace-benchmark/time.py new file mode 100644 index 0000000..e1506a8 --- /dev/null +++ b/scripts/babeltrace-benchmark/time.py @@ -0,0 +1,185 @@ +#!/usr/bin/python3 +# Copyright (C) 2019 - Jonathan Rajotte Julien +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import argparse +import os +import subprocess +import tempfile +import json +from collections import defaultdict + +_METRIC = { + "User time (seconds)": float, + "System time (seconds)": float, + "Percent of CPU this job got": percent_parser, + "Elapsed (wall clock) time (h:mm:ss or m:ss)": wall_clock_parser, + "Average shared text size (kbytes)": int, + "Average unshared data size (kbytes)": int, + "Average stack size (kbytes)": int, + "Average total size (kbytes)": int, + "Maximum resident set size (kbytes)": int, + "Average resident set size (kbytes)": int, + "Major (requiring I/O) page faults": int, + "Minor (reclaiming a frame) page faults": int, + "Voluntary context switches": int, + "Involuntary context switches": int, + "Swaps": int, + "File system inputs": int, + "File system outputs": int, + "Socket messages sent": int, + "Socket messages received": int, + "Signals delivered": int, + "Page size (bytes)": int, +} + + +def wall_clock_parser(value): + """ + Parse /usr/bin/time wall clock value. + Wall clock value is expressed in different formats depending on the actual + elapsed time. + """ + total = 0.0 + pos = value.find(".") + if value.find("."): + total += float(value[pos:]) + value = value[:pos] + + v_split = value.split(":") + if len(v_split) == 2: + total += float(v_split[0]) * 60.0 + total += float(v_split[1]) * 1.0 + elif len(v_split) == 3: + total += float(v_split[0]) * 360.0 + total += float(v_split[1]) * 60.0 + total += float(v_split[2]) * 1.0 + else: + return 0.0 + + return total + + +def percent_parser(value): + """ + Parse /usr/bin/time percent value. + """ + parsed = value.replace("%", "").replace("?", "") + if parsed: + return float(parsed) + return 0 + + +def parse(path, results): + """ + Parser and accumulator for /usr/bin/time results. + """ + with open(path, "r") as data: + for line in data: + if line.rfind(":") == -1: + continue + key, value = line.lstrip().rsplit(": ") + if key in _METRIC: + results[key].append(_METRIC[key](value)) + + return results + + +def save(path, results): + """ + Save the result in json format to path. + """ + with open(path, "w") as out: + json.dump(results, out, sort_keys=True, indent=4) + + +def run(command, iteration, output, stdout, stderr): + """ + Run the command throught /usr/bin/time n iterations and parse each result. + """ + results = defaultdict(list) + for i in range(iteration): + time_stdout = tempfile.NamedTemporaryFile(delete=False) + # We must delete this file later on. + time_stdout.close() + with open(stdout, "a+") as out, open(stderr, "a+") as err: + cmd = "/usr/bin/time -v --output='{}' {}".format(time_stdout.name, command) + ret = subprocess.run(cmd, shell=True, stdout=out, stderr=err) + if ret.returncode != 0: + print("Iteration: {}, Command failed: {}".format(str(i), cmd)) + results = parse(time_stdout.name, results) + os.remove(time_stdout.name) + save(output, results) + + +def main(): + """ + Run /usr/bin/time N time and collect the result. + The resulting json have the following form: + { + "/usr/bin/time": { + "User time (seconds)": [], + "System time (seconds)": [], + "Percent of CPU this job got": [], + "Elapsed (wall clock) time (h:mm:ss or m:ss)": [], + "Average shared text size (kbytes)": [], + "Average unshared data size (kbytes)": [], + "Average stack size (kbytes)": [], + "Average total size (kbytes)": [], + "Maximum resident set size (kbytes)": [], + "Average resident set size (kbytes)": [], + "Major (requiring I/O) page faults": [], + "Minor (reclaiming a frame) page faults": [], + "Voluntary context switches": [], + "Involuntary context switches": [], + "Swaps": [], + "File system inputs": [], + "File system outputs": [], + "Socket messages sent": [], + "Socket messages received": [], + "Signals delivered": [], + "Page size (bytes)": [], + } + } + """ + parser = argparse.ArgumentParser( + description="Run command N time using /usr/bin/time and collect the statistics" + ) + parser.add_argument("--output", help="Where to same the result", required=True) + parser.add_argument("--command", help="The command to benchmark", required=True) + parser.add_argument( + "--iteration", + type=int, + default=5, + help="The number of iteration to run the command (default: 5)", + required=True, + ) + parser.add_argument( + "--stdout", + default="/dev/null", + help="Where to append the stdout of each command (default: /dev/null)", + ) + parser.add_argument( + "--stderr", + default=os.path.join(os.getcwd(), "stderr.out"), + help="Where to append the stderr of each command (default: $CWD/stderr.out)", + ) + + args = parser.parse_args() + run(args.command, args.iteration, args.output, args.stdout, args.stderr) + + +if __name__ == "__main__": + main() -- 2.34.1