X-Git-Url: http://git.lttng.org./?a=blobdiff_plain;f=scripts%2Fbabeltrace-benchmark%2Fbenchmark.py;h=f48a5009730a63f72a83ad45260cf26b896f3e7e;hb=b2f18af4604f35660d9162f65f87e8d59f35e8ea;hp=518dce70ab117c0286ecfeb5fb53be5ffed02750;hpb=056f7519e4772cdd4636d386a5252d9e23866729;p=lttng-ci.git diff --git a/scripts/babeltrace-benchmark/benchmark.py b/scripts/babeltrace-benchmark/benchmark.py index 518dce7..f48a500 100644 --- a/scripts/babeltrace-benchmark/benchmark.py +++ b/scripts/babeltrace-benchmark/benchmark.py @@ -38,6 +38,69 @@ from minio.error import ResponseError BENCHMARK_TYPES = ["dummy", "text"] DEFAULT_BUCKET = "lava" +invalid_commits = { + "ec9a9794af488a9accce7708a8b0d8188b498789", # Does not build + "8c99128c640cbce71fb8a6caa15e4c672252b662", # Block on configure + "f3847c753f1b4f12353c38d97b0577d9993d19fb", # Does not build + "e0111295f17ddfcc33ec771a8deac505473a06ad", # Does not build + "d0d4e0ed487ea23aaf0d023513c0a4d86901b79b", # Does not build + "c24f7ab4dd9edeb5e50b0070fd9d9e8691057dde", # Does not build + "ce67f5614a4db3b2de4d887eca52135b439b4937", # Does not build + "80aff5efc66679fd934cef433c0e698694748385", # Does not build + "f4f11e84942d36fcc8a597d226928bce2ccac4b3", # Does not build + "ae466a6e1b856d96cf5112a371b4df2b732503ec", # Does not build + "ade5c95e2a4f90f839f222fc1a66175b3b199922", # Configuration fails + "30341532906d62808e9d66fb115f5edb4e6f5706", # Configuration fails + "006c5ffb42f32e802136e3c27a63accb59b4d6c4", # Does not build + "88488ff5bdcd7679ff1f04fe6cff0d24b4f8fc0c", # Does not build + # Other errors + "7c7301d5827bd10ec7c34da7ffc5fe74e5047d38", + "a0df3abf88616cb0799f87f4eb57c54268e63448", + "b7045dd71bc0524ad6b5db96df365e98e237d395", + "cf7b259eaa602abcef308d2b5dd8e6c9ee995d8b", + "90a55a4ef47cac7b568f5f0a8a78bd760f82d23c", + "baa5e3aa82a82c9d0fa59e3c586c0168bb5dc267", + "af9f8da7ba4a9b16fc36d637b8c3a0c7a8774da2", + "fe748379adbd385efdfc7acae9c2340fb8b7d717", + "baa5e3aa82a82c9d0fa59e3c586c0168bb5dc267", + "af9f8da7ba4a9b16fc36d637b8c3a0c7a8774da2", + "fe748379adbd385efdfc7acae9c2340fb8b7d717", + "929627965e33e06dc77254d81e8ec1d66cc06590", + "48a0e52c4632a60cd43423f2f34f10de350bf868", + "b7fa35fce415b33207a9eba111069ed31ef122a0", + "828c8a25785e0cedaeb6987256a4dfc3c43b982f", + "213489680861e4d796173513effac7023312ec2d", + "430a5ccbbd15782501ca56bb148f3850126277ad", + "629d19044c43b195498d0a4e002906c54b6186d5", + "c423217ed1640b4152739f7e5613775d46c25050", + # Elfutils + "776a2a252c9875caa1e8b4f41cb8cc12c79611c3", + "435aa29aff0527d36aafa1b657ae70b9db5f9ea5", + "95651695473495501fc6b2c4a1cf6a78cfb3cd6a", + "e0748fb2ba8994c136bcc0b67d3044f09841cf8e", + "9e632b22e1310fe773edc32ab08a60602f4b2861", + "271fb6907a6f4705a1c799d925394243eae51d68", + "328342cd737582216dc7b8b7d558b2a1bf8ea5e8", + "ae5c1a4481be68fae027910b141354c1d86daa64", + "e6938018975e45d35dab5fef795fe7344eef7d62", + "e015bae2ef343b30c890eebb9182a8be13d12ed0", + "5e8a0751ae0c418a615025d1da10bc84f91b3d97", + "887d26fa0fd0ae0c5c15e4b885473c4cdc0bf078", + "e97fe75eac59fc39a6e4f3c4f9f3301835a0315e", + "8b130e7f1d6a41fb5c64a014c15246ba74b79470", + "f4f8f79893b18199b38edc3330093a9403c4c737", +} + +def json_type(string): + """ + Argpase type for json args. + We expect a base dictionary. + """ + passed_json = json.loads(string) + if not isinstance(passed_json, dict): + msg = "%r is not a dict" % string + raise argparse.ArgumentTypeError(msg) + return passed_json def graph_get_color(branch): """ @@ -126,7 +189,7 @@ def get_benchmark_results(client, commit, workdir): results = {} benchmark_valid = True for b_type in BENCHMARK_TYPES: - prefix = "/results/benchmarks/babeltrace/{}/".format(b_type) + prefix = "/results/benchmarks/babeltrace/{}".format(b_type) result_file = get_file(client, prefix, commit, workdir) if not result_file: """ @@ -186,6 +249,50 @@ def plot_raw_value(branch, benchmark_type, x_data, y_data, labels, latest_values plt.ylabel("User + system time (s)") plt.xlabel("Latest commits") plt.legend() + plt.grid(True) + + # Put tick on the right side + ax.tick_params(labeltop=False, labelright=True) + + plt.tight_layout() + return + + +def plot_delta_between_point( + branch, benchmark_type, x_data, y_data, labels, latest_values +): + """ + Plot the graph of delta between each sequential commit. + """ + local_abs_max = 100 + + # Transform y_data to a list of for which the reference is the first + # element. + local_y_data = [] + for pos, y in enumerate(y_data): + if pos == 0: + local_y_data.append(0.0) + continue + local_y_data.append(y - y_data[pos - 1]) + + plt.plot(x_data, local_y_data, "o", label=branch, color=graph_get_color(branch)) + + # Get max absolute value to align the y axis with zero in the middle. + if local_y_data: + local_abs_max = abs(max(local_y_data, key=abs)) * 1.3 + + plt.ylim(ymin=local_abs_max * -1, ymax=local_abs_max) + + ax = plt.gca() + plt.xticks(x_data, labels, rotation=90, family="monospace") + plt.title( + graph_get_title(branch, benchmark_type) + " Delta to previous commit", + fontweight="bold", + ) + plt.ylabel("Seconds") + plt.xlabel("Latest commits") + plt.legend() + plt.grid(True) # Put tick on the right side ax.tick_params(labeltop=False, labelright=True) @@ -246,6 +353,7 @@ def plot_ratio(branch, benchmark_type, x_data, y_data, labels, latest_values): plt.ylabel("Ratio") plt.xlabel("Latest commits") plt.legend() + plt.grid(True) # Put tick on the right side ax.tick_params(labeltop=False, labelright=True) @@ -253,6 +361,7 @@ def plot_ratio(branch, benchmark_type, x_data, y_data, labels, latest_values): plt.tight_layout() return + def generate_graph(branches, report_name, git_path): # The PDF document @@ -305,32 +414,41 @@ def generate_graph(branches, report_name, git_path): plot_raw_value(branch, b_type, x_data, y_data, labels, latest_values) pdf_pages.savefig(fig) - fig = plt.figure(figsize=(width, 8.27), dpi=100) # Use the mean of each sanitize dataset here, we do not care for # variance for ratio. At least not yet. y_data = [mean(sanitize_dataset(c[1][b_type])[0]) for c in results] + fig = plt.figure(figsize=(width, 8.27), dpi=100) plot_ratio(branch, b_type, x_data, y_data, labels, latest_values) pdf_pages.savefig(fig) + fig = plt.figure(figsize=(width, 8.27), dpi=100) + plot_delta_between_point( + branch, b_type, x_data, y_data, labels, latest_values + ) + pdf_pages.savefig(fig) + pdf_pages.close() -def launch_jobs(branches, git_path, wait_for_completion, debug): +def launch_jobs(branches, git_path, wait_for_completion, debug, force): """ Lauch jobs for all missing results. """ client = get_client() + commits_to_test = set() for branch, cutoff in branches.items(): - commits = get_git_log(branch, cutoff, git_path) - + commits = [x for x in get_git_log(branch, cutoff, git_path) if x not in invalid_commits] with tempfile.TemporaryDirectory() as workdir: for commit in commits: b_results = get_benchmark_results(client, commit, workdir)[0] - if b_results: + if b_results and not force: continue - lava_submit.submit( - commit, wait_for_completion=wait_for_completion, debug=debug - ) + commits_to_test.add(commit) + for index, commit in enumerate(commits_to_test): + print("Job {}/{}".format(index+1, len(commits_to_test))) + lava_submit.submit( + commit, wait_for_completion=wait_for_completion, debug=debug + ) def main(): @@ -347,6 +465,9 @@ def main(): parser.add_argument( "--generate-jobs", action="store_true", help="Generate and send jobs" ) + parser.add_argument( + "--force-jobs", action="store_true", help="Force the queueing of jobs to lava" + ) parser.add_argument( "--do-not-wait-on-completion", action="store_true", @@ -368,19 +489,35 @@ def main(): parser.add_argument( "--repo-path", help="The location of the git repo to use.", required=True ) + parser.add_argument( + "--overwrite-branches-cutoff", + help="A dictionary of the form {" + "'branch_name': 'commit_hash_cutoff',...}. Allow custom graphing and" + "jobs generation.", + required=False, type=json_type + ) args = parser.parse_args() + if args.overwrite_branches_cutoff: + bt_branches = args.overwrite_branches_cutoff + if not os.path.exists(args.repo_path): print("Repository location does not exists.") return 1 if args.generate_jobs: print("Launching jobs for:") + for branch, cutoff in bt_branches.items(): print("\t Branch {} with cutoff {}".format(branch, cutoff)) + launch_jobs( - bt_branches, args.repo_path, not args.do_not_wait_on_completion, args.debug + bt_branches, + args.repo_path, + not args.do_not_wait_on_completion, + args.debug, + args.force_jobs, ) if args.generate_report: