Commit 239aea87 authored by ufebl's avatar ufebl

Allow for more than two compiler flags

parent 789f9bcb
......@@ -55,7 +55,7 @@ if True:#__name__ == '__main__':
parser.add_argument("--ci_testing", action="store_true", default=False,
help="In mode X the succeeding test cases of later modes/phases should also succeed in "
"this mode, and failing test cases of prior modes/phases should also fail in this phase.")
parser.add_argument("--bench_compiler_flags", action="store", default="", type=str, nargs=2,
parser.add_argument("--bench_compiler_flags", action="store", default="", type=str, nargs="*",
help="Set the different compiler flags/modes that are compared, 'javac' for the java compiler")
parser.add_argument("--bench_runs", action="store", type=int, default=10,
help="Number of times to run a benchmarked code")
......
......@@ -149,7 +149,7 @@ class BenchExecTest(BasicSyntaxTest):
results.append(self._bench_command("./a.out", timeout=timeout, input_str=input_str))
os.chdir(cwd)
assert len(results) == 2
assert len(results) >= 2 # We want to compare the first two, so we need at least two
if not results[0].is_correct or not results[1].is_correct:
incorrect_flags = [self.env.bench_compiler_flags[i] for (i, res) in enumerate(results) if not res.is_correct]
test_result.incorrect_msg = "Running with {} failed".format(", ".join(incorrect_flags))
......@@ -161,13 +161,13 @@ class BenchExecTest(BasicSyntaxTest):
msg_parts.append("min(0/1) = {:.0%}".format(rel_min))
rel_mean = results[0].mean() / results[1].mean()
msg_parts.append("mean(0/1) = {:.0%}".format(rel_mean))
msg_parts.append("-1 / std = {:.0%}".format(( - 1) / stddev))
msg_parts.append("-1 / std = {:.0%}".format(( rel_mean- 1) / stddev))
for (i, res) in enumerate(results):
test_result.add_short_text("min({})".format(i), res.min())
test_result.add_short_text("max({})".format(i), res.max())
test_result.add_short_text("mean({})".format(i), res.mean())
test_result.add_short_text("stddev({})".format(i), res.stddev())
if (rel_mean - 1) / stddev <= 1:
if rel_mean > 1.1 and (rel_mean - 1) / stddev <= 1:
msg_parts.append("first faster")
test_result.incorrect_msg = ", ".join(msg_parts)
test_result.has_succeeded = False
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment