Commit 859fa958 authored by ufebl's avatar ufebl

Fix bench mode

parent c962e39f
......@@ -4,17 +4,19 @@ import os
import shutil
import signal
from os import path
from typing import List, Tuple
from typing import List, Tuple, Optional
import math
import subprocess
import re
import time
from mjtest.environment import TestMode, Environment
from mjtest.test.syntax_tests import BasicSyntaxTest
from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult
from mjtest.test.exec_tests import JavaExecTest
from mjtest.util.shell import SigKill
from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict
......@@ -33,6 +35,9 @@ class _RunResult:
m = self.mean()
return math.sqrt(sum(map(lambda x: (x - m) ** 2, self.runs)) / self.number())
def max(self) -> float:
return max(self.runs)
def min(self) -> float:
return min(self.runs)
......@@ -42,28 +47,34 @@ class _RunResult:
class BenchExecTest(BasicSyntaxTest):
"""
Simple benchmark test. The new compiler mode shouldn't be slower than the old ones (or javac)
Simple benchmark test. The new compiler mode shouldn't create slower binaries than the old ones (or javac)
"""
FILE_ENDINGS = [".java", ".mj"]
INVALID_FILE_ENDINGS = [".inf.java", ".inf.mj"]
MODE = TestMode.compile_firm
FILE_ENDINGS = [".java"]
INVALID_FILE_ENDINGS = [".mj", ".inf.java", ".inf.mj", ".input.mj", ".input.java", ".invalid.java", ".invalid.mj"]
MODE = TestMode.bench
INPUT_FILE_REGEXP = r'(\.[0-9]+)\.input(c?)$'
def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str):
super().__init__(env, type, file, preprocessed_file)
def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str):
super().__init__(env, type, file, preprocessed_file, log_file_mode)
self._has_input_file = bool(re.search(self.INPUT_FILE_REGEXP, file))
self._input_file = file if self._has_input_file else None
self._has_character_input = file.endswith("c") if self._has_input_file else False
self._should_succeed = True
def _bench_command(self, cmd: str, *args: Tuple[str]) -> _RunResult:
def _bench_command(self, cmd: str, *args: Tuple[str], timeout: int, input_str: str) -> _RunResult:
runs = [] # type: List[float]
for i in range(0, self.env.bench_runs):
try:
start = time.time()
subprocess.check_call([cmd] + list(args), stdout=subprocess.DEVNULL)
runs.append(time.time() - start)
self.env.run_command(cmd, *args, timeout=timeout, input_str=input_str)
runs.append((time.time() - start) * 1000)
except subprocess.CalledProcessError:
return _RunResult([], False)
return _RunResult(runs, True)
def run(self) -> BasicDiffTestResult:
is_big_testcase = "big" in self.file
timeout = self.env.big_timeout if is_big_testcase else self.env.timeout
......@@ -72,10 +83,27 @@ class BenchExecTest(BasicSyntaxTest):
shutil.copy(self.preprocessed_file, path.join(tmp_dir, base_filename + ".java"))
cwd = os.getcwd()
os.chdir(tmp_dir)
input_str = None
test_result = ExtensibleTestResult(self)
results = [] # type: List[_RunResult]
# read input
if self._has_input_file:
if self._has_character_input:
with open(self._input_file, "r") as f:
input_str = f.read()
else:
with open(self._input_file, "r") as f:
chars = bytearray('ascii', 'ignore') # type: bytearray
for line in f.readlines():
for part in line.split(" "):
part = part.strip()
if len(part) > 0:
chars.append(int(part))
input_str = chars.decode()
results = [] # type: List[_RunResult]
for compiler_flag in self.env.bench_compiler_flags:
if compiler_flag == "javac":
......@@ -93,11 +121,12 @@ class BenchExecTest(BasicSyntaxTest):
if not main_class:
_LOG.debug("Can't find a main class, using the file name instead")
main_class = base_filename
results.append(self._bench_command("java", main_class))
results.append(self._bench_command("java", main_class, timeout=timeout, input_str=input_str))
else:
try:
compiler_flag = compiler_flag.replace("\\-", "-")
out, err, rtcode = self.env.run_command(self.env.mj_run_cmd, compiler_flag, base_filename + ".java",
out, err, rtcode = self.env.run_command(self.env.mj_run_cmd, *(compiler_flag.split(" ") + [base_filename + ".java"]),
timeout=timeout)
if rtcode != 0:
test_result.incorrect_msg = "file can't be compiled"
......@@ -116,7 +145,9 @@ class BenchExecTest(BasicSyntaxTest):
except:
os.chdir(cwd)
raise
results.append(self._bench_command("./a.out"))
results.append(self._bench_command("./a.out", timeout=timeout, input_str=input_str))
os.chdir(cwd)
assert len(results) == 2
if not results[0].is_correct or not results[1].is_correct:
......@@ -130,9 +161,10 @@ class BenchExecTest(BasicSyntaxTest):
msg_parts.append("min(0/1) = {:.0%}".format(rel_min))
rel_mean = results[0].mean() / results[1].mean()
msg_parts.append("mean(0/1) = {:.0%}".format(rel_mean))
msg_parts.append("-1 / std = {:.0%}".format((rel_mean - 1) / stddev))
msg_parts.append("-1 / std = {:.0%}".format(( - 1) / stddev))
for (i, res) in enumerate(results):
test_result.add_short_text("min({})".format(i), res.min())
test_result.add_short_text("max({})".format(i), res.max())
test_result.add_short_text("mean({})".format(i), res.mean())
test_result.add_short_text("stddev({})".format(i), res.stddev())
if (rel_mean - 1) / stddev <= 1:
......@@ -140,10 +172,25 @@ class BenchExecTest(BasicSyntaxTest):
test_result.incorrect_msg = ", ".join(msg_parts)
test_result.has_succeeded = False
return test_result
test_result.correct_msg = ", ".join(msg_parts)
#test_result.correct_msg = ", ".join(msg_parts)
test_result.has_succeeded = True
return test_result
# Fetch different class files for tests with input
@staticmethod
def is_file_ending_valid(cls, file: str) -> bool:
if re.search(cls.INPUT_FILE_REGEXP, file) and ".java_output" not in file and "precprocessor" not in file:
if not JavaExecTest._test_case_file_for_input_file(file):
_LOG.error("Skip {} that hasn't a corresponding test case file".format(file))
return True
return super().is_file_ending_valid(cls, file)
@staticmethod
def _get_test_case_file(cls, file: str) -> Optional[str]:
if re.search(JavaExecTest.INPUT_FILE_REGEXP, file):
return JavaExecTest._test_case_file_for_input_file(file)
return file
TestCase.TEST_CASE_CLASSES[TestMode.bench].append(BenchExecTest)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment