bench.py 10.3 KB
Newer Older
1 2 3 4 5 6
import hashlib
import logging
import os
import shutil
import signal
from os import path
ufebl's avatar
ufebl committed
7
from typing import List, Tuple, Optional
8 9 10 11

import math

import subprocess
ufebl's avatar
ufebl committed
12
import re
13 14 15 16 17 18

import time

from mjtest.environment import TestMode, Environment
from mjtest.test.syntax_tests import BasicSyntaxTest
from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult
ufebl's avatar
ufebl committed
19
from mjtest.test.exec_tests import JavaExecTest
20
from mjtest.util.shell import SigKill
21
from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict, decode
22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37

_LOG = logging.getLogger("bench_tests")

class _RunResult:

    def __init__(self, runs: List[float], is_correct: bool):
        self.runs = runs
        self.is_correct = is_correct

    def mean(self) -> float:
        return sum(self.runs) / self.number()

    def stddev(self) -> float:
        m = self.mean()
        return math.sqrt(sum(map(lambda x: (x - m) ** 2, self.runs)) / self.number())

ufebl's avatar
ufebl committed
38 39 40
    def max(self) -> float:
        return max(self.runs)

41 42 43 44 45 46 47 48 49
    def min(self) -> float:
        return min(self.runs)

    def number(self) -> int:
        return len(self.runs)


class BenchExecTest(BasicSyntaxTest):
    """
ufebl's avatar
ufebl committed
50
    Simple benchmark test. The new compiler mode shouldn't create slower binaries than the old ones (or javac)
51 52
    """

ufebl's avatar
ufebl committed
53 54 55 56
    FILE_ENDINGS = [".java"]
    INVALID_FILE_ENDINGS = [".mj", ".inf.java", ".inf.mj", ".input.mj", ".input.java", ".invalid.java", ".invalid.mj"]
    MODE = TestMode.bench
    INPUT_FILE_REGEXP = r'(\.[0-9]+)\.input(c?)$'
57

58
    def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str = ""):
ufebl's avatar
ufebl committed
59
        super().__init__(env, type, file, preprocessed_file, log_file_mode)
ufebl's avatar
ufebl committed
60 61 62 63
        self._has_input_file = bool(re.search(self.INPUT_FILE_REGEXP, file))
        self._input_file = file if self._has_input_file else None
        self._has_character_input = file.endswith("c") if self._has_input_file else False

64 65
        self._should_succeed = True

66
    def _bench_command(self, cmd: str, *args: Tuple[str], compiler_flag: str, timeout: int, input_bytes: str) -> _RunResult:
67 68 69 70
        runs = []  # type: List[float]
        for i in range(0, self.env.bench_runs):
            try:
                start = time.time()
ufebl's avatar
ufebl committed
71

72 73 74 75 76 77 78 79 80 81
                out, err, rtcode = self.env.run_command(cmd, *args, timeout=timeout, input_bytes=input_bytes)
                if rtcode == 0:
                    runs.append((time.time() - start) * 1000)
                else: 
                    _LOG.error("File \"{}\" can't be run with flags \"{}\" error code: {}".format(self.preprocessed_file, compiler_flag, rtcode))
                    _LOG.error("Error: " + err.decode());
                    return _RunResult([-1,0], False);
            except SigKill as sig:
                _LOG.error("File \"{}\" can't be run with flags \"{}\"".format(self.preprocessed_file, compiler_flag))
                _LOG.error("Error: " + sig.name.strip() +" " + sig.retcode)
ufebl's avatar
ufebl committed
82
                return _RunResult([-1,0], False);
83 84 85 86
            except:
                raise
#            except subprocess.CalledProcessError:
#                return _RunResult([-1,0], False);
87 88
        return _RunResult(runs, True)

ufebl's avatar
ufebl committed
89

90 91 92 93 94 95 96 97
    def run(self) -> BasicDiffTestResult:
        is_big_testcase = "big" in self.file
        timeout = self.env.big_timeout if is_big_testcase else self.env.timeout
        base_filename = path.basename(self.file).split(".")[0]
        tmp_dir = self.env.create_pid_local_tmpdir()
        shutil.copy(self.preprocessed_file, path.join(tmp_dir, base_filename + ".java"))
        cwd = os.getcwd()
        os.chdir(tmp_dir)
ufebl's avatar
ufebl committed
98
        input_bytes = None
99 100 101

        test_result = ExtensibleTestResult(self)

ufebl's avatar
ufebl committed
102 103 104
        # read input
        if self._has_input_file:
            if self._has_character_input:
ufebl's avatar
ufebl committed
105 106
                with open(self._input_file, "rb") as f:
                    input_bytes = f.read()
ufebl's avatar
ufebl committed
107
            else:
ufebl's avatar
ufebl committed
108 109
                with open(self._input_file, "r", errors="backslashreplace") as f:
                    input_bytes = bytearray(encoding='ascii', errors='ignore')  # type: bytearray
ufebl's avatar
ufebl committed
110 111 112 113 114 115 116 117
                    for line in f.readlines():
                        for part in line.split(" "):
                            part = part.strip()
                            if len(part) > 0:
                                chars.append(int(part))

       
        results = [] # type: List[_RunResult]
118 119 120 121 122 123 124

        for compiler_flag in self.env.bench_compiler_flags:
            if compiler_flag == "javac":
                _, err, javac_rtcode = \
                    self.env.run_command("javac", base_filename + ".java", timeout=timeout)
                if javac_rtcode != 0:
                    _LOG.error("File \"{}\" isn't valid Java".format(self.preprocessed_file))
ufebl's avatar
ufebl committed
125 126 127 128 129 130 131 132 133 134 135 136 137
                    #test_result.incorrect_msg = "invalid java code, but output file missing"
                    #test_result.set_error_code(javac_rtcode)
                    #test_result.add_long_text("Javac error message", err.decode())
                    #test_result.add_file("Source file", self.preprocessed_file)
                    #os.chdir(cwd)
                    #return test_result
                    results.append(_RunResult([-1,0], False));
                else:
                    main_class = get_main_class_name(base_filename + ".java")
                    if not main_class:
                        _LOG.debug("Can't find a main class, using the file name instead")
                        main_class = base_filename
                
138
                    results.append(self._bench_command("java", main_class, compiler_flag="javac", timeout=timeout, input_bytes=input_bytes))
139 140 141
            else:
                try:
                    compiler_flag = compiler_flag.replace("\\-", "-")
ufebl's avatar
ufebl committed
142
                    out, err, rtcode = self.env.run_command(self.env.mj_run_cmd, *(compiler_flag.split(" ") + [base_filename + ".java"]),
143
                    compiler_flag = compiler_flag,
144 145
                                                            timeout=timeout)
                    if rtcode != 0:
ufebl's avatar
ufebl committed
146 147
                        _LOG.error("File \"{}\" can't be compiled with flags \"{}\"".format(self.preprocessed_file, compiler_flag))
                        _LOG.error("Error: " + err.decode());
ufebl's avatar
ufebl committed
148 149 150 151 152 153 154 155 156 157
                        #test_result.incorrect_msg = "file can't be compiled"
                        #test_result.set_error_code(rtcode)
                        #test_result.add_long_text("Error output", err.decode())
                        #test_result.add_long_text("Output", out.decode())
                        #test_result.add_file("Source file", self.preprocessed_file)
                        #os.chdir(cwd)
                        #return test_result
                        results.append(_RunResult([-1,0], False));
                    else:
                        results.append(self._bench_command("./a.out", timeout=timeout, input_bytes=input_bytes))
158
                except SigKill as sig:
ufebl's avatar
ufebl committed
159 160
                    _LOG.error("File \"{}\" can't be compiled with flags \"{}\"".format(self.preprocessed_file, compiler_flag))
                    _LOG.error("Errorcide: " + śig.retcode);
161 162 163 164 165
                    #test_result.incorrect_msg = "file can't be compiled: " + sig.name
                    #test_result.set_error_code(sig.retcode)
                    #test_result.add_file("Source file", self.preprocessed_file)
                    #os.chdir(cwd)
                    #return test_result
ufebl's avatar
ufebl committed
166
                    results.append(_RunResult([-1,0], False));
167 168 169
                except:
                    os.chdir(cwd)
                    raise
ufebl's avatar
ufebl committed
170

171
                
ufebl's avatar
ufebl committed
172

173
        os.chdir(cwd)
ufebl's avatar
ufebl committed
174
        assert len(results) >= 2 # We want to compare the first two, so we need at least two
175 176 177 178 179 180 181 182 183 184 185 186
        for (i, res) in enumerate(results):
            if not results[i]:
                incorrect_flags = [self.env.bench_compiler_flags[i] for (i, res) in enumerate(results) if not res.is_correct]
                test_result.incorrect_msg = "Running with {} failed".format(", ".join(incorrect_flags))
                test_result.has_succeeded = False
                # add negative results
                for (i, res) in enumerate(results):
                    test_result.add_short_text("min({})".format(i), res.min())
                    test_result.add_short_text("max({})".format(i), res.max())
                    test_result.add_short_text("mean({})".format(i), res.mean())
                    test_result.add_short_text("stddev({})".format(i), res.stddev())
                return test_result
187 188 189 190 191 192
        msg_parts = []
        stddev = max(results[0].stddev() / results[0].mean(), results[1].stddev() / results[1].mean())
        rel_min = results[0].min() / results[1].min()
        msg_parts.append("min(0/1) = {:.0%}".format(rel_min))
        rel_mean = results[0].mean() / results[1].mean()
        msg_parts.append("mean(0/1) = {:.0%}".format(rel_mean))
ufebl's avatar
ufebl committed
193
        msg_parts.append("-1 / std = {:.0%}".format(( rel_mean- 1) / stddev))
194 195
        for (i, res) in enumerate(results):
            test_result.add_short_text("min({})".format(i), res.min())
ufebl's avatar
ufebl committed
196
            test_result.add_short_text("max({})".format(i), res.max())
197 198
            test_result.add_short_text("mean({})".format(i), res.mean())
            test_result.add_short_text("stddev({})".format(i), res.stddev())
199 200
        #if (rel_mean - 1) / stddev <= 1:
        if results[0].mean() - results[0].stddev() > results[1].mean() + results[1].stddev():
201 202 203 204 205 206 207 208 209
            msg_parts.append("first faster")
            test_result.incorrect_msg = ", ".join(msg_parts)
            test_result.has_succeeded = False
            return test_result
        test_result.correct_msg = ", ".join(msg_parts)
        test_result.has_succeeded = True
        return test_result


ufebl's avatar
ufebl committed
210 211 212 213 214 215 216 217 218 219 220 221 222 223 224
    # Fetch different class files for tests with input

    @staticmethod
    def is_file_ending_valid(cls, file: str) -> bool:
        if re.search(cls.INPUT_FILE_REGEXP, file) and ".java_output" not in file and "precprocessor" not in file:
            if not JavaExecTest._test_case_file_for_input_file(file):
                _LOG.error("Skip {} that hasn't a corresponding test case file".format(file))
            return True
        return super().is_file_ending_valid(cls, file)

    @staticmethod
    def _get_test_case_file(cls, file: str) -> Optional[str]:
        if re.search(JavaExecTest.INPUT_FILE_REGEXP, file):
            return JavaExecTest._test_case_file_for_input_file(file)
        return file
225 226

TestCase.TEST_CASE_CLASSES[TestMode.bench].append(BenchExecTest)