exec_tests.py 11.5 KB
Newer Older
1
import hashlib
2 3 4
import logging
import os
import shutil
5
import signal
6 7 8
from os import path
from mjtest.environment import TestMode, Environment
from mjtest.test.syntax_tests import BasicSyntaxTest
9
from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult
10
from mjtest.util.shell import SigKill
11
from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict
12 13 14 15 16 17 18 19

_LOG = logging.getLogger("exec_tests")

class JavaExecTest(BasicSyntaxTest):
    """
    The MiniJava compiler should behave the same as javac
    """

20 21
    FILE_ENDINGS = [".java", ".mj"]
    INVALID_FILE_ENDINGS = [".inf.java", ".inf.mj"]
22
    OUTPUT_FILE_ENDING = ".out"
23
    MODE = TestMode.compile_firm
24 25 26 27

    def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str):
        super().__init__(env, type, file, preprocessed_file)
        self._expected_output_file = file + self.OUTPUT_FILE_ENDING
28 29 30 31
        prev_out_dir = path.join(path.dirname(file), ".java_output")
        if not path.exists(prev_out_dir):
            os.mkdir(prev_out_dir)
        self._prev_out_file = path.join(prev_out_dir, path.basename(self._expected_output_file))
32
        self._prev_out_hash_file = self._prev_out_file + "_hash"
33
        self._has_expected_output_file = path.exists(self._expected_output_file)
34
        if not self._has_expected_output_file:
35 36
            if path.exists(self._prev_out_file) and path.exists(self._prev_out_hash_file) \
                    and self._check_hash_sum(self.preprocessed_file, self._prev_out_hash_file):
37 38 39
                self._has_expected_output_file = True
                self._expected_output_file = self._prev_out_file
                _LOG.info("Reuse old java output file \"{}\"".format(path.relpath(self._prev_out_file)))
40 41 42
        self._should_succeed = True

    def run(self) -> BasicDiffTestResult:
43 44
        is_big_testcase = "big" in self.file
        timeout = self.env.big_timeout if is_big_testcase else self.env.timeout
45
        base_filename = path.basename(self.file).split(".")[0]
Johannes Bechberger's avatar
Johannes Bechberger committed
46
        tmp_dir = self.env.create_pid_local_tmpdir()
47 48 49 50
        shutil.copy(self.preprocessed_file, path.join(tmp_dir, base_filename + ".java"))
        cwd = os.getcwd()
        os.chdir(tmp_dir)
        exp_out = None
51
        #print(base_filename, get_main_class_name(base_filename + ".java"))
52 53 54

        test_result = ExtensibleTestResult(self)

55
        if not self._has_expected_output_file:
56
            _, err, javac_rtcode = \
57
                self.env.run_command("javac", base_filename + ".java", timeout=timeout)
58 59
            if javac_rtcode != 0:
                _LOG.error("File \"{}\" isn't valid Java".format(self.preprocessed_file))
60
                test_result.incorrect_msg = "invalid java code, but output file missing"
61
                test_result.set_error_code(javac_rtcode)
62 63
                test_result.add_long_text("Javac error message", err.decode())
                test_result.add_file("Source file", self.preprocessed_file)
64
                os.chdir(cwd)
65
                return test_result
66
            main_class = get_main_class_name(base_filename + ".java")
67 68 69
            if not main_class:
                _LOG.debug("Can't find a main class, using the file name instead")
                main_class = base_filename
70
            exp_out, err, java_rtcode = \
71
                self.env.run_command("java", get_main_class_name(base_filename + ".java"), timeout=timeout)
72
            if javac_rtcode != 0:
73
                test_result.add_long_text("Java output: ", exp_out.decode())
74
                test_result.incorrect_msg = "java runtime error"
75
                test_result.set_error_code(java_rtcode)
76 77 78 79
                test_result.add_long_text("Java error message", err.decode())
                test_result.add_file("Source file", self.preprocessed_file)
                os.chdir(cwd)
                return test_result
80
            exp_out = exp_out.decode()
81

82 83 84
            with open(self._prev_out_file, "w") as f:
                f.write(exp_out)
                f.flush()
85 86 87
            with open(self._prev_out_hash_file, "w") as f:
                f.write(self._hash_sum_for_file(base_filename + ".java"))
                f.flush()
88 89 90
        if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE:
            with open(self._expected_output_file, "r") as f:
                exp_out = f.read()
91

92
        try:
93 94
            out, err, rtcode = None, None, None
            try:
95
                out, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java", timeout=timeout)
96 97
                if rtcode != 0:
                    test_result.incorrect_msg = "file can't be compiled"
98
                    test_result.set_error_code(rtcode)
99 100 101 102 103 104 105
                    test_result.add_long_text("Error output", err.decode())
                    test_result.add_long_text("Output", out.decode())
                    test_result.add_file("Source file", self.preprocessed_file)
                    os.chdir(cwd)
                    return test_result
            except SigKill as sig:
                test_result.incorrect_msg = "file can't be compiled: " + sig.name
106
                test_result.set_error_code(sig.retcode)
107 108 109 110 111 112 113
                test_result.add_file("Source file", self.preprocessed_file)
                os.chdir(cwd)
                return test_result
            except:
                os.chdir(cwd)
                raise
            try:
114
                out, err, rtcode = self.env.run_command("./a.out", timeout=timeout)
115 116 117 118 119 120 121 122
                if rtcode != 0:
                    test_result.incorrect_msg = "binary can't be run, non zero error code"
                    test_result.set_error_code(rtcode)
                    test_result.add_long_text("Error output", err.decode())
                    test_result.add_long_text("Output", out.decode())
                    test_result.add_file("Source file", self.preprocessed_file)
                    os.chdir(cwd)
                    return test_result
123
            except SigKill as sig:
124 125
                test_result.incorrect_msg = "binary can't be run: " + sig.name.strip()
                test_result.set_error_code(sig.retcode)
126 127 128 129 130 131
                test_result.add_file("Source file", self.preprocessed_file)
                os.chdir(cwd)
                return test_result
            except:
                os.chdir(cwd)
                raise
132
            out = out.decode()
133
            if self.type == self.MODE and self.env.mode == self.MODE:
134
                test_result.add_long_text("Output", out)
135 136
                if exp_out.strip() != out.strip():
                    test_result.incorrect_msg = "incorrect output"
137
                    test_result.has_succeeded = False
138 139 140
                    test_result.add_short_text("Expected output file", self._expected_output_file)
                    test_result.add_long_text("Expected output", exp_out)
                    test_result.add_long_text("Actual output", out)
141
                    test_result.add_diff("Output diff [expected <-> actual]", exp_out, out, with_line_numbers=True)
142 143 144
                    test_result.add_file("Source file", self.preprocessed_file)
                os.chdir(cwd)
                return test_result
Johannes Bechberger's avatar
Johannes Bechberger committed
145
            return BasicTestResult(self, rtcode, out, err.decode())
146 147
        except SigKill as sig:
            os.chdir(cwd)
148
            assert False
149 150
        except:
            os.chdir(cwd)
151
            assert False
152

153 154 155
    def _check_hash_sum(self, file: str, hash_sum_file: str) -> bool:
        old_hash = ""
        with open(hash_sum_file, "r") as f:
156 157 158 159
            try:
                old_hash = f.readline().strip()
            except UnicodeDecodeError:
                _LOG.exception("Decoding a hash sum for java output caching failed")
160
        return self._hash_sum_for_file(file) == old_hash
161

162 163 164
    def _hash_sum_for_file(self, file: str) -> str:
        with open(file, "r") as f:
            return hashlib.sha256(f.read().encode()).hexdigest()
165

166 167 168 169 170
TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(JavaExecTest)


class JavaInfiniteLoopTest(BasicSyntaxTest):

171 172
    FILE_ENDINGS = [".inf.java", ".inf.mj"]
    OUTPUT_FILE_ENDING = ".out"
173 174 175 176
    MODE = TestMode.compile_firm

    def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str):
        super().__init__(env, type, file, preprocessed_file)
177 178
        self._output_file = self.file + self.OUTPUT_FILE_ENDING
        self._has_output_file = path.exists(self._output_file)
179 180 181 182 183 184 185 186

    def run(self) -> BasicTestResult:
        base_filename = path.basename(self.file).split(".")[0]
        tmp_dir = self.env.create_pid_local_tmpdir()
        shutil.copy(self.preprocessed_file, path.join(tmp_dir, base_filename + ".java"))
        cwd = os.getcwd()
        os.chdir(tmp_dir)
        timeout = 1
187 188 189 190
        err = None
        out = None
        rtcode = None
        test_result = ExtensibleTestResult(self)
191
        try:
192 193 194
            out, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java")
            if rtcode != 0:
                test_result.incorrect_msg = "file can't be compiled"
195
                test_result.set_error_code(rtcode)
196 197 198 199 200
                test_result.add_long_text("Error output", err.decode())
                test_result.add_long_text("Output", out.decode())
                test_result.add_file("Source file", self.preprocessed_file)
                os.chdir(cwd)
                return test_result
201
            out, err, rtcode = self.env.run_command("./a.out", timeout=timeout)
202 203 204 205 206 207 208 209
            if rtcode != 0:
                test_result.incorrect_msg = "binary can't be run, non zero error code"
                test_result.set_error_code(rtcode)
                test_result.add_long_text("Error output", err.decode())
                test_result.add_long_text("Output", out.decode())
                test_result.add_file("Source file", self.preprocessed_file)
                os.chdir(cwd)
                return test_result
210 211
        except SigKill as sig:
            if sig.retcode == signal.SIGXCPU:
Johannes Bechberger's avatar
Johannes Bechberger committed
212
                out, _, _, _ = self.env.run_command_till_timeout("./" + base_filename, timeout=1)
213
                test_result.add_long_text("Output", out.decode())
214 215 216 217 218
                if self._has_output_file:
                    out = out.decode().strip()
                    exp_out = ""
                    with open(self._output_file, "r") as f:
                        exp_out = f.read().strip()
219
                    test_result.add_long_text("Expected output start", exp_out)
220
                    if not out.startswith(exp_out):
221 222 223 224 225 226 227 228 229
                        test_result.incorrect_msg = "incorrect output start"
                        test_result.has_succeeded = False
                    test_result.add_file("Source file", self.preprocessed_file)
                    os.chdir(cwd)
                    return test_result
                else:
                    test_result.add_file("Source file", self.preprocessed_file)
                    os.chdir(cwd)
                    return test_result
230
            else:
231
                test_result.incorrect_msg = "binary can't be run: " + sig.name
232
                test_result.set_error_code(sig.retcode)
233 234 235
                test_result.add_file("Source file", self.preprocessed_file)
                os.chdir(cwd)
                return test_result
236 237 238 239
        except:
            os.chdir(cwd)
            raise
        os.chdir(cwd)
240 241 242 243 244 245
        test_result.incorrect_msg = "run shorter than one second"
        test_result.add_long_text("Output", out)
        test_result.has_succeeded = False
        test_result.add_file("Source file", self.preprocessed_file)
        os.chdir(cwd)
        return test_result
246 247 248


TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(JavaInfiniteLoopTest)