exec_tests.py 14 KB
Newer Older
1
import hashlib
2
3
4
import logging
import os
import shutil
5
import signal
6
from os import path
7
8
9
10

import re
from typing import Optional

11
12
import errno

13
14
from mjtest.environment import TestMode, Environment
from mjtest.test.syntax_tests import BasicSyntaxTest
15
from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult
16
from mjtest.util.shell import SigKill
17
from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict
18
19
20
21
22
23
24
25

_LOG = logging.getLogger("exec_tests")

class JavaExecTest(BasicSyntaxTest):
    """
    The MiniJava compiler should behave the same as javac
    """

26
    FILE_ENDINGS = [".java", ".mj"]
27
    INVALID_FILE_ENDINGS = [".inf.java", ".inf.mj", ".input.mj", ".input.java"]
28
    OUTPUT_FILE_ENDING = ".out"
29
    MODE = TestMode.compile_firm
30
    INPUT_FILE_REGEXP = r'(\.[0-9]+)\.input(c?)$'
31

32
33
    def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str):
        super().__init__(env, type, file, preprocessed_file, log_file_mode)
34
35
36
        self._has_input_file = bool(re.search(self.INPUT_FILE_REGEXP, file))
        self._input_file = file if self._has_input_file else None
        self._has_character_input = file.endswith("c") if self._has_input_file else False
37
        self._expected_output_file = file + self.OUTPUT_FILE_ENDING
38
39
40
41
        prev_out_dir = path.join(path.dirname(file), ".java_output")
        if not path.exists(prev_out_dir):
            os.mkdir(prev_out_dir)
        self._prev_out_file = path.join(prev_out_dir, path.basename(self._expected_output_file))
42
        self._prev_out_hash_file = self._prev_out_file + "_hash"
43
        self._has_expected_output_file = path.exists(self._expected_output_file)
44
        if not self._has_expected_output_file:
45
46
            if path.exists(self._prev_out_file) and path.exists(self._prev_out_hash_file) \
                    and self._check_hash_sum(self.preprocessed_file, self._prev_out_hash_file):
47
48
49
                self._has_expected_output_file = True
                self._expected_output_file = self._prev_out_file
                _LOG.info("Reuse old java output file \"{}\"".format(path.relpath(self._prev_out_file)))
50
51
52
        self._should_succeed = True

    def run(self) -> BasicDiffTestResult:
53
54
        is_big_testcase = "big" in self.file
        timeout = self.env.big_timeout if is_big_testcase else self.env.timeout
55
        base_filename = path.basename(self.file).split(".")[0]
Johannes Bechberger's avatar
Johannes Bechberger committed
56
        tmp_dir = self.env.create_pid_local_tmpdir()
57
58
59
60
        shutil.copy(self.preprocessed_file, path.join(tmp_dir, base_filename + ".java"))
        cwd = os.getcwd()
        os.chdir(tmp_dir)
        exp_out = None
61
        input_str = None
62
        #print(base_filename, get_main_class_name(base_filename + ".java"))
63
64
65

        test_result = ExtensibleTestResult(self)

66
67
68
69
70
71
72
73
74
75
76
77
78
79
        if self._has_input_file:
            if self._has_character_input:
                with open(self._input_file, "r") as f:
                    input_str = f.read()
            else:
                with open(self._input_file, "r") as f:
                    chars = bytearray('ascii', 'ignore')  # type: bytearray
                    for line in f.readlines():
                        for part in line.split(" "):
                            part = part.strip()
                            if len(part) > 0:
                                chars.append(int(part))
                input_str = chars.decode()

80
        if not self._has_expected_output_file:
81
            _, err, javac_rtcode = \
82
                self.env.run_command("javac", base_filename + ".java", timeout=timeout)
83
84
            if javac_rtcode != 0:
                _LOG.error("File \"{}\" isn't valid Java".format(self.preprocessed_file))
85
                test_result.incorrect_msg = "invalid java code, but output file missing"
86
                test_result.set_error_code(javac_rtcode)
87
88
                test_result.add_long_text("Javac error message", err.decode())
                test_result.add_file("Source file", self.preprocessed_file)
89
                os.chdir(cwd)
90
                return test_result
91
            main_class = get_main_class_name(base_filename + ".java")
92
93
94
            if not main_class:
                _LOG.debug("Can't find a main class, using the file name instead")
                main_class = base_filename
95
            exp_out, err, java_rtcode = \
96
                self.env.run_command("java", get_main_class_name(base_filename + ".java"), timeout=timeout, input_str=input_str)
97
            if javac_rtcode != 0:
98
                test_result.add_long_text("Java output: ", exp_out.decode())
99
                test_result.incorrect_msg = "java runtime error"
100
                test_result.set_error_code(java_rtcode)
101
102
                test_result.add_long_text("Java error message", err.decode())
                test_result.add_file("Source file", self.preprocessed_file)
103
104
                if self._input_file:
                    test_result.add_file("Input file", self.file)
105
106
                os.chdir(cwd)
                return test_result
107
            exp_out = exp_out.decode()
108

109
110
111
            with open(self._prev_out_file, "w") as f:
                f.write(exp_out)
                f.flush()
112
113
114
            with open(self._prev_out_hash_file, "w") as f:
                f.write(self._hash_sum_for_file(base_filename + ".java"))
                f.flush()
115
116
117
        if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE:
            with open(self._expected_output_file, "r") as f:
                exp_out = f.read()
118

119
        try:
120
121
            out, err, rtcode = None, None, None
            try:
122
                out, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java", timeout=timeout)
123
124
                if rtcode != 0:
                    test_result.incorrect_msg = "file can't be compiled"
125
                    test_result.set_error_code(rtcode)
126
127
128
129
130
131
132
                    test_result.add_long_text("Error output", err.decode())
                    test_result.add_long_text("Output", out.decode())
                    test_result.add_file("Source file", self.preprocessed_file)
                    os.chdir(cwd)
                    return test_result
            except SigKill as sig:
                test_result.incorrect_msg = "file can't be compiled: " + sig.name
133
                test_result.set_error_code(sig.retcode)
134
135
136
137
138
139
140
                test_result.add_file("Source file", self.preprocessed_file)
                os.chdir(cwd)
                return test_result
            except:
                os.chdir(cwd)
                raise
            try:
141
                out, err, rtcode = self.env.run_command("./a.out", timeout=timeout, input_str=input_str)
142
143
144
145
146
147
                if rtcode != 0:
                    test_result.incorrect_msg = "binary can't be run, non zero error code"
                    test_result.set_error_code(rtcode)
                    test_result.add_long_text("Error output", err.decode())
                    test_result.add_long_text("Output", out.decode())
                    test_result.add_file("Source file", self.preprocessed_file)
148
149
                    if self._input_file:
                        test_result.add_file("Input file", self.file)
150
151
                    os.chdir(cwd)
                    return test_result
152
            except SigKill as sig:
153
154
                test_result.incorrect_msg = "binary can't be run: " + sig.name.strip()
                test_result.set_error_code(sig.retcode)
155
                test_result.add_file("Source file", self.preprocessed_file)
156
157
                if self._input_file:
                    test_result.add_file("Input file", self.file)
158
159
160
161
162
                os.chdir(cwd)
                return test_result
            except:
                os.chdir(cwd)
                raise
163
            out = out.decode()
164
            if self.type == self.MODE and self.env.mode == self.MODE:
165
                test_result.add_long_text("Output", out)
166
167
                if exp_out.strip() != out.strip():
                    test_result.incorrect_msg = "incorrect output"
168
                    test_result.has_succeeded = False
169
170
171
                    test_result.add_short_text("Expected output file", self._expected_output_file)
                    test_result.add_long_text("Expected output", exp_out)
                    test_result.add_long_text("Actual output", out)
172
                    test_result.add_diff("Output diff [expected <-> actual]", exp_out, out, with_line_numbers=True)
173
                    test_result.add_file("Source file", self.preprocessed_file)
174
175
                    if self._input_file:
                        test_result.add_file("Input file", self.file)
176
177
                os.chdir(cwd)
                return test_result
Johannes Bechberger's avatar
Johannes Bechberger committed
178
            return BasicTestResult(self, rtcode, out, err.decode())
179
180
        except SigKill as sig:
            os.chdir(cwd)
181
            assert False
182
        except BaseException:
183
            os.chdir(cwd)
184
            raise 
185

186
187
188
    def _check_hash_sum(self, file: str, hash_sum_file: str) -> bool:
        old_hash = ""
        with open(hash_sum_file, "r") as f:
189
190
191
192
            try:
                old_hash = f.readline().strip()
            except UnicodeDecodeError:
                _LOG.exception("Decoding a hash sum for java output caching failed")
193
        return self._hash_sum_for_file(file) == old_hash
194

195
    @classmethod
196
197
198
    def _hash_sum_for_file(self, file: str) -> str:
        with open(file, "r") as f:
            return hashlib.sha256(f.read().encode()).hexdigest()
199

200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
    @classmethod
    def _test_case_file_for_input_file(cls, input_file: str) -> Optional[str]:
        base = re.sub(cls.INPUT_FILE_REGEXP, "", input_file)
        if path.exists(base + ".input.mj"):
            return base + ".input.mj"
        if path.exists(base + ".input.java"):
            return base + ".input.java"
        return None

    @staticmethod
    def is_file_ending_valid(cls, file: str) -> bool:
        if re.search(cls.INPUT_FILE_REGEXP, file) and ".java_output" not in file and "precprocessor" not in file:
            if not JavaExecTest._test_case_file_for_input_file(file):
                _LOG.error("Skip {} that hasn't a corresponding test case file".format(file))
            return True
        return super().is_file_ending_valid(cls, file)

    @staticmethod
    def _get_test_case_file(cls, file: str) -> Optional[str]:
        if re.search(JavaExecTest.INPUT_FILE_REGEXP, file):
            return JavaExecTest._test_case_file_for_input_file(file)
        return file

223
224
225
226
227
TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(JavaExecTest)


class JavaInfiniteLoopTest(BasicSyntaxTest):

228
229
    FILE_ENDINGS = [".inf.java", ".inf.mj"]
    OUTPUT_FILE_ENDING = ".out"
230
231
    MODE = TestMode.compile_firm

232
233
    def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str):
        super().__init__(env, type, file, preprocessed_file, log_file_mode)
234
235
        self._output_file = self.file + self.OUTPUT_FILE_ENDING
        self._has_output_file = path.exists(self._output_file)
236
237
238
239
240
241
242
243

    def run(self) -> BasicTestResult:
        base_filename = path.basename(self.file).split(".")[0]
        tmp_dir = self.env.create_pid_local_tmpdir()
        shutil.copy(self.preprocessed_file, path.join(tmp_dir, base_filename + ".java"))
        cwd = os.getcwd()
        os.chdir(tmp_dir)
        timeout = 1
244
245
246
247
        err = None
        out = None
        rtcode = None
        test_result = ExtensibleTestResult(self)
248
        try:
249
250
251
            out, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java")
            if rtcode != 0:
                test_result.incorrect_msg = "file can't be compiled"
252
                test_result.set_error_code(rtcode)
253
254
255
256
257
                test_result.add_long_text("Error output", err.decode())
                test_result.add_long_text("Output", out.decode())
                test_result.add_file("Source file", self.preprocessed_file)
                os.chdir(cwd)
                return test_result
Johannes Bechberger's avatar
Johannes Bechberger committed
258
            out, err, rtcode = self.env.run_command("./a.out", timeout=timeout)
259
260
261
262
263
264
265
266
            if rtcode != 0:
                test_result.incorrect_msg = "binary can't be run, non zero error code"
                test_result.set_error_code(rtcode)
                test_result.add_long_text("Error output", err.decode())
                test_result.add_long_text("Output", out.decode())
                test_result.add_file("Source file", self.preprocessed_file)
                os.chdir(cwd)
                return test_result
267
268
        except SigKill as sig:
            if sig.retcode == signal.SIGXCPU:
269
                out, _, _, _ = self.env.run_command_till_timeout("./a.out", timeout=1)
270
                test_result.add_long_text("Output", out.decode())
271
272
273
274
275
                if self._has_output_file:
                    out = out.decode().strip()
                    exp_out = ""
                    with open(self._output_file, "r") as f:
                        exp_out = f.read().strip()
276
                    test_result.add_long_text("Expected output start", exp_out)
277
                    if not out.startswith(exp_out):
278
279
280
281
282
283
284
285
286
                        test_result.incorrect_msg = "incorrect output start"
                        test_result.has_succeeded = False
                    test_result.add_file("Source file", self.preprocessed_file)
                    os.chdir(cwd)
                    return test_result
                else:
                    test_result.add_file("Source file", self.preprocessed_file)
                    os.chdir(cwd)
                    return test_result
287
            else:
288
                test_result.incorrect_msg = "sdfbinary can't be run: " + sig.name
289
                test_result.set_error_code(sig.retcode)
290
291
292
                test_result.add_file("Source file", self.preprocessed_file)
                os.chdir(cwd)
                return test_result
293
        except BaseException as e:
294
            os.chdir(cwd)
295
            print(e)
296
297
            raise
        os.chdir(cwd)
298
299
300
301
302
303
        test_result.incorrect_msg = "run shorter than one second"
        test_result.add_long_text("Output", out)
        test_result.has_succeeded = False
        test_result.add_file("Source file", self.preprocessed_file)
        os.chdir(cwd)
        return test_result
304
305


306
TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(JavaInfiniteLoopTest)