exec_tests.py 18.6 KB
Newer Older
1
import hashlib
2
3
4
import logging
import os
import shutil
5
import signal
6
from os import path
7
8
9
10

import re
from typing import Optional

11
12
import errno

13
14
from mjtest.environment import TestMode, Environment
from mjtest.test.syntax_tests import BasicSyntaxTest
15
from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult
16
from mjtest.util.shell import SigKill
17
from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict
18
19
20
21
22
23
24
25

_LOG = logging.getLogger("exec_tests")

class JavaExecTest(BasicSyntaxTest):
    """
    The MiniJava compiler should behave the same as javac
    """

26
    FILE_ENDINGS = [".java", ".mj"]
27
    INVALID_FILE_ENDINGS = [".inf.java", ".inf.mj", ".input.mj", ".input.java", ".invalid.java", ".invalid.mj"]
28
    OUTPUT_FILE_ENDING = ".out"
29
    MODE = TestMode.compile_firm
30
    INPUT_FILE_REGEXP = r'(\.[0-9]+)\.input(c?)$'
31

32
33
    def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str):
        super().__init__(env, type, file, preprocessed_file, log_file_mode)
34
35
36
        self._has_input_file = bool(re.search(self.INPUT_FILE_REGEXP, file))
        self._input_file = file if self._has_input_file else None
        self._has_character_input = file.endswith("c") if self._has_input_file else False
37
        self._expected_output_file = file + self.OUTPUT_FILE_ENDING
38
39
40
41
        prev_out_dir = path.join(path.dirname(file), ".java_output")
        if not path.exists(prev_out_dir):
            os.mkdir(prev_out_dir)
        self._prev_out_file = path.join(prev_out_dir, path.basename(self._expected_output_file))
42
        self._prev_out_hash_file = self._prev_out_file + "_hash"
43
        self._has_expected_output_file = path.exists(self._expected_output_file)
44
        if not self._has_expected_output_file:
45
46
            if path.exists(self._prev_out_file) and path.exists(self._prev_out_hash_file) \
                    and self._check_hash_sum(self.preprocessed_file, self._prev_out_hash_file):
47
48
49
                self._has_expected_output_file = True
                self._expected_output_file = self._prev_out_file
                _LOG.info("Reuse old java output file \"{}\"".format(path.relpath(self._prev_out_file)))
50
        self._should_succeed = True
51
        self._only_compile = self.MODE.endswith("-only")
52
53

    def run(self) -> BasicDiffTestResult:
54
55
        is_big_testcase = "big" in self.file
        timeout = self.env.big_timeout if is_big_testcase else self.env.timeout
56
        base_filename = path.basename(self.file).split(".")[0]
Johannes Bechberger's avatar
Johannes Bechberger committed
57
        tmp_dir = self.env.create_pid_local_tmpdir()
58
59
60
61
        shutil.copy(self.preprocessed_file, path.join(tmp_dir, base_filename + ".java"))
        cwd = os.getcwd()
        os.chdir(tmp_dir)
        exp_out = None
62
        input_str = None
63
        #print(base_filename, get_main_class_name(base_filename + ".java"))
64
65
66

        test_result = ExtensibleTestResult(self)

67
68
69
70
71
72
73
74
75
76
77
78
79
80
        if self._has_input_file:
            if self._has_character_input:
                with open(self._input_file, "r") as f:
                    input_str = f.read()
            else:
                with open(self._input_file, "r") as f:
                    chars = bytearray('ascii', 'ignore')  # type: bytearray
                    for line in f.readlines():
                        for part in line.split(" "):
                            part = part.strip()
                            if len(part) > 0:
                                chars.append(int(part))
                input_str = chars.decode()

81
        if not self._has_expected_output_file and not self._only_compile:
82
            _, err, javac_rtcode = \
83
                self.env.run_command("javac", base_filename + ".java", timeout=timeout)
84
85
            if javac_rtcode != 0:
                _LOG.error("File \"{}\" isn't valid Java".format(self.preprocessed_file))
86
                test_result.incorrect_msg = "invalid java code, but output file missing"
87
                test_result.set_error_code(javac_rtcode)
88
89
                test_result.add_long_text("Javac error message", err.decode())
                test_result.add_file("Source file", self.preprocessed_file)
90
                os.chdir(cwd)
91
                return test_result
92
            main_class = get_main_class_name(base_filename + ".java")
93
94
95
            if not main_class:
                _LOG.debug("Can't find a main class, using the file name instead")
                main_class = base_filename
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
            if not self._only_compile:
                exp_out, err, java_rtcode = \
                    self.env.run_command("java", get_main_class_name(base_filename + ".java"), timeout=timeout, input_str=input_str)
                if javac_rtcode != 0:
                    test_result.add_long_text("Java output: ", exp_out.decode())
                    test_result.incorrect_msg = "java runtime error"
                    test_result.set_error_code(java_rtcode)
                    test_result.add_long_text("Java error message", err.decode())
                    test_result.add_file("Source file", self.preprocessed_file)
                    if self._input_file:
                        test_result.add_file("Input file", self.file)
                    os.chdir(cwd)
                    return test_result
                exp_out = exp_out.decode()

                with open(self._prev_out_file, "w") as f:
                    f.write(exp_out)
                    f.flush()
                with open(self._prev_out_hash_file, "w") as f:
                    f.write(self._hash_sum_for_file(base_filename + ".java"))
                    f.flush()
        if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE \
                and not self._only_compile:
119
120
            with open(self._expected_output_file, "r") as f:
                exp_out = f.read()
121

122
        try:
123
124
            out, err, rtcode = None, None, None
            try:
125
                out, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java", timeout=-1)
126
127
                if rtcode != 0:
                    test_result.incorrect_msg = "file can't be compiled"
128
                    test_result.set_error_code(rtcode)
129
130
131
132
133
134
135
                    test_result.add_long_text("Error output", err.decode())
                    test_result.add_long_text("Output", out.decode())
                    test_result.add_file("Source file", self.preprocessed_file)
                    os.chdir(cwd)
                    return test_result
            except SigKill as sig:
                test_result.incorrect_msg = "file can't be compiled: " + sig.name
136
                test_result.set_error_code(sig.retcode)
137
138
139
140
141
142
                test_result.add_file("Source file", self.preprocessed_file)
                os.chdir(cwd)
                return test_result
            except:
                os.chdir(cwd)
                raise
143
144
145
146
147
148
            if self._only_compile:
                if not os.path.exists("a.out"):
                    test_result.incorrect_msg = "binary cannot be found"
                    test_result.set_error_code(1)
                    test_result.add_file("Source file", self.preprocessed_file)
                return test_result
149
            try:
150
                out, err, rtcode = self.env.run_command("./a.out", timeout=timeout, input_str=input_str)
151
152
153
154
155
156
                if rtcode != 0:
                    test_result.incorrect_msg = "binary can't be run, non zero error code"
                    test_result.set_error_code(rtcode)
                    test_result.add_long_text("Error output", err.decode())
                    test_result.add_long_text("Output", out.decode())
                    test_result.add_file("Source file", self.preprocessed_file)
157
158
                    if self._input_file:
                        test_result.add_file("Input file", self.file)
159
160
                    os.chdir(cwd)
                    return test_result
161
            except SigKill as sig:
162
163
                test_result.incorrect_msg = "binary can't be run: " + sig.name.strip()
                test_result.set_error_code(sig.retcode)
164
                test_result.add_file("Source file", self.preprocessed_file)
165
166
                if self._input_file:
                    test_result.add_file("Input file", self.file)
167
168
169
170
171
                os.chdir(cwd)
                return test_result
            except:
                os.chdir(cwd)
                raise
172
            out = out.decode()
173
            if self.type == self.MODE and self.env.mode == self.MODE:
174
175
                if exp_out.strip() != out.strip():
                    test_result.incorrect_msg = "incorrect output"
176
                    test_result.has_succeeded = False
177
178
179
                    test_result.add_short_text("Expected output file", self._expected_output_file)
                    test_result.add_long_text("Expected output", exp_out)
                    test_result.add_long_text("Actual output", out)
180
                    test_result.add_diff("Output diff [expected <-> actual]", exp_out, out, with_line_numbers=True)
181
                    test_result.add_file("Source file", self.preprocessed_file)
182
183
                    if self._input_file:
                        test_result.add_file("Input file", self.file)
Johannes Bechberger's avatar
Fix #72    
Johannes Bechberger committed
184
185
                else:
                    test_result.add_long_text("Output", out)
186
187
                os.chdir(cwd)
                return test_result
Johannes Bechberger's avatar
Johannes Bechberger committed
188
            return BasicTestResult(self, rtcode, out, err.decode())
189
190
        except SigKill as sig:
            os.chdir(cwd)
191
            assert False
192
        except BaseException:
193
            os.chdir(cwd)
Johannes Bechberger's avatar
Johannes Bechberger committed
194
            raise
195

196
197
198
    def _check_hash_sum(self, file: str, hash_sum_file: str) -> bool:
        old_hash = ""
        with open(hash_sum_file, "r") as f:
199
200
201
202
            try:
                old_hash = f.readline().strip()
            except UnicodeDecodeError:
                _LOG.exception("Decoding a hash sum for java output caching failed")
203
        return self._hash_sum_for_file(file) == old_hash
204

205
    @classmethod
206
207
208
    def _hash_sum_for_file(self, file: str) -> str:
        with open(file, "r") as f:
            return hashlib.sha256(f.read().encode()).hexdigest()
209

210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
    @classmethod
    def _test_case_file_for_input_file(cls, input_file: str) -> Optional[str]:
        base = re.sub(cls.INPUT_FILE_REGEXP, "", input_file)
        if path.exists(base + ".input.mj"):
            return base + ".input.mj"
        if path.exists(base + ".input.java"):
            return base + ".input.java"
        return None

    @staticmethod
    def is_file_ending_valid(cls, file: str) -> bool:
        if re.search(cls.INPUT_FILE_REGEXP, file) and ".java_output" not in file and "precprocessor" not in file:
            if not JavaExecTest._test_case_file_for_input_file(file):
                _LOG.error("Skip {} that hasn't a corresponding test case file".format(file))
            return True
        return super().is_file_ending_valid(cls, file)

    @staticmethod
    def _get_test_case_file(cls, file: str) -> Optional[str]:
        if re.search(JavaExecTest.INPUT_FILE_REGEXP, file):
            return JavaExecTest._test_case_file_for_input_file(file)
        return file

233
234
TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(JavaExecTest)

Johannes Bechberger's avatar
Johannes Bechberger committed
235
236
237
238
class JavaCompileExecTest(JavaExecTest):
    MODE = TestMode.compile

TestCase.TEST_CASE_CLASSES[TestMode.compile].append(JavaCompileExecTest)
239

240
241
242
243
244
245
246
247
248
249
class JavaCompileOnlyTest(JavaExecTest):
    MODE = TestMode.compile_only

TestCase.TEST_CASE_CLASSES[TestMode.compile_only].append(JavaCompileOnlyTest)

class JavaCompileFirmOnlyTest(JavaExecTest):
    MODE = TestMode.compile_firm_only

TestCase.TEST_CASE_CLASSES[TestMode.compile_firm_only].append(JavaCompileFirmOnlyTest)

250
251
class JavaInfiniteLoopTest(BasicSyntaxTest):

252
253
    FILE_ENDINGS = [".inf.java", ".inf.mj"]
    OUTPUT_FILE_ENDING = ".out"
254
255
    MODE = TestMode.compile_firm

256
257
    def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str):
        super().__init__(env, type, file, preprocessed_file, log_file_mode)
258
259
        self._output_file = self.file + self.OUTPUT_FILE_ENDING
        self._has_output_file = path.exists(self._output_file)
260
261
262
263
264
265
266
267

    def run(self) -> BasicTestResult:
        base_filename = path.basename(self.file).split(".")[0]
        tmp_dir = self.env.create_pid_local_tmpdir()
        shutil.copy(self.preprocessed_file, path.join(tmp_dir, base_filename + ".java"))
        cwd = os.getcwd()
        os.chdir(tmp_dir)
        timeout = 1
268
269
270
271
        err = None
        out = None
        rtcode = None
        test_result = ExtensibleTestResult(self)
272
        try:
273
274
275
            out, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java")
            if rtcode != 0:
                test_result.incorrect_msg = "file can't be compiled"
276
                test_result.set_error_code(rtcode)
277
278
279
280
281
                test_result.add_long_text("Error output", err.decode())
                test_result.add_long_text("Output", out.decode())
                test_result.add_file("Source file", self.preprocessed_file)
                os.chdir(cwd)
                return test_result
Johannes Bechberger's avatar
Johannes Bechberger committed
282
            out, err, rtcode = self.env.run_command("./a.out", timeout=timeout)
283
284
285
286
287
288
289
290
            if rtcode != 0:
                test_result.incorrect_msg = "binary can't be run, non zero error code"
                test_result.set_error_code(rtcode)
                test_result.add_long_text("Error output", err.decode())
                test_result.add_long_text("Output", out.decode())
                test_result.add_file("Source file", self.preprocessed_file)
                os.chdir(cwd)
                return test_result
291
292
        except SigKill as sig:
            if sig.retcode == signal.SIGXCPU:
293
                out, _, _, _ = self.env.run_command_till_timeout("./a.out", timeout=1)
294
                test_result.add_long_text("Output", out.decode())
295
296
297
298
299
                if self._has_output_file:
                    out = out.decode().strip()
                    exp_out = ""
                    with open(self._output_file, "r") as f:
                        exp_out = f.read().strip()
300
                    test_result.add_long_text("Expected output start", exp_out)
301
                    if not out.startswith(exp_out):
302
303
304
305
306
307
308
309
310
                        test_result.incorrect_msg = "incorrect output start"
                        test_result.has_succeeded = False
                    test_result.add_file("Source file", self.preprocessed_file)
                    os.chdir(cwd)
                    return test_result
                else:
                    test_result.add_file("Source file", self.preprocessed_file)
                    os.chdir(cwd)
                    return test_result
311
            else:
312
                test_result.incorrect_msg = "binary can't be run: " + sig.name
313
                test_result.set_error_code(sig.retcode)
314
315
316
                test_result.add_file("Source file", self.preprocessed_file)
                os.chdir(cwd)
                return test_result
317
        except BaseException as e:
318
            os.chdir(cwd)
319
            print(e)
320
321
            raise
        os.chdir(cwd)
322
323
324
325
326
327
        test_result.incorrect_msg = "run shorter than one second"
        test_result.add_long_text("Output", out)
        test_result.has_succeeded = False
        test_result.add_file("Source file", self.preprocessed_file)
        os.chdir(cwd)
        return test_result
328
329


330
TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(JavaInfiniteLoopTest)
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413


class InvalidJavaExecTest(BasicSyntaxTest):
    """
    The compiled binary should fail to execute properly
    """

    FILE_ENDINGS = [".invalid.java", ".invalid.mj"]
    MODE = TestMode.compile_firm

    def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str):
        super().__init__(env, type, file, preprocessed_file, log_file_mode)
        self._should_succeed = False

    def run(self) -> BasicDiffTestResult:
        is_big_testcase = "big" in self.file
        timeout = self.env.big_timeout if is_big_testcase else self.env.timeout
        base_filename = path.basename(self.file).split(".")[0]
        tmp_dir = self.env.create_pid_local_tmpdir()
        shutil.copy(self.preprocessed_file, path.join(tmp_dir, base_filename + ".java"))
        cwd = os.getcwd()
        os.chdir(tmp_dir)
        #print(base_filename, get_main_class_name(base_filename + ".java"))

        test_result = ExtensibleTestResult(self)
        test_result.require_error_string_in_error_case = False
        try:
            out, err, rtcode = None, None, None
            try:
                out, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java", timeout=-1)
                if rtcode != 0:
                    test_result.incorrect_msg = "file can't be compiled"
                    test_result.set_error_code(-rtcode)
                    test_result.add_long_text("Error output", err.decode())
                    test_result.add_long_text("Output", out.decode())
                    test_result.add_file("Source file", self.preprocessed_file)
                    os.chdir(cwd)
                    return test_result
            except SigKill as sig:
                test_result.incorrect_msg = "file can't be compiled: " + sig.name
                test_result.set_error_code(sig.retcode)
                test_result.add_file("Source file", self.preprocessed_file)
                os.chdir(cwd)
                return test_result
            except:
                os.chdir(cwd)
                raise
            try:
                out, err, rtcode = self.env.run_command("./a.out", timeout=timeout)
                if rtcode != 0:
                    test_result.incorrect_msg = "binary can't be run, non zero error code"
                    test_result.set_error_code(rtcode)
                    test_result.add_long_text("Error output", err.decode())
                    test_result.add_long_text("Output", out.decode())
                    test_result.add_file("Source file", self.preprocessed_file)
                    os.chdir(cwd)
                    return test_result
            except SigKill as sig:
                test_result.incorrect_msg = "binary can't be run: " + sig.name.strip()
                test_result.set_error_code(sig.retcode)
                test_result.add_file("Source file", self.preprocessed_file)
                os.chdir(cwd)
                return test_result
            except:
                os.chdir(cwd)
                raise
            test_result = BasicTestResult(self, rtcode, out.decode(), err.decode())
            test_result.require_error_string_in_error_case = False
            return test_result
        except SigKill as sig:
            os.chdir(cwd)
            assert False
        except BaseException:
            os.chdir(cwd)
            raise


TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(InvalidJavaExecTest)

class InvalidJavaCompileExecTest(InvalidJavaExecTest):
    MODE = TestMode.compile

TestCase.TEST_CASE_CLASSES[TestMode.compile].append(InvalidJavaExecTest)