exec_tests.py 18.6 KB
Newer Older
1
import hashlib
2
3
4
import logging
import os
import shutil
5
import signal
6
from os import path
7
8
9
10

import re
from typing import Optional

11
12
import errno

13
14
from mjtest.environment import TestMode, Environment
from mjtest.test.syntax_tests import BasicSyntaxTest
15
from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult
16
from mjtest.util.shell import SigKill
17
from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict, decode
18
19
20
21
22
23
24
25

_LOG = logging.getLogger("exec_tests")

class JavaExecTest(BasicSyntaxTest):
    """
    The MiniJava compiler should behave the same as javac
    """

26
    FILE_ENDINGS = [".java", ".mj"]
27
    INVALID_FILE_ENDINGS = [".inf.java", ".inf.mj", ".input.mj", ".input.java", ".invalid.java", ".invalid.mj"]
28
    OUTPUT_FILE_ENDING = ".out"
29
    MODE = TestMode.compile_firm
30
    INPUT_FILE_REGEXP = r'(\.[0-9]+)\.input(c?)$'
31

32
33
    def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str):
        super().__init__(env, type, file, preprocessed_file, log_file_mode)
34
35
36
        self._has_input_file = bool(re.search(self.INPUT_FILE_REGEXP, file))
        self._input_file = file if self._has_input_file else None
        self._has_character_input = file.endswith("c") if self._has_input_file else False
37
        self._expected_output_file = file + self.OUTPUT_FILE_ENDING
38
39
40
41
        prev_out_dir = path.join(path.dirname(file), ".java_output")
        if not path.exists(prev_out_dir):
            os.mkdir(prev_out_dir)
        self._prev_out_file = path.join(prev_out_dir, path.basename(self._expected_output_file))
42
        self._prev_out_hash_file = self._prev_out_file + "_hash"
43
        self._has_expected_output_file = path.exists(self._expected_output_file)
44
        if not self._has_expected_output_file:
45
46
            if path.exists(self._prev_out_file) and path.exists(self._prev_out_hash_file) \
                    and self._check_hash_sum(self.preprocessed_file, self._prev_out_hash_file):
47
48
49
                self._has_expected_output_file = True
                self._expected_output_file = self._prev_out_file
                _LOG.info("Reuse old java output file \"{}\"".format(path.relpath(self._prev_out_file)))
50
        self._should_succeed = True
51
        self._only_compile = self.MODE.endswith("-only")
52
53

    def run(self) -> BasicDiffTestResult:
54
55
        is_big_testcase = "big" in self.file
        timeout = self.env.big_timeout if is_big_testcase else self.env.timeout
56
        base_filename = path.basename(self.file).split(".")[0]
Johannes Bechberger's avatar
Johannes Bechberger committed
57
        tmp_dir = self.env.create_pid_local_tmpdir()
58
59
60
61
        shutil.copy(self.preprocessed_file, path.join(tmp_dir, base_filename + ".java"))
        cwd = os.getcwd()
        os.chdir(tmp_dir)
        exp_out = None
62
        input_str = None
63
        #print(base_filename, get_main_class_name(base_filename + ".java"))
64
65
66

        test_result = ExtensibleTestResult(self)

67
68
69
70
71
72
73
74
75
76
77
78
        if self._has_input_file:
            if self._has_character_input:
                with open(self._input_file, "r") as f:
                    input_str = f.read()
            else:
                with open(self._input_file, "r") as f:
                    chars = bytearray('ascii', 'ignore')  # type: bytearray
                    for line in f.readlines():
                        for part in line.split(" "):
                            part = part.strip()
                            if len(part) > 0:
                                chars.append(int(part))
79
                input_str = decode(chars)
80

81
        if not self._has_expected_output_file and not self._only_compile:
82
            _, err, javac_rtcode = \
83
                self.env.run_command("javac", base_filename + ".java", timeout=timeout)
84
85
            if javac_rtcode != 0:
                _LOG.error("File \"{}\" isn't valid Java".format(self.preprocessed_file))
86
                test_result.incorrect_msg = "invalid java code, but output file missing"
87
                test_result.set_error_code(javac_rtcode)
88
                test_result.add_long_text("Javac error message", decode(err))
89
                test_result.add_file("Source file", self.preprocessed_file)
90
                os.chdir(cwd)
91
                return test_result
92
            main_class = get_main_class_name(base_filename + ".java")
93
94
95
            if not main_class:
                _LOG.debug("Can't find a main class, using the file name instead")
                main_class = base_filename
96
97
98
99
            if not self._only_compile:
                exp_out, err, java_rtcode = \
                    self.env.run_command("java", get_main_class_name(base_filename + ".java"), timeout=timeout, input_str=input_str)
                if javac_rtcode != 0:
100
                    test_result.add_long_text("Java output: ", decode(exp_out))
101
102
                    test_result.incorrect_msg = "java runtime error"
                    test_result.set_error_code(java_rtcode)
103
                    test_result.add_long_text("Java error message", decode(err))
104
105
106
107
108
                    test_result.add_file("Source file", self.preprocessed_file)
                    if self._input_file:
                        test_result.add_file("Input file", self.file)
                    os.chdir(cwd)
                    return test_result
109
                exp_out = decode(exp_out)
110
111
112
113
114
115
116
117
118

                with open(self._prev_out_file, "w") as f:
                    f.write(exp_out)
                    f.flush()
                with open(self._prev_out_hash_file, "w") as f:
                    f.write(self._hash_sum_for_file(base_filename + ".java"))
                    f.flush()
        if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE \
                and not self._only_compile:
119
120
            with open(self._expected_output_file, "r") as f:
                exp_out = f.read()
121
        try:
122
123
            out, err, rtcode = None, None, None
            try:
124
                out, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java", timeout=-1)
125
126
                if rtcode != 0:
                    test_result.incorrect_msg = "file can't be compiled"
127
                    test_result.set_error_code(rtcode)
128
129
                    test_result.add_long_text("Error output", decode(err))
                    test_result.add_long_text("Output", decode(out))
130
131
132
133
134
                    test_result.add_file("Source file", self.preprocessed_file)
                    os.chdir(cwd)
                    return test_result
            except SigKill as sig:
                test_result.incorrect_msg = "file can't be compiled: " + sig.name
135
                test_result.set_error_code(sig.retcode)
136
137
138
139
140
141
                test_result.add_file("Source file", self.preprocessed_file)
                os.chdir(cwd)
                return test_result
            except:
                os.chdir(cwd)
                raise
142
143
144
145
146
147
            if self._only_compile:
                if not os.path.exists("a.out"):
                    test_result.incorrect_msg = "binary cannot be found"
                    test_result.set_error_code(1)
                    test_result.add_file("Source file", self.preprocessed_file)
                return test_result
148
            try:
149
                out, err, rtcode = self.env.run_command("./a.out", timeout=timeout, input_str=input_str)
150
151
152
                if rtcode != 0:
                    test_result.incorrect_msg = "binary can't be run, non zero error code"
                    test_result.set_error_code(rtcode)
153
154
                    test_result.add_long_text("Error output", decode(err))
                    test_result.add_long_text("Output", decode(out))
155
                    test_result.add_file("Source file", self.preprocessed_file)
156
157
                    if self._input_file:
                        test_result.add_file("Input file", self.file)
158
159
                    os.chdir(cwd)
                    return test_result
160
            except SigKill as sig:
161
162
                test_result.incorrect_msg = "binary can't be run: " + sig.name.strip()
                test_result.set_error_code(sig.retcode)
163
                test_result.add_file("Source file", self.preprocessed_file)
164
165
                if self._input_file:
                    test_result.add_file("Input file", self.file)
166
167
168
169
170
                os.chdir(cwd)
                return test_result
            except:
                os.chdir(cwd)
                raise
171
            out = decode(out)
172
            if self.type == self.MODE and self.env.mode == self.MODE:
173
174
                if exp_out.strip() != out.strip():
                    test_result.incorrect_msg = "incorrect output"
175
                    test_result.has_succeeded = False
176
177
178
                    test_result.add_short_text("Expected output file", self._expected_output_file)
                    test_result.add_long_text("Expected output", exp_out)
                    test_result.add_long_text("Actual output", out)
179
                    test_result.add_diff("Output diff [expected <-> actual]", exp_out, out, with_line_numbers=True)
180
                    test_result.add_file("Source file", self.preprocessed_file)
181
182
                    if self._input_file:
                        test_result.add_file("Input file", self.file)
Johannes Bechberger's avatar
Fix #72    
Johannes Bechberger committed
183
184
                else:
                    test_result.add_long_text("Output", out)
185
186
                os.chdir(cwd)
                return test_result
187
            return BasicTestResult(self, rtcode, out, decode(err))
188
189
        except SigKill as sig:
            os.chdir(cwd)
190
            assert False
191
        except BaseException:
192
            os.chdir(cwd)
Johannes Bechberger's avatar
Johannes Bechberger committed
193
            raise
194

195
196
197
    def _check_hash_sum(self, file: str, hash_sum_file: str) -> bool:
        old_hash = ""
        with open(hash_sum_file, "r") as f:
198
199
200
201
            try:
                old_hash = f.readline().strip()
            except UnicodeDecodeError:
                _LOG.exception("Decoding a hash sum for java output caching failed")
202
        return self._hash_sum_for_file(file) == old_hash
203

204
    @classmethod
205
206
207
    def _hash_sum_for_file(self, file: str) -> str:
        with open(file, "r") as f:
            return hashlib.sha256(f.read().encode()).hexdigest()
208

209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
    @classmethod
    def _test_case_file_for_input_file(cls, input_file: str) -> Optional[str]:
        base = re.sub(cls.INPUT_FILE_REGEXP, "", input_file)
        if path.exists(base + ".input.mj"):
            return base + ".input.mj"
        if path.exists(base + ".input.java"):
            return base + ".input.java"
        return None

    @staticmethod
    def is_file_ending_valid(cls, file: str) -> bool:
        if re.search(cls.INPUT_FILE_REGEXP, file) and ".java_output" not in file and "precprocessor" not in file:
            if not JavaExecTest._test_case_file_for_input_file(file):
                _LOG.error("Skip {} that hasn't a corresponding test case file".format(file))
            return True
        return super().is_file_ending_valid(cls, file)

    @staticmethod
    def _get_test_case_file(cls, file: str) -> Optional[str]:
        if re.search(JavaExecTest.INPUT_FILE_REGEXP, file):
            return JavaExecTest._test_case_file_for_input_file(file)
        return file

232
233
TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(JavaExecTest)

Johannes Bechberger's avatar
Johannes Bechberger committed
234
235
236
237
class JavaCompileExecTest(JavaExecTest):
    MODE = TestMode.compile

TestCase.TEST_CASE_CLASSES[TestMode.compile].append(JavaCompileExecTest)
238

239
240
241
242
243
244
245
246
247
248
class JavaCompileOnlyTest(JavaExecTest):
    MODE = TestMode.compile_only

TestCase.TEST_CASE_CLASSES[TestMode.compile_only].append(JavaCompileOnlyTest)

class JavaCompileFirmOnlyTest(JavaExecTest):
    MODE = TestMode.compile_firm_only

TestCase.TEST_CASE_CLASSES[TestMode.compile_firm_only].append(JavaCompileFirmOnlyTest)

249
250
class JavaInfiniteLoopTest(BasicSyntaxTest):

251
252
    FILE_ENDINGS = [".inf.java", ".inf.mj"]
    OUTPUT_FILE_ENDING = ".out"
253
254
    MODE = TestMode.compile_firm

255
256
    def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str):
        super().__init__(env, type, file, preprocessed_file, log_file_mode)
257
258
        self._output_file = self.file + self.OUTPUT_FILE_ENDING
        self._has_output_file = path.exists(self._output_file)
259
260
261
262
263
264
265
266

    def run(self) -> BasicTestResult:
        base_filename = path.basename(self.file).split(".")[0]
        tmp_dir = self.env.create_pid_local_tmpdir()
        shutil.copy(self.preprocessed_file, path.join(tmp_dir, base_filename + ".java"))
        cwd = os.getcwd()
        os.chdir(tmp_dir)
        timeout = 1
267
268
269
270
        err = None
        out = None
        rtcode = None
        test_result = ExtensibleTestResult(self)
271
        try:
272
273
274
            out, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java")
            if rtcode != 0:
                test_result.incorrect_msg = "file can't be compiled"
275
                test_result.set_error_code(rtcode)
276
277
                test_result.add_long_text("Error output", decode(err))
                test_result.add_long_text("Output", decode(out))
278
279
280
                test_result.add_file("Source file", self.preprocessed_file)
                os.chdir(cwd)
                return test_result
Johannes Bechberger's avatar
Johannes Bechberger committed
281
            out, err, rtcode = self.env.run_command("./a.out", timeout=timeout)
282
283
284
            if rtcode != 0:
                test_result.incorrect_msg = "binary can't be run, non zero error code"
                test_result.set_error_code(rtcode)
285
286
                test_result.add_long_text("Error output", decode(err))
                test_result.add_long_text("Output", decode(out))
287
288
289
                test_result.add_file("Source file", self.preprocessed_file)
                os.chdir(cwd)
                return test_result
290
291
        except SigKill as sig:
            if sig.retcode == signal.SIGXCPU:
292
                out, _, _, _ = self.env.run_command_till_timeout("./a.out", timeout=1)
293
                test_result.add_long_text("Output", decode(out))
294
                if self._has_output_file:
295
                    out = decode(out).strip()
296
297
298
                    exp_out = ""
                    with open(self._output_file, "r") as f:
                        exp_out = f.read().strip()
299
                    test_result.add_long_text("Expected output start", exp_out)
300
                    if not out.startswith(exp_out):
301
302
303
304
305
306
307
308
309
                        test_result.incorrect_msg = "incorrect output start"
                        test_result.has_succeeded = False
                    test_result.add_file("Source file", self.preprocessed_file)
                    os.chdir(cwd)
                    return test_result
                else:
                    test_result.add_file("Source file", self.preprocessed_file)
                    os.chdir(cwd)
                    return test_result
310
            else:
311
                test_result.incorrect_msg = "binary can't be run: " + sig.name
312
                test_result.set_error_code(sig.retcode)
313
314
315
                test_result.add_file("Source file", self.preprocessed_file)
                os.chdir(cwd)
                return test_result
316
        except BaseException as e:
317
            os.chdir(cwd)
318
            print(e)
319
320
            raise
        os.chdir(cwd)
321
322
323
324
325
326
        test_result.incorrect_msg = "run shorter than one second"
        test_result.add_long_text("Output", out)
        test_result.has_succeeded = False
        test_result.add_file("Source file", self.preprocessed_file)
        os.chdir(cwd)
        return test_result
327
328


329
TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(JavaInfiniteLoopTest)
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362


class InvalidJavaExecTest(BasicSyntaxTest):
    """
    The compiled binary should fail to execute properly
    """

    FILE_ENDINGS = [".invalid.java", ".invalid.mj"]
    MODE = TestMode.compile_firm

    def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str):
        super().__init__(env, type, file, preprocessed_file, log_file_mode)
        self._should_succeed = False

    def run(self) -> BasicDiffTestResult:
        is_big_testcase = "big" in self.file
        timeout = self.env.big_timeout if is_big_testcase else self.env.timeout
        base_filename = path.basename(self.file).split(".")[0]
        tmp_dir = self.env.create_pid_local_tmpdir()
        shutil.copy(self.preprocessed_file, path.join(tmp_dir, base_filename + ".java"))
        cwd = os.getcwd()
        os.chdir(tmp_dir)
        #print(base_filename, get_main_class_name(base_filename + ".java"))

        test_result = ExtensibleTestResult(self)
        test_result.require_error_string_in_error_case = False
        try:
            out, err, rtcode = None, None, None
            try:
                out, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java", timeout=-1)
                if rtcode != 0:
                    test_result.incorrect_msg = "file can't be compiled"
                    test_result.set_error_code(-rtcode)
363
364
                    test_result.add_long_text("Error output", decode(err))
                    test_result.add_long_text("Output", decode(out))
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
                    test_result.add_file("Source file", self.preprocessed_file)
                    os.chdir(cwd)
                    return test_result
            except SigKill as sig:
                test_result.incorrect_msg = "file can't be compiled: " + sig.name
                test_result.set_error_code(sig.retcode)
                test_result.add_file("Source file", self.preprocessed_file)
                os.chdir(cwd)
                return test_result
            except:
                os.chdir(cwd)
                raise
            try:
                out, err, rtcode = self.env.run_command("./a.out", timeout=timeout)
                if rtcode != 0:
                    test_result.incorrect_msg = "binary can't be run, non zero error code"
                    test_result.set_error_code(rtcode)
382
383
                    test_result.add_long_text("Error output", decode(err))
                    test_result.add_long_text("Output", decode(out))
384
385
386
387
388
389
390
391
392
393
394
395
                    test_result.add_file("Source file", self.preprocessed_file)
                    os.chdir(cwd)
                    return test_result
            except SigKill as sig:
                test_result.incorrect_msg = "binary can't be run: " + sig.name.strip()
                test_result.set_error_code(sig.retcode)
                test_result.add_file("Source file", self.preprocessed_file)
                os.chdir(cwd)
                return test_result
            except:
                os.chdir(cwd)
                raise
396
            test_result = BasicTestResult(self, rtcode, decode(out), decode(err))
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
            test_result.require_error_string_in_error_case = False
            return test_result
        except SigKill as sig:
            os.chdir(cwd)
            assert False
        except BaseException:
            os.chdir(cwd)
            raise


TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(InvalidJavaExecTest)

class InvalidJavaCompileExecTest(InvalidJavaExecTest):
    MODE = TestMode.compile

TestCase.TEST_CASE_CLASSES[TestMode.compile].append(InvalidJavaExecTest)