import hashlib import logging import os import shutil import signal from os import path import re from typing import Optional import errno from mjtest.environment import TestMode, Environment from mjtest.test.syntax_tests import BasicSyntaxTest from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult from mjtest.util.shell import SigKill from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict _LOG = logging.getLogger("exec_tests") class JavaExecTest(BasicSyntaxTest): """ The MiniJava compiler should behave the same as javac """ FILE_ENDINGS = [".java", ".mj"] INVALID_FILE_ENDINGS = [".inf.java", ".inf.mj", ".input.mj", ".input.java"] OUTPUT_FILE_ENDING = ".out" MODE = TestMode.compile_firm INPUT_FILE_REGEXP = r'(\.[0-9]+)\.input(c?)$' def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str): super().__init__(env, type, file, preprocessed_file, log_file_mode) self._has_input_file = bool(re.search(self.INPUT_FILE_REGEXP, file)) self._input_file = file if self._has_input_file else None self._has_character_input = file.endswith("c") if self._has_input_file else False self._expected_output_file = file + self.OUTPUT_FILE_ENDING prev_out_dir = path.join(path.dirname(file), ".java_output") if not path.exists(prev_out_dir): os.mkdir(prev_out_dir) self._prev_out_file = path.join(prev_out_dir, path.basename(self._expected_output_file)) self._prev_out_hash_file = self._prev_out_file + "_hash" self._has_expected_output_file = path.exists(self._expected_output_file) if not self._has_expected_output_file: if path.exists(self._prev_out_file) and path.exists(self._prev_out_hash_file) \ and self._check_hash_sum(self.preprocessed_file, self._prev_out_hash_file): self._has_expected_output_file = True self._expected_output_file = self._prev_out_file _LOG.info("Reuse old java output file \"{}\"".format(path.relpath(self._prev_out_file))) self._should_succeed = True def run(self) -> BasicDiffTestResult: is_big_testcase = "big" in self.file timeout = self.env.big_timeout if is_big_testcase else self.env.timeout base_filename = path.basename(self.file).split(".")[0] tmp_dir = self.env.create_pid_local_tmpdir() shutil.copy(self.preprocessed_file, path.join(tmp_dir, base_filename + ".java")) cwd = os.getcwd() os.chdir(tmp_dir) exp_out = None input_str = None #print(base_filename, get_main_class_name(base_filename + ".java")) test_result = ExtensibleTestResult(self) if self._has_input_file: if self._has_character_input: with open(self._input_file, "r") as f: input_str = f.read() else: with open(self._input_file, "r") as f: chars = bytearray('ascii', 'ignore') # type: bytearray for line in f.readlines(): for part in line.split(" "): part = part.strip() if len(part) > 0: chars.append(int(part)) input_str = chars.decode() if not self._has_expected_output_file: _, err, javac_rtcode = \ self.env.run_command("javac", base_filename + ".java", timeout=timeout) if javac_rtcode != 0: _LOG.error("File \"{}\" isn't valid Java".format(self.preprocessed_file)) test_result.incorrect_msg = "invalid java code, but output file missing" test_result.set_error_code(javac_rtcode) test_result.add_long_text("Javac error message", err.decode()) test_result.add_file("Source file", self.preprocessed_file) os.chdir(cwd) return test_result main_class = get_main_class_name(base_filename + ".java") if not main_class: _LOG.debug("Can't find a main class, using the file name instead") main_class = base_filename exp_out, err, java_rtcode = \ self.env.run_command("java", get_main_class_name(base_filename + ".java"), timeout=timeout, input_str=input_str) if javac_rtcode != 0: test_result.add_long_text("Java output: ", exp_out.decode()) test_result.incorrect_msg = "java runtime error" test_result.set_error_code(java_rtcode) test_result.add_long_text("Java error message", err.decode()) test_result.add_file("Source file", self.preprocessed_file) if self._input_file: test_result.add_file("Input file", self.file) os.chdir(cwd) return test_result exp_out = exp_out.decode() with open(self._prev_out_file, "w") as f: f.write(exp_out) f.flush() with open(self._prev_out_hash_file, "w") as f: f.write(self._hash_sum_for_file(base_filename + ".java")) f.flush() if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE: with open(self._expected_output_file, "r") as f: exp_out = f.read() try: out, err, rtcode = None, None, None try: out, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java", timeout=timeout) if rtcode != 0: test_result.incorrect_msg = "file can't be compiled" test_result.set_error_code(rtcode) test_result.add_long_text("Error output", err.decode()) test_result.add_long_text("Output", out.decode()) test_result.add_file("Source file", self.preprocessed_file) os.chdir(cwd) return test_result except SigKill as sig: test_result.incorrect_msg = "file can't be compiled: " + sig.name test_result.set_error_code(sig.retcode) test_result.add_file("Source file", self.preprocessed_file) os.chdir(cwd) return test_result except: os.chdir(cwd) raise try: out, err, rtcode = self.env.run_command("./a.out", timeout=timeout, input_str=input_str) if rtcode != 0: test_result.incorrect_msg = "binary can't be run, non zero error code" test_result.set_error_code(rtcode) test_result.add_long_text("Error output", err.decode()) test_result.add_long_text("Output", out.decode()) test_result.add_file("Source file", self.preprocessed_file) if self._input_file: test_result.add_file("Input file", self.file) os.chdir(cwd) return test_result except SigKill as sig: test_result.incorrect_msg = "binary can't be run: " + sig.name.strip() test_result.set_error_code(sig.retcode) test_result.add_file("Source file", self.preprocessed_file) if self._input_file: test_result.add_file("Input file", self.file) os.chdir(cwd) return test_result except: os.chdir(cwd) raise out = out.decode() if self.type == self.MODE and self.env.mode == self.MODE: test_result.add_long_text("Output", out) if exp_out.strip() != out.strip(): test_result.incorrect_msg = "incorrect output" test_result.has_succeeded = False test_result.add_short_text("Expected output file", self._expected_output_file) test_result.add_long_text("Expected output", exp_out) test_result.add_long_text("Actual output", out) test_result.add_diff("Output diff [expected <-> actual]", exp_out, out, with_line_numbers=True) test_result.add_file("Source file", self.preprocessed_file) if self._input_file: test_result.add_file("Input file", self.file) os.chdir(cwd) return test_result return BasicTestResult(self, rtcode, out, err.decode()) except SigKill as sig: os.chdir(cwd) assert False except BaseException: os.chdir(cwd) raise def _check_hash_sum(self, file: str, hash_sum_file: str) -> bool: old_hash = "" with open(hash_sum_file, "r") as f: try: old_hash = f.readline().strip() except UnicodeDecodeError: _LOG.exception("Decoding a hash sum for java output caching failed") return self._hash_sum_for_file(file) == old_hash @classmethod def _hash_sum_for_file(self, file: str) -> str: with open(file, "r") as f: return hashlib.sha256(f.read().encode()).hexdigest() @classmethod def _test_case_file_for_input_file(cls, input_file: str) -> Optional[str]: base = re.sub(cls.INPUT_FILE_REGEXP, "", input_file) if path.exists(base + ".input.mj"): return base + ".input.mj" if path.exists(base + ".input.java"): return base + ".input.java" return None @staticmethod def is_file_ending_valid(cls, file: str) -> bool: if re.search(cls.INPUT_FILE_REGEXP, file) and ".java_output" not in file and "precprocessor" not in file: if not JavaExecTest._test_case_file_for_input_file(file): _LOG.error("Skip {} that hasn't a corresponding test case file".format(file)) return True return super().is_file_ending_valid(cls, file) @staticmethod def _get_test_case_file(cls, file: str) -> Optional[str]: if re.search(JavaExecTest.INPUT_FILE_REGEXP, file): return JavaExecTest._test_case_file_for_input_file(file) return file TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(JavaExecTest) class JavaCompileExecTest(JavaExecTest): MODE = TestMode.compile TestCase.TEST_CASE_CLASSES[TestMode.compile].append(JavaCompileExecTest) class JavaInfiniteLoopTest(BasicSyntaxTest): FILE_ENDINGS = [".inf.java", ".inf.mj"] OUTPUT_FILE_ENDING = ".out" MODE = TestMode.compile_firm def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str): super().__init__(env, type, file, preprocessed_file, log_file_mode) self._output_file = self.file + self.OUTPUT_FILE_ENDING self._has_output_file = path.exists(self._output_file) def run(self) -> BasicTestResult: base_filename = path.basename(self.file).split(".")[0] tmp_dir = self.env.create_pid_local_tmpdir() shutil.copy(self.preprocessed_file, path.join(tmp_dir, base_filename + ".java")) cwd = os.getcwd() os.chdir(tmp_dir) timeout = 1 err = None out = None rtcode = None test_result = ExtensibleTestResult(self) try: out, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java") if rtcode != 0: test_result.incorrect_msg = "file can't be compiled" test_result.set_error_code(rtcode) test_result.add_long_text("Error output", err.decode()) test_result.add_long_text("Output", out.decode()) test_result.add_file("Source file", self.preprocessed_file) os.chdir(cwd) return test_result out, err, rtcode = self.env.run_command("./a.out", timeout=timeout) if rtcode != 0: test_result.incorrect_msg = "binary can't be run, non zero error code" test_result.set_error_code(rtcode) test_result.add_long_text("Error output", err.decode()) test_result.add_long_text("Output", out.decode()) test_result.add_file("Source file", self.preprocessed_file) os.chdir(cwd) return test_result except SigKill as sig: if sig.retcode == signal.SIGXCPU: out, _, _, _ = self.env.run_command_till_timeout("./a.out", timeout=1) test_result.add_long_text("Output", out.decode()) if self._has_output_file: out = out.decode().strip() exp_out = "" with open(self._output_file, "r") as f: exp_out = f.read().strip() test_result.add_long_text("Expected output start", exp_out) if not out.startswith(exp_out): test_result.incorrect_msg = "incorrect output start" test_result.has_succeeded = False test_result.add_file("Source file", self.preprocessed_file) os.chdir(cwd) return test_result else: test_result.add_file("Source file", self.preprocessed_file) os.chdir(cwd) return test_result else: test_result.incorrect_msg = "sdfbinary can't be run: " + sig.name test_result.set_error_code(sig.retcode) test_result.add_file("Source file", self.preprocessed_file) os.chdir(cwd) return test_result except BaseException as e: os.chdir(cwd) print(e) raise os.chdir(cwd) test_result.incorrect_msg = "run shorter than one second" test_result.add_long_text("Output", out) test_result.has_succeeded = False test_result.add_file("Source file", self.preprocessed_file) os.chdir(cwd) return test_result TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(JavaInfiniteLoopTest)