diff --git a/mjtest/test/exec_tests.py b/mjtest/test/exec_tests.py index b02fe44b5af43c42b9fb424b1357162cf26cf2e3..e728cbd5fdd49815473e5acc92bff4d12938db7e 100644 --- a/mjtest/test/exec_tests.py +++ b/mjtest/test/exec_tests.py @@ -6,9 +6,9 @@ import signal from os import path from mjtest.environment import TestMode, Environment from mjtest.test.syntax_tests import BasicSyntaxTest -from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult +from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult from mjtest.util.shell import SigKill -from mjtest.util.utils import get_main_class_name +from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict _LOG = logging.getLogger("exec_tests") @@ -41,22 +41,38 @@ class JavaExecTest(BasicSyntaxTest): def run(self) -> BasicDiffTestResult: base_filename = path.basename(self.file).split(".")[0] - tmp_dir = self.env.create_pid_local_tmpdir() + tmp_dir = self.env.create_tmpdir() shutil.copy(self.preprocessed_file, path.join(tmp_dir, base_filename + ".java")) cwd = os.getcwd() os.chdir(tmp_dir) exp_out = None #print(base_filename, get_main_class_name(base_filename + ".java")) + + test_result = ExtensibleTestResult(self) + if not self._has_expected_output_file: - _, _, javac_rtcode = \ + _, err, javac_rtcode = \ self.env.run_command("javac", base_filename + ".java") if javac_rtcode != 0: _LOG.error("File \"{}\" isn't valid Java".format(self.preprocessed_file)) + test_result.incorrect_msg = "invalid java code, but output file missing" + test_result.error_code = javac_rtcode + test_result.add_long_text("Javac error message", err.decode()) + test_result.add_file("Source file", self.preprocessed_file) os.chdir(cwd) - raise InterruptedError() - exp_out, _, _ = \ + return test_result + exp_out, err, java_rtcode = \ self.env.run_command("java", get_main_class_name(base_filename + ".java")) + test_result.add_long_text("Java output: ", exp_out.decode()) + if javac_rtcode != 0: + test_result.incorrect_msg = "java runtime error" + test_result.error_code = java_rtcode + test_result.add_long_text("Java error message", err.decode()) + test_result.add_file("Source file", self.preprocessed_file) + os.chdir(cwd) + return test_result exp_out = exp_out.decode().strip() + with open(self._prev_out_file, "w") as f: f.write(exp_out) f.flush() @@ -66,20 +82,64 @@ class JavaExecTest(BasicSyntaxTest): if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE: with open(self._expected_output_file, "r") as f: exp_out = f.read() + test_result.add_short_text("Expected output file", self._expected_output_file) + test_result.add_long_text("Expected output", exp_out) try: - _, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java") - out, _, _ = self.env.run_command("./" + base_filename) + out, err, rtcode = None, None, None + try: + out, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java") + if rtcode != 0: + test_result.incorrect_msg = "file can't be compiled" + test_result.error_code = rtcode + test_result.add_long_text("Error output", err.decode()) + test_result.add_long_text("Output", out.decode()) + test_result.add_file("Source file", self.preprocessed_file) + os.chdir(cwd) + return test_result + except SigKill as sig: + test_result.incorrect_msg = "file can't be compiled: " + sig.name + test_result.error_code = sig.retcode + test_result.add_file("Source file", self.preprocessed_file) + os.chdir(cwd) + return test_result + except: + os.chdir(cwd) + raise + try: + out, err, rtcode = self.env.run_command("./" + base_filename) + if rtcode != 0: + test_result.incorrect_msg = "file can't be run" + test_result.error_code = rtcode + test_result.add_long_text("Error output", err.decode()) + test_result.add_long_text("Output", out.decode()) + test_result.add_file("Source file", self.preprocessed_file) + os.chdir(cwd) + return test_result + except SigKill as sig: + test_result.incorrect_msg = "binary can't be run: " + sig.name + test_result.error_code = sig.retcode + test_result.add_file("Source file", self.preprocessed_file) + os.chdir(cwd) + return test_result + except: + os.chdir(cwd) + raise out = out.decode().strip() - os.chdir(cwd) if self.type == self.MODE and self.env.mode == self.MODE: - return BasicDiffTestResult(self, rtcode, out, err.decode(), exp_out) + test_result.add_long_text("Output", out.decode()) + if exp_out.strip() != out.strip(): + test_result.incorrect_msg = "incorrect output" + test_result.add_diff("Output diff [expected <-> actual]", exp_out, out) + test_result.add_file("Source file", self.preprocessed_file) + os.chdir(cwd) + return test_result return BasicTestResult(self, rtcode, out, err.decode()) except SigKill as sig: os.chdir(cwd) - return BasicTestResult(self, sig.retcode, "", exp_out, sig.name) + assert False except: os.chdir(cwd) - raise + assert False def _check_hash_sum(self, file: str, hash_sum_file: str) -> bool: old_hash = "" diff --git a/mjtest/test/tests.py b/mjtest/test/tests.py index eca75b854dd1610e1d333a99d7d9071aadef32ca..0869ade6530280d21faadd0d92086abce8ad8654 100644 --- a/mjtest/test/tests.py +++ b/mjtest/test/tests.py @@ -296,6 +296,99 @@ class TestResult: raise NotImplementedError() +class ExtensibleTestResult(TestResult): + + def __init__(self, test_case: TestCase): + super().__init__(test_case, None) + self.messages = [] # type: List[TestResultMessage] + self.incorrect_msg = None # type: Optional[str] + self.has_succeeded = True # type: bool + self._contains_error_str = True # type: bool + + def add_error_output(self, title: str, error_output: str): + """ + Checks for "error" string + """ + self._contains_error_str = self._contains_error_str and error_output is not None and "error" in error_output + self.messages.append(TestResultMessage(title, c)) + + def add_long_text(self, title: str, content: str, with_line_numbers: bool = True): + self.messages.append(TestResultMessage(title, content, multiline=True, with_line_numbers=with_line_numbers)) + + def add_short_text(self, title, content: str): + self.messages.append(TestResultMessage(title, content, multiline=False, with_line_numbers=False)) + + def add_file(self, title: str, file_name: str, with_line_numbers: bool = True): + with open(file_name, "r") as f: + file_content = os.linesep.join([line.rstrip() for line in f]) + self.add_long_text(title, file_content, with_line_numbers) + + def succeeded(self): + return self.has_succeeded + + def is_correct(self): + if self.succeeded(): + return super().is_correct() + else: + return super().is_correct() and self._contains_error_str + + def short_message(self) -> str: + if self.is_correct(): + return "correct" + else: + if not self.succeeded() and not self.test_case.should_succeed() and not self._contains_error_str: + return "the error output doesn't contain the word \"error\"" + return self.incorrect_msg + + def long_message(self) -> str: + texts = [self.short_message().capitalize()] + if self.error_code is not None: + texts.append("Error code: {}".format(self.error_code)) + for msg in self.messages: + if msg.multiline: + texts.append(msg.title + ":") + texts.append("") + if msg.with_line_numbers: + texts.append(self._ident(msg.content)) + else: + texts.append(msg.content) + else: + texts.append("{}: {}".format(msg.title, msg.content)) + + def _ident(self, text: Union[str,List[str]]) -> str: + arr = text if isinstance(text, list) else text.split("\n") + if len(arr) == 0 or text == "": + return "" + arr = ["[{:04d}] {:s}".format(i + 1, l) for (i, l) in enumerate(arr)] + return "\n".join(arr) + + def add_diff(self, title: str, first: str, second: str, with_line_numbers: bool): + self.add_long_text(title, "".join(difflib.Differ().compare(first.splitlines(True), second.splitlines(True))), + with_line_numbers=with_line_numbers) + + +class TestResultMessage: + + def __init__(self, title: str, content: str, multiline: bool, with_line_numbers: bool): + self.title = title + self.content = content + self.multiline = multiline + self.with_line_numbers = with_line_numbers + + +class TestResultFactory: + def __init__(self): + self._texts = [] # type: [str, str, bool] + self.return_code = 0 + self.short_error_message = None # type: str + + def add_short_texts(self, title: str, content: str): + self._texts.append((title, content, True)) + + def add_long_message(self, title: str, content: str): + self._texts.append((title, content, True)) + + class BasicTestResult(TestResult): def __init__(self, test_case: TestCase, error_code: int, output: str = None, error_output: str = None, @@ -310,6 +403,10 @@ class BasicTestResult(TestResult): self.add_additional_text("Output", output) if error_output: self.add_additional_text("Error output", error_output) + self.has_succeeded = error_code == 0 + + def succeeded(self): + return self.has_succeeded def is_correct(self): if self.succeeded(): diff --git a/mjtest/util/utils.py b/mjtest/util/utils.py index 3b40cd358fb06b77f746826bfb81c659b079ad86..9d6a6f22b24e31ae97c17a83bb63a2f2615d9b0a 100644 --- a/mjtest/util/utils.py +++ b/mjtest/util/utils.py @@ -1,7 +1,7 @@ import logging from os import path import sys -from typing import Tuple, Optional +from typing import Tuple, Optional, Any, List, Callable import re COLOR_OUTPUT_IF_POSSIBLE = False @@ -57,3 +57,60 @@ def get_main_class_name(file: str) -> Optional[str]: elif "String[]" in line and "main" in line and "void" in line and "static" in line and "public" in line: return current_class return None + + +class InsertionTimeOrderedDict: + """ + A dictionary which's elements are sorted by their insertion time. + """ + + def __init__(self): + self._dict = {} + self._keys = [] + dict() + + def __delitem__(self, key): + """ Remove the entry with the passed key """ + del(self._dict[key]) + del(self._keys[self._keys.index(key)]) + + def __getitem__(self, key): + """ Get the entry with the passed key """ + return self._dict[key] + + def __setitem__(self, key, value): + """ Set the value of the item with the passed key """ + if key not in self._dict: + self._keys.append(key) + self._dict[key] = value + + def __iter__(self): + """ Iterate over all keys """ + return self._keys.__iter__() + + def values(self) -> List: + """ Rerturns all values of this dictionary. They are sorted by their insertion time. """ + return [self._dict[key] for key in self._keys] + + def keys(self) -> List: + """ Returns all keys of this dictionary. They are sorted by their insertion time. """ + return self._keys + + def __len__(self): + """ Returns the number of items in this dictionary """ + return len(self._keys) + + @classmethod + def from_list(cls, items: Optional[list], key_func: Callable[[Any], Any]) -> 'InsertionTimeOrderedDict': + """ + Creates an ordered dict out of a list of elements. + :param items: list of elements + :param key_func: function that returns a key for each passed list element + :return: created ordered dict with the elements in the same order as in the passed list + """ + if items is None: + return InsertionTimeOrderedDict() + ret = InsertionTimeOrderedDict() + for item in items: + ret[key_func(item)] = item + return ret \ No newline at end of file