Fix bugs in compile-firm mode test runners

mjtest/test/exec_tests.py

@@ -6,9 +6,9 @@ import signal
 from os import path
 from mjtest.environment import TestMode, Environment
 from mjtest.test.syntax_tests import BasicSyntaxTest
-from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult
+from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult
 from mjtest.util.shell import SigKill
-from mjtest.util.utils import get_main_class_name
+from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict
 
 _LOG = logging.getLogger("exec_tests")
 
@@ -41,22 +41,38 @@ class JavaExecTest(BasicSyntaxTest):
 
     def run(self) -> BasicDiffTestResult:
         base_filename = path.basename(self.file).split(".")[0]
-        tmp_dir = self.env.create_pid_local_tmpdir()
+        tmp_dir = self.env.create_tmpdir()
         shutil.copy(self.preprocessed_file, path.join(tmp_dir, base_filename + ".java"))
         cwd = os.getcwd()
         os.chdir(tmp_dir)
         exp_out = None
         #print(base_filename, get_main_class_name(base_filename + ".java"))
+
+        test_result = ExtensibleTestResult(self)
+
         if not self._has_expected_output_file:
-            _, _, javac_rtcode = \
+            _, err, javac_rtcode = \
                 self.env.run_command("javac", base_filename + ".java")
             if javac_rtcode != 0:
                 _LOG.error("File \"{}\" isn't valid Java".format(self.preprocessed_file))
+                test_result.incorrect_msg = "invalid java code, but output file missing"
+                test_result.error_code = javac_rtcode
+                test_result.add_long_text("Javac error message", err.decode())
+                test_result.add_file("Source file", self.preprocessed_file)
                 os.chdir(cwd)
-                raise InterruptedError()
-            exp_out, _, _ = \
+                return test_result
+            exp_out, err, java_rtcode = \
                 self.env.run_command("java", get_main_class_name(base_filename + ".java"))
+            test_result.add_long_text("Java output: ", exp_out.decode())
+            if javac_rtcode != 0:
+                test_result.incorrect_msg = "java runtime error"
+                test_result.error_code = java_rtcode
+                test_result.add_long_text("Java error message", err.decode())
+                test_result.add_file("Source file", self.preprocessed_file)
+                os.chdir(cwd)
+                return test_result
             exp_out = exp_out.decode().strip()
+
             with open(self._prev_out_file, "w") as f:
                 f.write(exp_out)
                 f.flush()
@@ -66,20 +82,64 @@ class JavaExecTest(BasicSyntaxTest):
         if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE:
             with open(self._expected_output_file, "r") as f:
                 exp_out = f.read()
+            test_result.add_short_text("Expected output file", self._expected_output_file)
+            test_result.add_long_text("Expected output", exp_out)
         try:
-            _, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java")
-            out, _, _ = self.env.run_command("./" + base_filename)
-            out = out.decode().strip()
+            out, err, rtcode = None, None, None
+            try:
+                out, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java")
+                if rtcode != 0:
+                    test_result.incorrect_msg = "file can't be compiled"
+                    test_result.error_code = rtcode
+                    test_result.add_long_text("Error output", err.decode())
+                    test_result.add_long_text("Output", out.decode())
+                    test_result.add_file("Source file", self.preprocessed_file)
+                    os.chdir(cwd)
+                    return test_result
+            except SigKill as sig:
+                test_result.incorrect_msg = "file can't be compiled: " + sig.name
+                test_result.error_code = sig.retcode
+                test_result.add_file("Source file", self.preprocessed_file)
+                os.chdir(cwd)
+                return test_result
+            except:
                 os.chdir(cwd)
+                raise
+            try:
+                out, err, rtcode = self.env.run_command("./" + base_filename)
+                if rtcode != 0:
+                    test_result.incorrect_msg = "file can't be run"
+                    test_result.error_code = rtcode
+                    test_result.add_long_text("Error output", err.decode())
+                    test_result.add_long_text("Output", out.decode())
+                    test_result.add_file("Source file", self.preprocessed_file)
+                    os.chdir(cwd)
+                    return test_result
+            except SigKill as sig:
+                test_result.incorrect_msg = "binary can't be run: " + sig.name
+                test_result.error_code = sig.retcode
+                test_result.add_file("Source file", self.preprocessed_file)
+                os.chdir(cwd)
+                return test_result
+            except:
+                os.chdir(cwd)
+                raise
+            out = out.decode().strip()
             if self.type == self.MODE and self.env.mode == self.MODE:
-                return BasicDiffTestResult(self, rtcode, out, err.decode(), exp_out)
+                test_result.add_long_text("Output", out.decode())
+                if exp_out.strip() != out.strip():
+                    test_result.incorrect_msg = "incorrect output"
+                    test_result.add_diff("Output diff [expected <-> actual]", exp_out, out)
+                    test_result.add_file("Source file", self.preprocessed_file)
+                os.chdir(cwd)
+                return test_result
             return BasicTestResult(self, rtcode, out, err.decode())
         except SigKill as sig:
             os.chdir(cwd)
-            return BasicTestResult(self, sig.retcode, "", exp_out, sig.name)
+            assert False
         except:
             os.chdir(cwd)
-            raise
+            assert False
 
     def _check_hash_sum(self, file: str, hash_sum_file: str) -> bool:
         old_hash = ""

mjtest/test/tests.py

@@ -296,6 +296,99 @@ class TestResult:
         raise NotImplementedError()
 
 
+class ExtensibleTestResult(TestResult):
+
+    def __init__(self, test_case: TestCase):
+        super().__init__(test_case, None)
+        self.messages = []  # type: List[TestResultMessage]
+        self.incorrect_msg = None  # type: Optional[str]
+        self.has_succeeded = True  # type: bool
+        self._contains_error_str = True  # type: bool
+
+    def add_error_output(self, title: str, error_output: str):
+        """
+        Checks for "error" string
+        """
+        self._contains_error_str = self._contains_error_str and error_output is not None and "error" in error_output
+        self.messages.append(TestResultMessage(title, c))
+
+    def add_long_text(self, title: str, content: str, with_line_numbers: bool = True):
+        self.messages.append(TestResultMessage(title, content, multiline=True, with_line_numbers=with_line_numbers))
+
+    def add_short_text(self, title, content: str):
+        self.messages.append(TestResultMessage(title, content, multiline=False, with_line_numbers=False))
+
+    def add_file(self, title: str, file_name: str, with_line_numbers: bool = True):
+        with open(file_name, "r") as f:
+            file_content = os.linesep.join([line.rstrip() for line in f])
+        self.add_long_text(title, file_content, with_line_numbers)
+
+    def succeeded(self):
+        return self.has_succeeded
+
+    def is_correct(self):
+        if self.succeeded():
+            return super().is_correct()
+        else:
+            return super().is_correct() and self._contains_error_str
+
+    def short_message(self) -> str:
+        if self.is_correct():
+            return "correct"
+        else:
+            if not self.succeeded() and not self.test_case.should_succeed() and not self._contains_error_str:
+                return "the error output doesn't contain the word \"error\""
+            return self.incorrect_msg
+
+    def long_message(self) -> str:
+        texts = [self.short_message().capitalize()]
+        if self.error_code is not None:
+            texts.append("Error code: {}".format(self.error_code))
+        for msg in self.messages:
+            if msg.multiline:
+                texts.append(msg.title + ":")
+                texts.append("")
+                if msg.with_line_numbers:
+                    texts.append(self._ident(msg.content))
+                else:
+                    texts.append(msg.content)
+            else:
+                texts.append("{}: {}".format(msg.title, msg.content))
+
+    def _ident(self, text: Union[str,List[str]]) -> str:
+        arr = text if isinstance(text, list) else text.split("\n")
+        if len(arr) == 0 or text == "":
+            return ""
+        arr = ["[{:04d}]    {:s}".format(i + 1, l) for (i, l) in enumerate(arr)]
+        return "\n".join(arr)
+
+    def add_diff(self, title: str, first: str, second: str, with_line_numbers: bool):
+        self.add_long_text(title, "".join(difflib.Differ().compare(first.splitlines(True), second.splitlines(True))),
+                           with_line_numbers=with_line_numbers)
+
+
+class TestResultMessage:
+
+    def __init__(self, title: str, content: str, multiline: bool, with_line_numbers: bool):
+        self.title = title
+        self.content = content
+        self.multiline = multiline
+        self.with_line_numbers = with_line_numbers
+
+
+class TestResultFactory:
+    def __init__(self):
+        self._texts = []  # type: [str, str, bool]
+        self.return_code = 0
+        self.short_error_message = None  # type: str
+
+    def add_short_texts(self, title: str, content: str):
+        self._texts.append((title, content, True))
+
+    def add_long_message(self, title: str, content: str):
+        self._texts.append((title, content, True))
+
+
 class BasicTestResult(TestResult):
 
     def __init__(self, test_case: TestCase, error_code: int, output: str = None, error_output: str = None,
@@ -310,6 +403,10 @@ class BasicTestResult(TestResult):
             self.add_additional_text("Output", output)
         if error_output:
             self.add_additional_text("Error output", error_output)
+        self.has_succeeded = error_code == 0
+
+    def succeeded(self):
+        return self.has_succeeded
 
     def is_correct(self):
         if self.succeeded():

mjtest/util/utils.py

 import logging
 from os import path
 import sys
-from typing import Tuple, Optional
+from typing import Tuple, Optional, Any, List, Callable
 import re
 
 COLOR_OUTPUT_IF_POSSIBLE = False
@@ -57,3 +57,60 @@ def get_main_class_name(file: str) -> Optional[str]:
             elif "String[]" in line and "main" in line and "void" in line and "static" in line and "public" in line:
                 return current_class
     return None
+
+
+class InsertionTimeOrderedDict:
+    """
+    A dictionary which's elements are sorted by their insertion time.
+    """
+
+    def __init__(self):
+        self._dict = {}
+        self._keys = []
+        dict()
+
+    def __delitem__(self, key):
+        """ Remove the entry with the passed key """
+        del(self._dict[key])
+        del(self._keys[self._keys.index(key)])
+
+    def __getitem__(self, key):
+        """ Get the entry with the passed key """
+        return self._dict[key]
+
+    def __setitem__(self, key, value):
+        """ Set the value of the item with the passed key """
+        if key not in self._dict:
+            self._keys.append(key)
+        self._dict[key] = value
+
+    def __iter__(self):
+        """ Iterate over all keys """
+        return self._keys.__iter__()
+
+    def values(self) -> List:
+        """ Rerturns all values of this dictionary. They are sorted by their insertion time. """
+        return [self._dict[key] for key in self._keys]
+
+    def keys(self) -> List:
+        """ Returns all keys of this dictionary. They are sorted by their insertion time. """
+        return self._keys
+
+    def __len__(self):
+        """ Returns the number of items in this dictionary """
+        return len(self._keys)
+
+    @classmethod
+    def from_list(cls, items: Optional[list], key_func: Callable[[Any], Any]) -> 'InsertionTimeOrderedDict':
+        """
+        Creates an ordered dict out of a list of elements.
+        :param items: list of elements
+        :param key_func: function that returns a key for each passed list element
+        :return: created ordered dict with the elements in the same order as in the passed list
+        """
+        if items is None:
+            return InsertionTimeOrderedDict()
+        ret = InsertionTimeOrderedDict()
+        for item in items:
+            ret[key_func(item)] = item
+        return ret
\ No newline at end of file