Add asts test runner

README.mdwn

@@ -16,7 +16,7 @@ The test cases are divided in 5 'modes':
 - __lexer__: Test cases that check the lexed token (and their correct output)
 - __syntax__: Test cases that just check whether `./run --parsecheck` accepts as correct or rejects
  them.
-- __ast__: Test cases that check the generated ast.
+- __ast__: Test cases that check the generated ast by using the pretty printing functionality.
 - __semantic__: Test cases that check semantic checking of MiniJava programs
 - __exec__: Test cases that check the correct compilation of MiniJava programs.
 
@@ -74,14 +74,13 @@ Test types for the ast mode
 <tr><th>File ending(s) of test cases</th><th>Expected behaviour to complete a test of this type</th></tr>
 <tr>
     <td><code>.valid.mj</code> <code>.mj</code>
-    <td>Return code is <code>0</code> and the output matches the expected output (located in the file `[test file].out`</td>
-</tr>
-<tr>
-    <td><code>.invalid.mj</code>
-    <td>Return code is <code>&gt; 0</code> and the error output contains the word <code>error</code></td>
+    <td>Pretty printing the source file should result in the same output as pretty printing the already pretty printed file.
+    The sorted lexer output for the last should be the same as the sorted lexer output for the source file.</td>
 </tr>
 </table>
 
+It uses all syntax mode tests implicitly.
+
 
 Test runner
 -----------

mjtest/environment.py

@@ -20,6 +20,10 @@ class TestMode:
 
     exec = "exec"
 
+    USE_TESTS_OF_OTHER = {
+        ast: [syntax]
+    }
+
 """ All 'success' tests of the n.th mode can used as 'success' tests for the n-1.th mode"""
 TEST_MODES = [TestMode.lexer, TestMode.syntax, TestMode.ast, TestMode.semantic, TestMode.exec]
 
@@ -102,7 +106,7 @@ class Environment:
         mode_flag = {
             TestMode.lexer: "--lextest",
             TestMode.syntax: "--parsetest",
-            TestMode.ast: "--parse-ast"
+            TestMode.ast: "--pretty-print"
         }[mode]
         cmd = [self.mj_run_cmd, mode_flag] + list(args)
         return execute(cmd, timeout=self.timeout)

mjtest/test/ast_tests.py

+import difflib
+import os
 import shutil, logging
+from typing import Tuple
 from mjtest.environment import Environment, TestMode
+from mjtest.test.syntax_tests import BasicSyntaxTest
 from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult
 from os import path
 
 _LOG = logging.getLogger("tests")
 
 
-class ASTDiffTest(TestCase):
+class ASTPrettyPrintTest(BasicSyntaxTest):
 
-    FILE_ENDINGS = [".invalid.mj", ".valid.mj", ".mj"]
-    OUTPUT_FILE_ENDING = ".out"
-    MODE = TestMode.ast
+    FILE_ENDINGS = [".mj", ".valid.mj"]
+    INVALID_FILE_ENDINGS = [".invalid.mj"]
 
     def __init__(self, env: Environment, type: str, file: str):
         super().__init__(env, type, file)
-        self._should_succeed = not file.endswith(".invalid.mj")
-        self._expected_output_file = file + self.OUTPUT_FILE_ENDING
-        self._has_expected_output_file = path.exists(self._expected_output_file)
-
-    def should_succeed(self) -> bool:
-        return self._should_succeed
-
-    def short_name(self) -> str:
-        return path.basename(self.file)
-
-    def run(self) -> BasicDiffTestResult:
-        out, err, rtcode = self.env.run_mj_command(self.MODE, self.file)
-        exp_out = ""
-        if rtcode == 0 and self.should_succeed():
-            if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE:
-                with open(self._expected_output_file, "r") as f:
-                    exp_out = f.read()
-            #else:
-            #    _LOG.error("Expected output file for test case {}:{} is missing.".format(self.MODE, self.short_name()))
-        if self.type == self.MODE and self.env.mode == self.MODE:
-            return BasicDiffTestResult(self, rtcode, out.decode(), err.decode(), exp_out)
-        return BasicTestResult(self, rtcode, out.decode(), err.decode())
-
-class LexerDiffTest(ASTDiffTest):
-
-    MODE = TestMode.lexer
-
-TestCase.TEST_CASE_CLASSES[TestMode.ast].append(ASTDiffTest)
-TestCase.TEST_CASE_CLASSES[TestMode.lexer].append(LexerDiffTest)
\ No newline at end of file
+
+    def run(self) -> BasicTestResult:
+        tmp_file = self.env.create_tmpfile()
+        rtcode, out, err = self._pretty_print(self.file, tmp_file)
+        if rtcode > 0:
+            os.remove(tmp_file)
+            return BasicTestResult(self, rtcode, out, err)
+        _file = self.file
+        tmp_file2 = self.env.create_tmpfile()
+        rtcode, out2, err2 = self._pretty_print(tmp_file, tmp_file2)
+        if rtcode > 0:
+            os.remove(tmp_file2)
+            btr = BasicTestResult(self, rtcode, out2, err2)
+            btr.add_additional_text("Prior out", out)
+            btr.add_additional_text("Prior err", err)
+            return btr
+        rtcode_lex, out_lex, err_lex = self.env.run_mj_command(TestMode.lexer, self.file)
+        rtcode_lex2, out_lex2, err_lex2 = self.env.run_mj_command(TestMode.lexer, tmp_file2)
+        os.remove(tmp_file2)
+        out_lex = self._sort_lexed(out_lex.decode())
+        out_lex2 = self._sort_lexed(out_lex2.decode())
+        incorrect_msg, rtcode = "", 0
+        if rtcode_lex + rtcode_lex2:
+            incorrect_msg, rtcode = "Lexing failed", 1
+        elif out != out2:
+            incorrect_msg, rtcode = "Not idempotent", 1
+        elif out_lex != out_lex2:
+            incorrect_msg, rtcode = "Sorted and lexed second pretty print differs from original", 1
+        btr = BasicTestResult(self, rtcode, incorrect_msg=incorrect_msg)
+        btr.add_additional_text("First round output", out)
+        btr.add_additional_text("Second round output", out2)
+        btr.add_additional_text("Diff", self._diff(out, out2))
+        btr.add_additional_text("Original file, sorted and lexed", out_lex)
+        btr.add_additional_text("Second round output, sorted and lexed", out_lex2)
+        btr.add_additional_text("Diff", self._diff(out_lex, out_lex2))
+        return btr
+
+    def _diff(self, first: str, second: str) -> str:
+        return "".join(difflib.Differ().compare(first.splitlines(True), second.splitlines(True)))
+
+    def _sort_lexed(self, lexed: str) -> str:
+                #return "".join(difflib.Differ().compare(self.expected_output.splitlines(True), self.output.splitlines(True)))
+        return "".join(sorted(lexed.splitlines(True)))
+
+
+
+    def _pretty_print(self, input_file: str, output_file: str) -> Tuple[int, str, str]:
+        out, err, rtcode = self.env.run_mj_command(TestMode.ast, input_file)
+        with open(output_file, "w") as f:
+            print(out, file=f)
+        return rtcode, out.decode(), err.decode()
+
+TestCase.TEST_CASE_CLASSES["ast"].append(ASTPrettyPrintTest)
\ No newline at end of file

mjtest/test/tests.py

@@ -31,7 +31,11 @@ class TestSuite:
         self._load_test_cases()
 
     def _load_test_cases(self):
-        types = TEST_MODES#TEST_MODES[TEST_MODES.index(self.env.mode):]
+        types = [self.env.mode]#TEST_MODES#TEST_MODES[TEST_MODES.index(self.env.mode):]
+        if self.env.ci_testing:
+            types = TEST_MODES
+        elif self.env.mode in TestMode.USE_TESTS_OF_OTHER:
+            types += TestMode.USE_TESTS_OF_OTHER[self.env.mode]
         for type in types:
             self._load_test_case_type(type)
 
@@ -54,21 +58,27 @@ class TestSuite:
                     if len(t) > 0:
                         self.correct_test_cases[mode].add(t)
                         correct_test_cases.add(t)
+        m = mode
+        if m != self.env.mode and self.env.mode in TestMode.USE_TESTS_OF_OTHER and \
+            m in TestMode.USE_TESTS_OF_OTHER[self.env.mode]:
+            m = self.env.mode
         for file in sorted(os.listdir(dir)):
-            if not TestCase.has_valid_file_ending(mode, file):
+            if not TestCase.has_valid_file_ending(m, file):
                 _LOG.debug("Skip file " + file)
             elif self.env.only_incorrect_tests and file in correct_test_cases:
                 _LOG.info("Skip file {} as its test case was executed correctly the last run")
             else:
-                test_case = TestCase.create_from_file(self.env, mode, join(dir, file))
-                if not test_case.can_run():
+                test_case = TestCase.create_from_file(self.env, m, join(dir, file))
+                if not test_case:
+                    pass
+                elif not test_case.can_run():
                     _LOG.debug("Skip test case '{}' because it isn't suited".format(test_case.name()))
                 else:
-                    if mode not in self.test_cases:
-                        self.test_cases[mode] = []
-                    self.test_cases[mode].append(test_case)
-        if mode in self.test_cases and len(self.test_cases[mode]) == 0:
-            del self.test_cases[mode]
+                    if m not in self.test_cases:
+                        self.test_cases[m] = []
+                    self.test_cases[m].append(test_case)
+        if m in self.test_cases and len(self.test_cases[m]) == 0:
+            del self.test_cases[m]
 
     def _log_file_for_type(self, type: str):
         return join(self.env.test_dir, type, ".mjtest_correct_testcases_" + self.env.mode)
@@ -185,6 +195,7 @@ class TestCase:
 
     TEST_CASE_CLASSES = dict((k, []) for k in TEST_MODES)
     FILE_ENDINGS = []
+    INVALID_FILE_ENDINGS = []
 
     def __init__(self, env: Environment, type: str, file: str):
         self.env = env
@@ -196,13 +207,14 @@ class TestCase:
 
     def can_run(self, mode: str = "") -> bool:
         mode = mode or self.env.mode
+        same_mode = self.type == mode
         types = TEST_MODES[TEST_MODES.index(self.env.mode):]
         if self.env.ci_testing:
-            return self.type == mode or \
+            return same_mode or \
                    (self.type in types and self.should_succeed()) or \
                    (self.type not in types and not self.should_succeed())
         else:
-            return self.type == mode
+            return same_mode
 
     def run(self) -> 'TestResult':
         raise NotImplementedError()
@@ -222,7 +234,8 @@ class TestCase:
     @classmethod
     def _test_case_class_for_file(cls, type: str, file: str):
         for t in cls.TEST_CASE_CLASSES[type]:
-            if any(file.endswith(e) for e in t.FILE_ENDINGS):
+            if any(file.endswith(e) for e in t.FILE_ENDINGS) and \
+                    not any(file.endswith(e) for e in t.INVALID_FILE_ENDINGS):
                 return t
         return False
 
@@ -257,12 +270,18 @@ class TestResult:
 
 class BasicTestResult(TestResult):
 
-    def __init__(self, test_case: TestCase, error_code: int, output: str, error_output: str):
+    def __init__(self, test_case: TestCase, error_code: int, output: str = None, error_output: str = None,
+                 incorrect_msg: str = "incorrect return code"):
         super().__init__(test_case, error_code)
+        self._incorrect_msg = incorrect_msg
         self._contains_error_str = "error" in error_output
         self.error_output = error_output
         self.output = output
         self.other_texts = [] # type: List[Tuple[str, str, bool]]
+        if output:
+            self.add_additional_text("Output", output)
+        if error_output:
+            self.add_additional_text("Error output", error_output)
 
     def is_correct(self):
         if self.succeeded():
@@ -276,7 +295,7 @@ class BasicTestResult(TestResult):
         else:
             if not self.succeeded() and not self.test_case.should_succeed() and not self._contains_error_str:
                 return "the error output doesn't contain the word \"error\""
-            return "incorrect return code"
+            return self._incorrect_msg
 
     def long_message(self) -> str:
         file_content = []
@@ -299,19 +318,10 @@ Source file:
 
 {}
 
-Output:
-
-{}
-
-Error output:
-
-{}
-
 Return code: {}
 
 {}
-""".format(self.short_message().capitalize(), self._ident(file_content),
-           self._ident(self.output), self._ident(self.error_output), self.error_code,
+""".format(self.short_message().capitalize(), self._ident(file_content), self.error_code,
            "\n".join(others))
 
     def add_additional_text(self, title: str, content: str):
@@ -364,6 +374,43 @@ class BasicDiffTestResult(BasicTestResult):
             return "incorrect return code"
 
 
+class DiffTest(TestCase):
+
+    FILE_ENDINGS = [".invalid.mj", ".valid.mj", ".mj"]
+    OUTPUT_FILE_ENDING = ".out"
+    MODE = TestMode.ast
+
+    def __init__(self, env: Environment, type: str, file: str):
+        super().__init__(env, type, file)
+        self._should_succeed = not file.endswith(".invalid.mj")
+        self._expected_output_file = file + self.OUTPUT_FILE_ENDING
+        self._has_expected_output_file = exists(self._expected_output_file)
+
+    def should_succeed(self) -> bool:
+        return self._should_succeed
+
+    def short_name(self) -> str:
+        return basename(self.file)
+
+    def run(self) -> BasicDiffTestResult:
+        out, err, rtcode = self.env.run_mj_command(self.MODE, self.file)
+        exp_out = ""
+        if rtcode == 0 and self.should_succeed():
+            if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE:
+                with open(self._expected_output_file, "r") as f:
+                    exp_out = f.read()
+            #else:
+            #    _LOG.error("Expected output file for test case {}:{} is missing.".format(self.MODE, self.short_name()))
+        if self.type == self.MODE and self.env.mode == self.MODE:
+            return BasicDiffTestResult(self, rtcode, out.decode(), err.decode(), exp_out)
+        return BasicTestResult(self, rtcode, out.decode(), err.decode())
+
+
+class LexerDiffTest(DiffTest):
+
+    MODE = TestMode.lexer
+
+TestCase.TEST_CASE_CLASSES[TestMode.lexer].append(LexerDiffTest)
 
 import mjtest.test.syntax_tests
 import mjtest.test.ast_tests
\ No newline at end of file