Add `ast` mode

README.mdwn

@@ -15,6 +15,7 @@ Test modes
 The test cases are divided in 3 'modes':
 - __syntax__: Test cases that just check whether `./run --parsecheck` accepts as correct or rejects
  them.
+- __ast__: Test cases that check the generated ast.
 - __semantic__: Test cases that check semantic checking of MiniJava programs
 - __exec__: Test cases that check the correct compilation of MiniJava programs.
 
@@ -27,13 +28,14 @@ The different types a test cases are differentiated by their file endings.
 
 Side note: An error code greater than 0 should result in an errror message on error output containing the word `error`.
 
+
 Test types for the syntax mode
 ------------------------------
 
 <table>
 <tr><th>File ending(s) of test cases</th><th>Expected behaviour to complete a test of this type</th></tr>
 <tr>
-    <td><code>.valid.mj</code><code>.mj</code>
+    <td><code>.valid.mj</code> <code>.mj</code>
     <td>Return code is <code>0</code>, i.e. the MiniJava is accepted as syntactically correct</td>
 </tr>
 <tr>
@@ -47,6 +49,23 @@ Test types for the syntax mode
 </tr>
 </table>
 
+
+Test types for the ast mode
+------------------------------
+
+<table>
+<tr><th>File ending(s) of test cases</th><th>Expected behaviour to complete a test of this type</th></tr>
+<tr>
+    <td><code>.valid.mj</code> <code>.mj</code>
+    <td>Return code is <code>0</code> and the output matches the expected output (located in the file `[test file].out`</td>
+</tr>
+<tr>
+    <td><code>.invalid.mj</code>
+    <td>Return code is <code>&gt; 0</code> and the error output contains the word <code>error</code></td>
+</tr>
+</table>
+
+
 Test runner
 -----------
 
@@ -73,12 +92,12 @@ Output of the `./mjt.py --help`
 ```
 usage: mjt.py [-h] [--only_incorrect_tests] [--produce_no_reports]
               [--parallel] [--log_level LOG_LEVEL]
-              {syntax,semantic,exec} MJ_RUN
+              {syntax,ast,semantic,exec} MJ_RUN
 
 MiniJava test runner
 
 positional arguments:
-  {syntax,semantic,exec}
+  {syntax,ast,semantic,exec}
                         What do you want to test?
   MJ_RUN                Command to run your MiniJava implementation, e.g.
                         `mj/run`, can be omitted by assigning the environment

mjtest/environment.py

@@ -11,12 +11,14 @@ from typing import Tuple, List
 class TestMode:
     syntax = "syntax"
 
+    ast = "ast"
+
     semantic = "semantic"
 
     exec = "exec"
 
 """ All 'success' tests of the n.th mode can used as 'success' tests for the n-1.th mode"""
-TEST_MODES = [TestMode.syntax, TestMode.semantic, TestMode.exec]
+TEST_MODES = [TestMode.syntax, TestMode.ast, TestMode.semantic, TestMode.exec]
 
 class Environment:
 
@@ -49,7 +51,7 @@ class Environment:
             self.test_dir = os.path.join(get_mjtest_basedir(), "tests")
         if not os.path.exists(self.test_dir):
             os.mkdir(self.test_dir)
-            for d in [TestMode.syntax, TestMode.semantic, TestMode.exec]:
+            for d in TEST_MODES:
                 os.mkdir(os.path.join(self.test_dir, d))
 
         self.only_incorrect_tests = only_incorrect_tests
@@ -92,7 +94,8 @@ class Environment:
         :return: (out, err, return code)
         """
         mode_flag = {
-            TestMode.syntax: "--parsetest"
+            TestMode.syntax: "--parsetest",
+            TestMode.ast: "--parse-ast"
         }[mode]
         cmd = [self.mj_run_cmd, mode_flag] + list(args)
         return execute(cmd, timeout=self.timeout)

mjtest/test/ast_tests.py

+import shutil, logging
+from mjtest.environment import Environment, TestMode
+from mjtest.test.tests import TestCase, BasicDiffTestResult
+from os import path
+
+_LOG = logging.getLogger("tests")
+
+
+class ASTDiffTest(TestCase):
+
+    FILE_ENDINGS = [".invalid.mj", ".valid.mj", ".mj"]
+    OUTPUT_FILE_ENDING = ".out"
+    MODE = TestMode.ast
+
+    def __init__(self, env: Environment, type: str, file: str):
+        super().__init__(env, type, file)
+        self._should_succeed = not file.endswith(".invalid.mj")
+        self._expected_output_file = file + self.OUTPUT_FILE_ENDING
+        self._has_expected_output_file = path.exists(self._expected_output_file)
+
+    def should_succeed(self) -> bool:
+        return self._should_succeed
+
+    def short_name(self) -> str:
+        return path.basename(self.file)
+
+    def run(self) -> BasicDiffTestResult:
+        out, err, rtcode = self.env.run_mj_command(self.MODE, self.file)
+        exp_out = ""
+        if rtcode > 0 and self.should_succeed():
+            if self._has_expected_output_file:
+                with open(self._expected_output_file, "r") as f:
+                    exp_out = f.read()
+            else:
+                _LOG.error("Expected output file for test case {}:{} is missing.".format(self.MODE, self.short_name()))
+        return BasicDiffTestResult(self, rtcode, out.decode(), err.decode(), exp_out)
+
+TestCase.TEST_CASE_CLASSES[TestMode.ast].append(ASTDiffTest)
\ No newline at end of file

mjtest/test/tests.py

@@ -11,6 +11,7 @@ from mjtest.util.parallelism import available_cpu_count
 from mjtest.util.utils import cprint, colored
 from pprint import pprint
 import shutil
+import difflib
 
 _LOG = logging.getLogger("tests")
 
@@ -40,7 +41,7 @@ class TestSuite:
         if exists(dir):
             self._load_test_case_dir(type, dir)
         else:
-            _LOG.warn("Test folder {} doesn't exist".format(dir))
+            _LOG.info("Test folder {} doesn't exist".format(dir))
 
     def _load_test_case_dir(self, mode: str, dir: str):
         self.test_cases[mode] = []
@@ -175,10 +176,7 @@ class TestCase:
     A single test case.
     """
 
-    TEST_CASE_CLASSES = {
-        TestMode.syntax: [],
-        TestMode.semantic: []
-    }
+    TEST_CASE_CLASSES = dict((k, []) for k in TEST_MODES)
     FILE_ENDINGS = []
 
     def __init__(self, env: Environment, type: str, file: str):
@@ -192,14 +190,8 @@ class TestCase:
     def can_run(self, mode: str = "") -> bool:
         mode = mode or self.env.mode
 
-        if mode == TestMode.exec:
-            return self.type == TestMode.exec
-        if mode == TestMode.semantic:
-            return self.type == TestMode.semantic \
-                   or (self.type == TestMode.exec and self.should_succeed())
-        if mode == TestMode.syntax:
-            return self.type == TestMode.syntax or \
-                   (self.can_run(TestMode.semantic) and self.should_succeed())
+        return self.type == mode or \
+               (self.type in TEST_MODES[TEST_MODES.index(self.env.mode):] and self.should_succeed())
 
     def run(self) -> 'TestResult':
         raise NotImplementedError()
@@ -324,4 +316,43 @@ Return code: {}
         arr = ["[{:04d}]    {:s}".format(i + 1, l) for (i, l) in enumerate(arr)]
         return "\n".join(arr)
 
-import mjtest.test.syntax_tests
\ No newline at end of file
+
+class BasicDiffTestResult(BasicTestResult):
+
+    def __init__(self, test_case: TestCase, error_code: int, output: str, error_output: str, expected_output: str):
+        super().__init__(test_case, error_code, output, error_output)
+        self.expected_output = expected_output
+        self._is_output_correct = self.expected_output.strip() == self.output
+        if self.is_correct():
+            self.add_additional_text("Expected and actual output", self.output)
+        elif self.succeeded() and self.test_case.should_succeed():
+            self.add_additional_text("Diff[expected output, actual output]", self._output_diff())
+            self.add_additional_text("Expected output", self.expected_output)
+            self.add_additional_text("Actual output", self.output)
+
+    def is_correct(self):
+        if self.succeeded():
+            return super().is_correct() and self.is_output_correct()
+        else:
+            return super().is_correct() and self._contains_error_str
+
+    def _output_diff(self) -> str:
+        return difflib.Differ().compare(self.expected_output, self.output)
+
+    def is_output_correct(self) -> str:
+        return self._is_output_correct
+
+    def short_message(self) -> str:
+        if self.is_correct():
+            return "correct"
+        else:
+            if not self.succeeded() and not self.test_case.should_succeed() and not self._contains_error_str:
+                return "the error output doesn't contain the word \"error\""
+            if self.succeeded() and self.test_case.should_succeed():
+                return "the actual output differs from the expected"
+            return "incorrect return code"
+
+
+
+import mjtest.test.syntax_tests
+import mjtest.test.ast_tests
\ No newline at end of file