Commit d3ccbe56 authored by Johannes Bechberger's avatar Johannes Bechberger

Add `ast` mode

parent 4bd5b32b
......@@ -15,6 +15,7 @@ Test modes
The test cases are divided in 3 'modes':
- __syntax__: Test cases that just check whether `./run --parsecheck` accepts as correct or rejects
them.
- __ast__: Test cases that check the generated ast.
- __semantic__: Test cases that check semantic checking of MiniJava programs
- __exec__: Test cases that check the correct compilation of MiniJava programs.
......@@ -27,13 +28,14 @@ The different types a test cases are differentiated by their file endings.
Side note: An error code greater than 0 should result in an errror message on error output containing the word `error`.
Test types for the syntax mode
------------------------------
<table>
<tr><th>File ending(s) of test cases</th><th>Expected behaviour to complete a test of this type</th></tr>
<tr>
<td><code>.valid.mj</code><code>.mj</code>
<td><code>.valid.mj</code> <code>.mj</code>
<td>Return code is <code>0</code>, i.e. the MiniJava is accepted as syntactically correct</td>
</tr>
<tr>
......@@ -47,6 +49,23 @@ Test types for the syntax mode
</tr>
</table>
Test types for the ast mode
------------------------------
<table>
<tr><th>File ending(s) of test cases</th><th>Expected behaviour to complete a test of this type</th></tr>
<tr>
<td><code>.valid.mj</code> <code>.mj</code>
<td>Return code is <code>0</code> and the output matches the expected output (located in the file `[test file].out`</td>
</tr>
<tr>
<td><code>.invalid.mj</code>
<td>Return code is <code>&gt; 0</code> and the error output contains the word <code>error</code></td>
</tr>
</table>
Test runner
-----------
......@@ -73,12 +92,12 @@ Output of the `./mjt.py --help`
```
usage: mjt.py [-h] [--only_incorrect_tests] [--produce_no_reports]
[--parallel] [--log_level LOG_LEVEL]
{syntax,semantic,exec} MJ_RUN
{syntax,ast,semantic,exec} MJ_RUN
MiniJava test runner
positional arguments:
{syntax,semantic,exec}
{syntax,ast,semantic,exec}
What do you want to test?
MJ_RUN Command to run your MiniJava implementation, e.g.
`mj/run`, can be omitted by assigning the environment
......
......@@ -11,12 +11,14 @@ from typing import Tuple, List
class TestMode:
syntax = "syntax"
ast = "ast"
semantic = "semantic"
exec = "exec"
""" All 'success' tests of the n.th mode can used as 'success' tests for the n-1.th mode"""
TEST_MODES = [TestMode.syntax, TestMode.semantic, TestMode.exec]
TEST_MODES = [TestMode.syntax, TestMode.ast, TestMode.semantic, TestMode.exec]
class Environment:
......@@ -49,7 +51,7 @@ class Environment:
self.test_dir = os.path.join(get_mjtest_basedir(), "tests")
if not os.path.exists(self.test_dir):
os.mkdir(self.test_dir)
for d in [TestMode.syntax, TestMode.semantic, TestMode.exec]:
for d in TEST_MODES:
os.mkdir(os.path.join(self.test_dir, d))
self.only_incorrect_tests = only_incorrect_tests
......@@ -92,7 +94,8 @@ class Environment:
:return: (out, err, return code)
"""
mode_flag = {
TestMode.syntax: "--parsetest"
TestMode.syntax: "--parsetest",
TestMode.ast: "--parse-ast"
}[mode]
cmd = [self.mj_run_cmd, mode_flag] + list(args)
return execute(cmd, timeout=self.timeout)
......
import shutil, logging
from mjtest.environment import Environment, TestMode
from mjtest.test.tests import TestCase, BasicDiffTestResult
from os import path
_LOG = logging.getLogger("tests")
class ASTDiffTest(TestCase):
FILE_ENDINGS = [".invalid.mj", ".valid.mj", ".mj"]
OUTPUT_FILE_ENDING = ".out"
MODE = TestMode.ast
def __init__(self, env: Environment, type: str, file: str):
super().__init__(env, type, file)
self._should_succeed = not file.endswith(".invalid.mj")
self._expected_output_file = file + self.OUTPUT_FILE_ENDING
self._has_expected_output_file = path.exists(self._expected_output_file)
def should_succeed(self) -> bool:
return self._should_succeed
def short_name(self) -> str:
return path.basename(self.file)
def run(self) -> BasicDiffTestResult:
out, err, rtcode = self.env.run_mj_command(self.MODE, self.file)
exp_out = ""
if rtcode > 0 and self.should_succeed():
if self._has_expected_output_file:
with open(self._expected_output_file, "r") as f:
exp_out = f.read()
else:
_LOG.error("Expected output file for test case {}:{} is missing.".format(self.MODE, self.short_name()))
return BasicDiffTestResult(self, rtcode, out.decode(), err.decode(), exp_out)
TestCase.TEST_CASE_CLASSES[TestMode.ast].append(ASTDiffTest)
\ No newline at end of file
......@@ -11,6 +11,7 @@ from mjtest.util.parallelism import available_cpu_count
from mjtest.util.utils import cprint, colored
from pprint import pprint
import shutil
import difflib
_LOG = logging.getLogger("tests")
......@@ -40,7 +41,7 @@ class TestSuite:
if exists(dir):
self._load_test_case_dir(type, dir)
else:
_LOG.warn("Test folder {} doesn't exist".format(dir))
_LOG.info("Test folder {} doesn't exist".format(dir))
def _load_test_case_dir(self, mode: str, dir: str):
self.test_cases[mode] = []
......@@ -175,10 +176,7 @@ class TestCase:
A single test case.
"""
TEST_CASE_CLASSES = {
TestMode.syntax: [],
TestMode.semantic: []
}
TEST_CASE_CLASSES = dict((k, []) for k in TEST_MODES)
FILE_ENDINGS = []
def __init__(self, env: Environment, type: str, file: str):
......@@ -192,14 +190,8 @@ class TestCase:
def can_run(self, mode: str = "") -> bool:
mode = mode or self.env.mode
if mode == TestMode.exec:
return self.type == TestMode.exec
if mode == TestMode.semantic:
return self.type == TestMode.semantic \
or (self.type == TestMode.exec and self.should_succeed())
if mode == TestMode.syntax:
return self.type == TestMode.syntax or \
(self.can_run(TestMode.semantic) and self.should_succeed())
return self.type == mode or \
(self.type in TEST_MODES[TEST_MODES.index(self.env.mode):] and self.should_succeed())
def run(self) -> 'TestResult':
raise NotImplementedError()
......@@ -324,4 +316,43 @@ Return code: {}
arr = ["[{:04d}] {:s}".format(i + 1, l) for (i, l) in enumerate(arr)]
return "\n".join(arr)
import mjtest.test.syntax_tests
\ No newline at end of file
class BasicDiffTestResult(BasicTestResult):
def __init__(self, test_case: TestCase, error_code: int, output: str, error_output: str, expected_output: str):
super().__init__(test_case, error_code, output, error_output)
self.expected_output = expected_output
self._is_output_correct = self.expected_output.strip() == self.output
if self.is_correct():
self.add_additional_text("Expected and actual output", self.output)
elif self.succeeded() and self.test_case.should_succeed():
self.add_additional_text("Diff[expected output, actual output]", self._output_diff())
self.add_additional_text("Expected output", self.expected_output)
self.add_additional_text("Actual output", self.output)
def is_correct(self):
if self.succeeded():
return super().is_correct() and self.is_output_correct()
else:
return super().is_correct() and self._contains_error_str
def _output_diff(self) -> str:
return difflib.Differ().compare(self.expected_output, self.output)
def is_output_correct(self) -> str:
return self._is_output_correct
def short_message(self) -> str:
if self.is_correct():
return "correct"
else:
if not self.succeeded() and not self.test_case.should_succeed() and not self._contains_error_str:
return "the error output doesn't contain the word \"error\""
if self.succeeded() and self.test_case.should_succeed():
return "the actual output differs from the expected"
return "incorrect return code"
import mjtest.test.syntax_tests
import mjtest.test.ast_tests
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment