diff --git a/README.mdwn b/README.mdwn index 3863195caa61801898b621ec4d76df7e74bb6a08..1ef195cf2bdcf6bcf5a5fa031d853d8beb1e6b29 100644 --- a/README.mdwn +++ b/README.mdwn @@ -16,7 +16,7 @@ The test cases are divided in 5 'modes': - __lexer__: Test cases that check the lexed token (and their correct output) - __syntax__: Test cases that just check whether `./run --parsecheck` accepts as correct or rejects them. -- __ast__: Test cases that check the generated ast. +- __ast__: Test cases that check the generated ast by using the pretty printing functionality. - __semantic__: Test cases that check semantic checking of MiniJava programs - __exec__: Test cases that check the correct compilation of MiniJava programs. @@ -74,14 +74,13 @@ Test types for the ast mode File ending(s) of test casesExpected behaviour to complete a test of this type .valid.mj .mj - Return code is 0 and the output matches the expected output (located in the file `[test file].out` - - - .invalid.mj - Return code is > 0 and the error output contains the word error + Pretty printing the source file should result in the same output as pretty printing the already pretty printed file. + The sorted lexer output for the last should be the same as the sorted lexer output for the source file. +It uses all syntax mode tests implicitly. + Test runner ----------- diff --git a/mjtest/environment.py b/mjtest/environment.py index 23dd3d44498c185de873b66183a65c217e858f16..bc689208cef0e3080fefc9991b4ba2c7dadbc56c 100644 --- a/mjtest/environment.py +++ b/mjtest/environment.py @@ -20,6 +20,10 @@ class TestMode: exec = "exec" + USE_TESTS_OF_OTHER = { + ast: [syntax] + } + """ All 'success' tests of the n.th mode can used as 'success' tests for the n-1.th mode""" TEST_MODES = [TestMode.lexer, TestMode.syntax, TestMode.ast, TestMode.semantic, TestMode.exec] @@ -102,7 +106,7 @@ class Environment: mode_flag = { TestMode.lexer: "--lextest", TestMode.syntax: "--parsetest", - TestMode.ast: "--parse-ast" + TestMode.ast: "--pretty-print" }[mode] cmd = [self.mj_run_cmd, mode_flag] + list(args) return execute(cmd, timeout=self.timeout) diff --git a/mjtest/test/ast_tests.py b/mjtest/test/ast_tests.py index fe08a5c276c46196f820d2c8623a5518ca0b528f..3990f9d054174fe756fdf5e60672d9e439fd46e0 100644 --- a/mjtest/test/ast_tests.py +++ b/mjtest/test/ast_tests.py @@ -1,45 +1,72 @@ +import difflib +import os import shutil, logging +from typing import Tuple from mjtest.environment import Environment, TestMode +from mjtest.test.syntax_tests import BasicSyntaxTest from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult from os import path _LOG = logging.getLogger("tests") -class ASTDiffTest(TestCase): +class ASTPrettyPrintTest(BasicSyntaxTest): - FILE_ENDINGS = [".invalid.mj", ".valid.mj", ".mj"] - OUTPUT_FILE_ENDING = ".out" - MODE = TestMode.ast + FILE_ENDINGS = [".mj", ".valid.mj"] + INVALID_FILE_ENDINGS = [".invalid.mj"] def __init__(self, env: Environment, type: str, file: str): super().__init__(env, type, file) - self._should_succeed = not file.endswith(".invalid.mj") - self._expected_output_file = file + self.OUTPUT_FILE_ENDING - self._has_expected_output_file = path.exists(self._expected_output_file) - - def should_succeed(self) -> bool: - return self._should_succeed - - def short_name(self) -> str: - return path.basename(self.file) - - def run(self) -> BasicDiffTestResult: - out, err, rtcode = self.env.run_mj_command(self.MODE, self.file) - exp_out = "" - if rtcode == 0 and self.should_succeed(): - if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE: - with open(self._expected_output_file, "r") as f: - exp_out = f.read() - #else: - # _LOG.error("Expected output file for test case {}:{} is missing.".format(self.MODE, self.short_name())) - if self.type == self.MODE and self.env.mode == self.MODE: - return BasicDiffTestResult(self, rtcode, out.decode(), err.decode(), exp_out) - return BasicTestResult(self, rtcode, out.decode(), err.decode()) - -class LexerDiffTest(ASTDiffTest): - - MODE = TestMode.lexer - -TestCase.TEST_CASE_CLASSES[TestMode.ast].append(ASTDiffTest) -TestCase.TEST_CASE_CLASSES[TestMode.lexer].append(LexerDiffTest) \ No newline at end of file + + def run(self) -> BasicTestResult: + tmp_file = self.env.create_tmpfile() + rtcode, out, err = self._pretty_print(self.file, tmp_file) + if rtcode > 0: + os.remove(tmp_file) + return BasicTestResult(self, rtcode, out, err) + _file = self.file + tmp_file2 = self.env.create_tmpfile() + rtcode, out2, err2 = self._pretty_print(tmp_file, tmp_file2) + if rtcode > 0: + os.remove(tmp_file2) + btr = BasicTestResult(self, rtcode, out2, err2) + btr.add_additional_text("Prior out", out) + btr.add_additional_text("Prior err", err) + return btr + rtcode_lex, out_lex, err_lex = self.env.run_mj_command(TestMode.lexer, self.file) + rtcode_lex2, out_lex2, err_lex2 = self.env.run_mj_command(TestMode.lexer, tmp_file2) + os.remove(tmp_file2) + out_lex = self._sort_lexed(out_lex.decode()) + out_lex2 = self._sort_lexed(out_lex2.decode()) + incorrect_msg, rtcode = "", 0 + if rtcode_lex + rtcode_lex2: + incorrect_msg, rtcode = "Lexing failed", 1 + elif out != out2: + incorrect_msg, rtcode = "Not idempotent", 1 + elif out_lex != out_lex2: + incorrect_msg, rtcode = "Sorted and lexed second pretty print differs from original", 1 + btr = BasicTestResult(self, rtcode, incorrect_msg=incorrect_msg) + btr.add_additional_text("First round output", out) + btr.add_additional_text("Second round output", out2) + btr.add_additional_text("Diff", self._diff(out, out2)) + btr.add_additional_text("Original file, sorted and lexed", out_lex) + btr.add_additional_text("Second round output, sorted and lexed", out_lex2) + btr.add_additional_text("Diff", self._diff(out_lex, out_lex2)) + return btr + + def _diff(self, first: str, second: str) -> str: + return "".join(difflib.Differ().compare(first.splitlines(True), second.splitlines(True))) + + def _sort_lexed(self, lexed: str) -> str: + #return "".join(difflib.Differ().compare(self.expected_output.splitlines(True), self.output.splitlines(True))) + return "".join(sorted(lexed.splitlines(True))) + + + + def _pretty_print(self, input_file: str, output_file: str) -> Tuple[int, str, str]: + out, err, rtcode = self.env.run_mj_command(TestMode.ast, input_file) + with open(output_file, "w") as f: + print(out, file=f) + return rtcode, out.decode(), err.decode() + +TestCase.TEST_CASE_CLASSES["ast"].append(ASTPrettyPrintTest) \ No newline at end of file diff --git a/mjtest/test/tests.py b/mjtest/test/tests.py index 029f7a7c916bbe47127cec66e79ee47f01b9b8dd..cf107d13510189c113e544b5d9bd6c5d62fc75fd 100644 --- a/mjtest/test/tests.py +++ b/mjtest/test/tests.py @@ -31,7 +31,11 @@ class TestSuite: self._load_test_cases() def _load_test_cases(self): - types = TEST_MODES#TEST_MODES[TEST_MODES.index(self.env.mode):] + types = [self.env.mode]#TEST_MODES#TEST_MODES[TEST_MODES.index(self.env.mode):] + if self.env.ci_testing: + types = TEST_MODES + elif self.env.mode in TestMode.USE_TESTS_OF_OTHER: + types += TestMode.USE_TESTS_OF_OTHER[self.env.mode] for type in types: self._load_test_case_type(type) @@ -54,21 +58,27 @@ class TestSuite: if len(t) > 0: self.correct_test_cases[mode].add(t) correct_test_cases.add(t) + m = mode + if m != self.env.mode and self.env.mode in TestMode.USE_TESTS_OF_OTHER and \ + m in TestMode.USE_TESTS_OF_OTHER[self.env.mode]: + m = self.env.mode for file in sorted(os.listdir(dir)): - if not TestCase.has_valid_file_ending(mode, file): + if not TestCase.has_valid_file_ending(m, file): _LOG.debug("Skip file " + file) elif self.env.only_incorrect_tests and file in correct_test_cases: _LOG.info("Skip file {} as its test case was executed correctly the last run") else: - test_case = TestCase.create_from_file(self.env, mode, join(dir, file)) - if not test_case.can_run(): + test_case = TestCase.create_from_file(self.env, m, join(dir, file)) + if not test_case: + pass + elif not test_case.can_run(): _LOG.debug("Skip test case '{}' because it isn't suited".format(test_case.name())) else: - if mode not in self.test_cases: - self.test_cases[mode] = [] - self.test_cases[mode].append(test_case) - if mode in self.test_cases and len(self.test_cases[mode]) == 0: - del self.test_cases[mode] + if m not in self.test_cases: + self.test_cases[m] = [] + self.test_cases[m].append(test_case) + if m in self.test_cases and len(self.test_cases[m]) == 0: + del self.test_cases[m] def _log_file_for_type(self, type: str): return join(self.env.test_dir, type, ".mjtest_correct_testcases_" + self.env.mode) @@ -185,6 +195,7 @@ class TestCase: TEST_CASE_CLASSES = dict((k, []) for k in TEST_MODES) FILE_ENDINGS = [] + INVALID_FILE_ENDINGS = [] def __init__(self, env: Environment, type: str, file: str): self.env = env @@ -196,13 +207,14 @@ class TestCase: def can_run(self, mode: str = "") -> bool: mode = mode or self.env.mode + same_mode = self.type == mode types = TEST_MODES[TEST_MODES.index(self.env.mode):] if self.env.ci_testing: - return self.type == mode or \ + return same_mode or \ (self.type in types and self.should_succeed()) or \ (self.type not in types and not self.should_succeed()) else: - return self.type == mode + return same_mode def run(self) -> 'TestResult': raise NotImplementedError() @@ -222,7 +234,8 @@ class TestCase: @classmethod def _test_case_class_for_file(cls, type: str, file: str): for t in cls.TEST_CASE_CLASSES[type]: - if any(file.endswith(e) for e in t.FILE_ENDINGS): + if any(file.endswith(e) for e in t.FILE_ENDINGS) and \ + not any(file.endswith(e) for e in t.INVALID_FILE_ENDINGS): return t return False @@ -257,12 +270,18 @@ class TestResult: class BasicTestResult(TestResult): - def __init__(self, test_case: TestCase, error_code: int, output: str, error_output: str): + def __init__(self, test_case: TestCase, error_code: int, output: str = None, error_output: str = None, + incorrect_msg: str = "incorrect return code"): super().__init__(test_case, error_code) + self._incorrect_msg = incorrect_msg self._contains_error_str = "error" in error_output self.error_output = error_output self.output = output self.other_texts = [] # type: List[Tuple[str, str, bool]] + if output: + self.add_additional_text("Output", output) + if error_output: + self.add_additional_text("Error output", error_output) def is_correct(self): if self.succeeded(): @@ -276,7 +295,7 @@ class BasicTestResult(TestResult): else: if not self.succeeded() and not self.test_case.should_succeed() and not self._contains_error_str: return "the error output doesn't contain the word \"error\"" - return "incorrect return code" + return self._incorrect_msg def long_message(self) -> str: file_content = [] @@ -299,19 +318,10 @@ Source file: {} -Output: - -{} - -Error output: - -{} - Return code: {} {} -""".format(self.short_message().capitalize(), self._ident(file_content), - self._ident(self.output), self._ident(self.error_output), self.error_code, +""".format(self.short_message().capitalize(), self._ident(file_content), self.error_code, "\n".join(others)) def add_additional_text(self, title: str, content: str): @@ -364,6 +374,43 @@ class BasicDiffTestResult(BasicTestResult): return "incorrect return code" +class DiffTest(TestCase): + + FILE_ENDINGS = [".invalid.mj", ".valid.mj", ".mj"] + OUTPUT_FILE_ENDING = ".out" + MODE = TestMode.ast + + def __init__(self, env: Environment, type: str, file: str): + super().__init__(env, type, file) + self._should_succeed = not file.endswith(".invalid.mj") + self._expected_output_file = file + self.OUTPUT_FILE_ENDING + self._has_expected_output_file = exists(self._expected_output_file) + + def should_succeed(self) -> bool: + return self._should_succeed + + def short_name(self) -> str: + return basename(self.file) + + def run(self) -> BasicDiffTestResult: + out, err, rtcode = self.env.run_mj_command(self.MODE, self.file) + exp_out = "" + if rtcode == 0 and self.should_succeed(): + if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE: + with open(self._expected_output_file, "r") as f: + exp_out = f.read() + #else: + # _LOG.error("Expected output file for test case {}:{} is missing.".format(self.MODE, self.short_name())) + if self.type == self.MODE and self.env.mode == self.MODE: + return BasicDiffTestResult(self, rtcode, out.decode(), err.decode(), exp_out) + return BasicTestResult(self, rtcode, out.decode(), err.decode()) + + +class LexerDiffTest(DiffTest): + + MODE = TestMode.lexer + +TestCase.TEST_CASE_CLASSES[TestMode.lexer].append(LexerDiffTest) import mjtest.test.syntax_tests import mjtest.test.ast_tests \ No newline at end of file