Add lexer mode and "all" option

mjtest/cli.py

@@ -2,28 +2,31 @@ import logging
 import os
 import sys
 import argparse
+from typing import Dict
 from mjtest.environment import TestMode, Environment, TEST_MODES
 from mjtest.test.tests import TestSuite
 
 # adapted from http://stackoverflow.com/a/8527629
+from mjtest.util.utils import cprint, colored
+
+
 class LogLevelChoices(argparse.Action):
     CHOICES = ["error", "warn", "info", "debug"]
-    def __call__(self, parser, namespace, values, option_string=None):
-        if values:
-            for value in values:
-                if value not in self.CHOICES:
-                    message = ("invalid choice: {0!r} (choose from {1})"
-                               .format(value,
-                                       ', '.join([repr(action)
-                                                  for action in self.CHOICES])))
+    def __call__(self, parser, namespace, value, option_string=None):
+        if value:
+            if value not in self.CHOICES:
+                message = ("invalid choice: {0!r} (choose from {1})"
+                           .format(value,
+                                   ', '.join([repr(action)
+                                              for action in self.CHOICES])))
 
-                    raise argparse.ArgumentError(self, message)
-            setattr(namespace, self.dest, values)
+                raise argparse.ArgumentError(self, message)
+            setattr(namespace, self.dest, value)
 
 if True:#__name__ == '__main__':
     parser = argparse.ArgumentParser(description="MiniJava test runner", add_help=True)
     parser.add_argument("mode",
-                        choices=TEST_MODES,
+                        choices=["all"] + TEST_MODES,
                         help="What do you want to test?")
     if os.getenv("MJ_RUN", None) is None:
         parser.add_argument("mj_run",
@@ -51,17 +54,34 @@ if True:#__name__ == '__main__':
     parser.add_argument("--log_level", action=LogLevelChoices, default="warn", const="log_level",
                         help="Logging level (error, warn, info or debug)")
     args = vars(parser.parse_args())
-    if os.getenv("MJ_RUN", None) is not None:
-        args["mj_run"] = os.getenv("MJ_RUN")
-    suite = TestSuite(Environment(**args))
-    ret = None
-    try:
-        ret = suite.run()
-    finally:
-        suite.env.clean_up()
-        suite.store()
 
-    if ret is None or ret.failed > 0:
-        sys.exit(1)
+    failed = 0
+    count = 0
+
+    def run(args: Dict[str, str]):
+        global failed, count
+        if os.getenv("MJ_RUN", None) is not None:
+            args["mj_run"] = os.getenv("MJ_RUN")
+        suite = TestSuite(Environment(**args))
+        ret = None
+        try:
+            ret = suite.run()
+        finally:
+            suite.env.clean_up()
+            suite.store()
+        if ret is not None:
+            failed += ret.failed
+            count += ret.count
+    if args["mode"] == "all":
+        for mode in TEST_MODES:
+            cprint("Run {} tests".format(mode), attrs=["bold"])
+            args["mode"] = mode
+            run(args)
+        if failed > 0:  # some tests failed
+            print(colored("Ran {} tests, of which ".format(count), "red") +
+                  colored("{} failed.".format(failed), "red", attrs=["bold"]))
+        else:
+            cprint("All {} run tests succeeded".format(count), "green")
     else:
-        sys.exit(0)
+        run(args)
+    sys.exit(1 if failed > 0 else 0)

mjtest/environment.py

@@ -9,6 +9,9 @@ from typing import Tuple, List
 
 
 class TestMode:
+
+    lexer = "lexer"
+
     syntax = "syntax"
 
     ast = "ast"
@@ -18,7 +21,7 @@ class TestMode:
     exec = "exec"
 
 """ All 'success' tests of the n.th mode can used as 'success' tests for the n-1.th mode"""
-TEST_MODES = [TestMode.syntax, TestMode.ast, TestMode.semantic, TestMode.exec]
+TEST_MODES = [TestMode.lexer, TestMode.syntax, TestMode.ast, TestMode.semantic, TestMode.exec]
 
 class Environment:
 
@@ -95,6 +98,7 @@ class Environment:
         :return: (out, err, return code)
         """
         mode_flag = {
+            TestMode.lexer: "--lextest",
             TestMode.syntax: "--parsetest",
             TestMode.ast: "--parse-ast"
         }[mode]

mjtest/test/ast_tests.py

@@ -28,11 +28,17 @@ class ASTDiffTest(TestCase):
         out, err, rtcode = self.env.run_mj_command(self.MODE, self.file)
         exp_out = ""
         if rtcode > 0 and self.should_succeed():
-            if self._has_expected_output_file:
+            if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE:
                 with open(self._expected_output_file, "r") as f:
                     exp_out = f.read()
-            else:
-                _LOG.error("Expected output file for test case {}:{} is missing.".format(self.MODE, self.short_name()))
+            #else:
+            #    _LOG.error("Expected output file for test case {}:{} is missing.".format(self.MODE, self.short_name()))
         return BasicDiffTestResult(self, rtcode, out.decode(), err.decode(), exp_out)
 
-TestCase.TEST_CASE_CLASSES[TestMode.ast].append(ASTDiffTest)
\ No newline at end of file
+
+class LexerDiffTest(ASTDiffTest):
+
+    MODE = TestMode.lexer
+
+TestCase.TEST_CASE_CLASSES[TestMode.ast].append(ASTDiffTest)
+TestCase.TEST_CASE_CLASSES[TestMode.lexer].append(LexerDiffTest)
\ No newline at end of file

mjtest/test/tests.py

@@ -31,7 +31,7 @@ class TestSuite:
         self._load_test_cases()
 
     def _load_test_cases(self):
-        types = TEST_MODES[TEST_MODES.index(self.env.mode):]
+        types = TEST_MODES#TEST_MODES[TEST_MODES.index(self.env.mode):]
         for type in types:
             self._load_test_case_type(type)
 
@@ -44,7 +44,6 @@ class TestSuite:
             _LOG.info("Test folder {} doesn't exist".format(dir))
 
     def _load_test_case_dir(self, mode: str, dir: str):
-        self.test_cases[mode] = []
         correct_test_cases = set()
         log_file = self._log_file_for_type(mode)
         if exists(log_file):
@@ -61,12 +60,14 @@ class TestSuite:
             elif self.env.only_incorrect_tests and file in correct_test_cases:
                 _LOG.info("Skip file {} as its test case was executed correctly the last run")
             else:
-                test_case = TestCase.create_from_file(self.env, self.env.mode, join(dir, file))
+                test_case = TestCase.create_from_file(self.env, mode, join(dir, file))
                 if not test_case.can_run():
                     _LOG.debug("Skip test case '{}' because it isn't suited".format(test_case.name()))
                 else:
+                    if mode not in self.test_cases:
+                        self.test_cases[mode] = []
                     self.test_cases[mode].append(test_case)
-        if len(self.test_cases[mode]) == 0:
+        if mode in self.test_cases and len(self.test_cases[mode]) == 0:
             del self.test_cases[mode]
 
     def _log_file_for_type(self, type: str):
@@ -96,7 +97,10 @@ class TestSuite:
                 cprint("All {} run tests succeeded".format(ret.count), "green")
             if self.env.produce_reports and (self.env.produce_all_reports or ret.failed > 0):
                 report_dir = self.env.report_dir + "." + ("successful" if ret.failed == 0 else "failed")
-                os.rename(self.env.report_dir, report_dir)
+                try:
+                    os.rename(self.env.report_dir, report_dir)
+                except IOError:
+                    pass
                 print("A full report for each test can be found at {}".format(
                     os.path.relpath(report_dir)))
 
@@ -192,9 +196,10 @@ class TestCase:
 
     def can_run(self, mode: str = "") -> bool:
         mode = mode or self.env.mode
-
+        types = TEST_MODES[TEST_MODES.index(self.env.mode):]
         return self.type == mode or \
-               (self.type in TEST_MODES[TEST_MODES.index(self.env.mode):] and self.should_succeed())
+               (self.type in types and self.should_succeed()) or \
+               (self.type not in types and not self.should_succeed())
 
     def run(self) -> 'TestResult':
         raise NotImplementedError()