Fix "only_incorrect_testcases" bug

And add "--produce_no_reports" option

README.mdwn

@@ -68,8 +68,8 @@ To get colored output install the python3 module `termcolor`.
 Output of the `./mjt.py --help`
 
 ```
-usage: mjt.py [-h] [--only_incorrect_tests] [--parallel]
-              [--log_level LOG_LEVEL]
+usage: mjt.py [-h] [--only_incorrect_tests] [--produce_no_reports]
+              [--parallel] [--log_level LOG_LEVEL]
               {syntax,semantic,exec} MJ_RUN
 
 MiniJava test runner
@@ -85,6 +85,8 @@ optional arguments:
   -h, --help            show this help message and exit
   --only_incorrect_tests
                         Only run the tests that were incorrect the last run
+  --produce_no_reports  Produce no long reports besides the command line
+                        output
   --parallel            Run the tests in parallel
   --log LOG             Logging level (error, warn, info or debug)
 ```

mjtest/cli.py

@@ -36,6 +36,8 @@ if True:#__name__ == '__main__':
     #                    help="Directory that contains all test cases, default is the 'tests' directory")
     parser.add_argument("--only_incorrect_tests", action="store_true", default=False,
                         help="Only run the tests that were incorrect the last run")
+    parser.add_argument("--produce_no_reports", action="store_true", default=False,
+                        help="Produce no long reports besides the command line output")
     parser.add_argument("--parallel", action="store_true", default=False,
                         help="Run the tests in parallel")
     #parser.add_argument("--timeout", action="store_const", default=30, const="timeout",
@@ -54,7 +56,8 @@ if True:#__name__ == '__main__':
     finally:
         suite.env.clean_up()
         suite.store()
-    if ret is None or ret:
+
+    if ret is None or ret.failed > 0:
         sys.exit(1)
     else:
         sys.exit(0)

mjtest/environment.py

@@ -29,7 +29,8 @@ class Environment:
 
     def __init__(self, mode, mj_run: str, tmp_dir: str = "", test_dir: str = "",
                  only_incorrect_tests: bool = False, parallel: bool = False,
-                 timeout: int = 30, report_dir: str = "", log_level: str = "warn"):
+                 timeout: int = 30, report_dir: str = "", log_level: str = "warn",
+                 produce_no_reports: bool = True):
         self.mode = mode
         self.mj_run_cmd = os.path.realpath(mj_run)
 
@@ -54,18 +55,21 @@ class Environment:
         self.only_incorrect_tests = only_incorrect_tests
         self.parallel = parallel
         self.timeout = timeout
-
-        if report_dir:
-            self.report_dir = os.path.abspath(os.path.expandvars(report_dir))
-            if not os.path.exists(report_dir):
-                os.mkdir(self.report_dir)
+        if not produce_no_reports:
+            if report_dir:
+                self.report_dir = os.path.abspath(os.path.expandvars(report_dir))
+                if not os.path.exists(report_dir):
+                    os.mkdir(self.report_dir)
+            else:
+                self.report_dir = os.path.join(get_mjtest_basedir(), "reports")
+                if not os.path.exists(self.report_dir):
+                    os.mkdir(self.report_dir)
+            self.report_dir = os.path.join(self.report_dir, datetime.now().strftime("%d-%m-%y_%H-%M-%S"))
+            os.mkdir(self.report_dir)
         else:
-            self.report_dir = os.path.join(get_mjtest_basedir(), "reports")
-            if not os.path.exists(self.report_dir):
-                os.mkdir(self.report_dir)
-        self.report_dir = os.path.join(self.report_dir, datetime.now().strftime("%d-%m-%y_%H-%M-%S"))
-        os.mkdir(self.report_dir)
+            self.report_dir = None
         logging.basicConfig(level=self.LOG_LEVELS[log_level])
+        self.produce_reports = not produce_no_reports # type: bool
 
     def create_tmpfile(self) -> str:
         return os.path.join(self.tmp_dir, str(os.times()))

mjtest/test/tests.py

 from collections import namedtuple
 import shutil
 from typing import Optional, List, Tuple, T, Union, Dict
+import collections
 from mjtest.environment import Environment, TestMode, TEST_MODES
 from os.path import join, exists, basename
 import logging
@@ -25,7 +26,7 @@ class TestSuite:
     def __init__(self, env: Environment):
         self.env = env
         self.test_cases = {} # type: Dict[str, List[TestCase]]
-        self.correct_test_cases = {} # type: Dict[str, List[str]]
+        self.correct_test_cases = collections.defaultdict(set) # type: Dict[str, Set[str]]
         self._load_test_cases()
 
     def _load_test_cases(self):
@@ -47,7 +48,12 @@ class TestSuite:
         log_file = self._log_file_for_type(mode)
         if exists(log_file):
             with open(log_file) as f:
-                correct_test_cases = set(f.readlines())
+                correct_test_cases = set()
+                for t in f.readlines():
+                    t = t.strip()
+                    if len(t) > 0:
+                        self.correct_test_cases[mode].add(t)
+                        correct_test_cases.add(t)
         for file in sorted(os.listdir(dir)):
             if not TestCase.has_valid_file_ending(mode, file):
                 _LOG.debug("Skip file " + file)
@@ -66,7 +72,7 @@ class TestSuite:
         return join(self.env.test_dir, type, ".mjtest_correct_testcases")
 
     def _add_correct_test_case(self, test_case: 'TestCase'):
-        self.correct_test_cases[test_case.type].append(basename(test_case.file))
+        self.correct_test_cases[test_case.type].add(basename(test_case.file))
 
     def run(self) -> RunResult:
         ret = RunResult(0, 0)
@@ -87,10 +93,11 @@ class TestSuite:
                       colored("{} failed.".format(ret.failed), "red", attrs=["bold"]))
             else:
                 cprint("All {} run tests succeeded".format(ret.count), "green")
-            report_dir = self.env.report_dir + "." + ("successful" if ret.failed == 0 else "failed")
-            os.rename(self.env.report_dir, report_dir)
-            print("A full report for each test can be found at {}".format(
-                os.path.relpath(report_dir)))
+            if self.env.produce_reports:
+                report_dir = self.env.report_dir + "." + ("successful" if ret.failed == 0 else "failed")
+                os.rename(self.env.report_dir, report_dir)
+                print("A full report for each test can be found at {}".format(
+                    os.path.relpath(report_dir)))
 
             return ret
 
@@ -109,13 +116,18 @@ class TestSuite:
     def _func(self, test_case: 'TestCase'):
         ret = self._run_test_case(test_case)
         if ret is not False and ret.is_correct():
-            return 0, [test_case]
-        return 1, []
+            return 0, test_case
+        return 1, test_case
 
     def _run_parallel(self, mode: str, parallel_jobs: int) -> RunResult:
         pool = multiprocessing.Pool(parallel_jobs)
         rets = pool.map(self._func, self.test_cases[mode])
-        return RunResult(len(rets), sum(map(lambda x: x[0], rets)))
+        result = RunResult(len(rets), sum(map(lambda x: x[0], rets)))
+        for (suc, test_case) in rets:
+            if suc == 0:
+                self._add_correct_test_case(test_case)
+        return result
+
 
     def _run_test_case(self, test_case: 'TestCase') -> Optional['TestResult']:
         try:
@@ -126,13 +138,14 @@ class TestSuite:
                 tc=test_case.name()), color, attrs=["bold"]) +
                   colored("" if ret.is_correct() else ret.short_message(), color))
             try:
-                if not exists(self.env.report_dir):
-                    os.mkdir(self.env.report_dir)
-                rep_dir = join(self.env.report_dir, test_case.type)
-                if not exists(rep_dir):
-                    os.mkdir(rep_dir)
-                suffix = ".correct" if ret.is_correct() else ".incorrect"
-                ret.store_at(join(rep_dir, test_case.short_name() + suffix))
+                if self.env.produce_reports:
+                    if not exists(self.env.report_dir):
+                        os.mkdir(self.env.report_dir)
+                    rep_dir = join(self.env.report_dir, test_case.type)
+                    if not exists(rep_dir):
+                        os.mkdir(rep_dir)
+                    suffix = ".correct" if ret.is_correct() else ".incorrect"
+                    ret.store_at(join(rep_dir, test_case.short_name() + suffix))
                 return ret
             except IOError:
                 _LOG.exception("Caught i/o error while trying to store the report for '{}'"