Add aditional options to the test runner

README.mdwn

@@ -91,7 +91,8 @@ Output of the `./mjt.py --help`
 
 ```
 usage: mjt.py [-h] [--only_incorrect_tests] [--produce_no_reports]
-              [--parallel] [--log_level LOG_LEVEL]
+              [--produce_all_reports] [--parallel]
+              [--output_no_incorrect_reports] [--log_level LOG_LEVEL]
               {syntax,ast,semantic,exec} MJ_RUN
 
 MiniJava test runner
@@ -109,8 +110,13 @@ optional arguments:
                         Only run the tests that were incorrect the last run
   --produce_no_reports  Produce no long reports besides the command line
                         output
+  --produce_all_reports
+                        Produce reports also for correct test cases
   --parallel            Run the tests in parallel
-  --log LOG             Logging level (error, warn, info or debug)
+  --output_no_incorrect_reports
+                        Output the long report for every incorrect test case
+  --log_level LOG_LEVEL
+                        Logging level (error, warn, info or debug)
 ```
 
 For Windows users: Using environment variables doesn't work and you have to use prefix `./mjt.py` with `python`.
@@ -127,7 +133,7 @@ This will…
 [FAIL   ] syntax:test.invalid                     the error output doesn't contain the word "error"
 ----------------------------------------
 Ran 1 tests, of which 1 failed.
-A full report for each test can be found at mjtest/reports/30.10.16:08:05:10.failed
+A full report for each test can be found at mjtest/reports/30-10-16_08-05-10.failed
 ```
 - … log that some test cases were executed correctly
 - … return with an error code of `0` if all tests executed correct

mjtest/cli.py

@@ -38,9 +38,11 @@ if True:#__name__ == '__main__':
                         help="Only run the tests that were incorrect the last run")
     parser.add_argument("--produce_no_reports", action="store_true", default=False,
                         help="Produce no long reports besides the command line output")
+    parser.add_argument("--produce_all_reports", action="store_true", default=False,
+                        help="Produce reports also for correct test cases")
     parser.add_argument("--parallel", action="store_true", default=False,
                         help="Run the tests in parallel")
-    parser.add_argument("--output_incorrect_reports", action="store_true", default=True,
+    parser.add_argument("--output_no_incorrect_reports", action="store_true", default=False,
                         help="Output the long report for every incorrect test case")
     #parser.add_argument("--timeout", action="store_const", default=30, const="timeout",
     #                    help="Abort a program after TIMEOUT seconds")

mjtest/environment.py

@@ -32,7 +32,8 @@ class Environment:
     def __init__(self, mode, mj_run: str, tmp_dir: str = "", test_dir: str = "",
                  only_incorrect_tests: bool = False, parallel: bool = False,
                  timeout: int = 30, report_dir: str = "", log_level: str = "warn",
-                 produce_no_reports: bool = True, output_incorrect_reports: bool = True):
+                 produce_no_reports: bool = True, output_no_incorrect_reports: bool = False,
+                 produce_all_reports: bool = False):
         self.mode = mode
         self.mj_run_cmd = os.path.realpath(mj_run)
 
@@ -60,19 +61,19 @@ class Environment:
         if not produce_no_reports:
             if report_dir:
                 self.report_dir = os.path.abspath(os.path.expandvars(report_dir))
-                if not os.path.exists(report_dir):
-                    os.mkdir(self.report_dir)
             else:
                 self.report_dir = os.path.join(get_mjtest_basedir(), "reports")
-                if not os.path.exists(self.report_dir):
-                    os.mkdir(self.report_dir)
+            try:
+                os.mkdir(self.report_dir)
+            except IOError:
+                pass
             self.report_dir = os.path.join(self.report_dir, datetime.now().strftime("%d-%m-%y_%H-%M-%S"))
-            os.mkdir(self.report_dir)
         else:
             self.report_dir = None
         logging.basicConfig(level=self.LOG_LEVELS[log_level])
         self.produce_reports = not produce_no_reports # type: bool
-        self.output_incorrect_reports = True
+        self.output_incorrect_reports = not output_no_incorrect_reports
+        self.produce_all_reports = produce_all_reports
 
     def create_tmpfile(self) -> str:
         return os.path.join(self.tmp_dir, str(os.times()))

mjtest/test/tests.py

@@ -94,7 +94,7 @@ class TestSuite:
                       colored("{} failed.".format(ret.failed), "red", attrs=["bold"]))
             else:
                 cprint("All {} run tests succeeded".format(ret.count), "green")
-            if self.env.produce_reports:
+            if self.env.produce_reports and (self.env.produce_all_reports or ret.failed > 0):
                 report_dir = self.env.report_dir + "." + ("successful" if ret.failed == 0 else "failed")
                 os.rename(self.env.report_dir, report_dir)
                 print("A full report for each test can be found at {}".format(
@@ -139,12 +139,15 @@ class TestSuite:
                 tc=test_case.name()), color, attrs=["bold"]) +
                   colored("" if ret.is_correct() else ret.short_message(), color))
             try:
-                if self.env.produce_reports:
+                if self.env.produce_reports and (self.env.produce_all_reports or not ret.is_correct()):
                     if not exists(self.env.report_dir):
                         os.mkdir(self.env.report_dir)
                     rep_dir = join(self.env.report_dir, test_case.type)
                     if not exists(rep_dir):
-                        os.mkdir(rep_dir)
+                        try:
+                            os.mkdir(rep_dir)
+                        except IOError:
+                            pass
                     suffix = ".correct" if ret.is_correct() else ".incorrect"
                     ret.store_at(join(rep_dir, test_case.short_name() + suffix))
                 if self.env.output_incorrect_reports and not ret.is_correct():