Add tests and improve test runner

mjtest/cli.py

@@ -40,6 +40,8 @@ if True:#__name__ == '__main__':
                         help="Produce no long reports besides the command line output")
     parser.add_argument("--parallel", action="store_true", default=False,
                         help="Run the tests in parallel")
+    parser.add_argument("--output_incorrect_reports", action="store_true", default=True,
+                        help="Output the long report for every incorrect test case")
     #parser.add_argument("--timeout", action="store_const", default=30, const="timeout",
     #                    help="Abort a program after TIMEOUT seconds")
     #parser.add_argument("--report_dir", action="store_const", default="", const="report_dir",

mjtest/environment.py

@@ -30,7 +30,7 @@ class Environment:
     def __init__(self, mode, mj_run: str, tmp_dir: str = "", test_dir: str = "",
                  only_incorrect_tests: bool = False, parallel: bool = False,
                  timeout: int = 30, report_dir: str = "", log_level: str = "warn",
-                 produce_no_reports: bool = True):
+                 produce_no_reports: bool = True, output_incorrect_reports: bool = True):
         self.mode = mode
         self.mj_run_cmd = os.path.realpath(mj_run)
 
@@ -70,6 +70,7 @@ class Environment:
             self.report_dir = None
         logging.basicConfig(level=self.LOG_LEVELS[log_level])
         self.produce_reports = not produce_no_reports # type: bool
+        self.output_incorrect_reports = True
 
     def create_tmpfile(self) -> str:
         return os.path.join(self.tmp_dir, str(os.times()))

mjtest/test/tests.py

@@ -146,6 +146,9 @@ class TestSuite:
                         os.mkdir(rep_dir)
                     suffix = ".correct" if ret.is_correct() else ".incorrect"
                     ret.store_at(join(rep_dir, test_case.short_name() + suffix))
+                if self.env.output_incorrect_reports and not ret.is_correct():
+                    print(colored("Report for failing test case {}".format(test_case.short_name()), "red", attrs=["bold"]))
+                    print(colored(ret.long_message(), "red"))
                 return ret
             except IOError:
                 _LOG.exception("Caught i/o error while trying to store the report for '{}'"