Commit 990a1dbb authored by Johannes Bechberger's avatar Johannes Bechberger

Add tests and improve test runner

parent 4c70f2c3
...@@ -40,6 +40,8 @@ if True:#__name__ == '__main__': ...@@ -40,6 +40,8 @@ if True:#__name__ == '__main__':
help="Produce no long reports besides the command line output") help="Produce no long reports besides the command line output")
parser.add_argument("--parallel", action="store_true", default=False, parser.add_argument("--parallel", action="store_true", default=False,
help="Run the tests in parallel") help="Run the tests in parallel")
parser.add_argument("--output_incorrect_reports", action="store_true", default=True,
help="Output the long report for every incorrect test case")
#parser.add_argument("--timeout", action="store_const", default=30, const="timeout", #parser.add_argument("--timeout", action="store_const", default=30, const="timeout",
# help="Abort a program after TIMEOUT seconds") # help="Abort a program after TIMEOUT seconds")
#parser.add_argument("--report_dir", action="store_const", default="", const="report_dir", #parser.add_argument("--report_dir", action="store_const", default="", const="report_dir",
......
...@@ -30,7 +30,7 @@ class Environment: ...@@ -30,7 +30,7 @@ class Environment:
def __init__(self, mode, mj_run: str, tmp_dir: str = "", test_dir: str = "", def __init__(self, mode, mj_run: str, tmp_dir: str = "", test_dir: str = "",
only_incorrect_tests: bool = False, parallel: bool = False, only_incorrect_tests: bool = False, parallel: bool = False,
timeout: int = 30, report_dir: str = "", log_level: str = "warn", timeout: int = 30, report_dir: str = "", log_level: str = "warn",
produce_no_reports: bool = True): produce_no_reports: bool = True, output_incorrect_reports: bool = True):
self.mode = mode self.mode = mode
self.mj_run_cmd = os.path.realpath(mj_run) self.mj_run_cmd = os.path.realpath(mj_run)
...@@ -70,6 +70,7 @@ class Environment: ...@@ -70,6 +70,7 @@ class Environment:
self.report_dir = None self.report_dir = None
logging.basicConfig(level=self.LOG_LEVELS[log_level]) logging.basicConfig(level=self.LOG_LEVELS[log_level])
self.produce_reports = not produce_no_reports # type: bool self.produce_reports = not produce_no_reports # type: bool
self.output_incorrect_reports = True
def create_tmpfile(self) -> str: def create_tmpfile(self) -> str:
return os.path.join(self.tmp_dir, str(os.times())) return os.path.join(self.tmp_dir, str(os.times()))
......
...@@ -146,6 +146,9 @@ class TestSuite: ...@@ -146,6 +146,9 @@ class TestSuite:
os.mkdir(rep_dir) os.mkdir(rep_dir)
suffix = ".correct" if ret.is_correct() else ".incorrect" suffix = ".correct" if ret.is_correct() else ".incorrect"
ret.store_at(join(rep_dir, test_case.short_name() + suffix)) ret.store_at(join(rep_dir, test_case.short_name() + suffix))
if self.env.output_incorrect_reports and not ret.is_correct():
print(colored("Report for failing test case {}".format(test_case.short_name()), "red", attrs=["bold"]))
print(colored(ret.long_message(), "red"))
return ret return ret
except IOError: except IOError:
_LOG.exception("Caught i/o error while trying to store the report for '{}'" _LOG.exception("Caught i/o error while trying to store the report for '{}'"
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment