Commit 9f96c9d8 authored by Johannes Bechberger's avatar Johannes Bechberger

Add aditional options to the test runner

parent fbb7ade3
...@@ -91,7 +91,8 @@ Output of the `./mjt.py --help` ...@@ -91,7 +91,8 @@ Output of the `./mjt.py --help`
``` ```
usage: mjt.py [-h] [--only_incorrect_tests] [--produce_no_reports] usage: mjt.py [-h] [--only_incorrect_tests] [--produce_no_reports]
[--parallel] [--log_level LOG_LEVEL] [--produce_all_reports] [--parallel]
[--output_no_incorrect_reports] [--log_level LOG_LEVEL]
{syntax,ast,semantic,exec} MJ_RUN {syntax,ast,semantic,exec} MJ_RUN
MiniJava test runner MiniJava test runner
...@@ -109,8 +110,13 @@ optional arguments: ...@@ -109,8 +110,13 @@ optional arguments:
Only run the tests that were incorrect the last run Only run the tests that were incorrect the last run
--produce_no_reports Produce no long reports besides the command line --produce_no_reports Produce no long reports besides the command line
output output
--produce_all_reports
Produce reports also for correct test cases
--parallel Run the tests in parallel --parallel Run the tests in parallel
--log LOG Logging level (error, warn, info or debug) --output_no_incorrect_reports
Output the long report for every incorrect test case
--log_level LOG_LEVEL
Logging level (error, warn, info or debug)
``` ```
For Windows users: Using environment variables doesn't work and you have to use prefix `./mjt.py` with `python`. For Windows users: Using environment variables doesn't work and you have to use prefix `./mjt.py` with `python`.
...@@ -127,7 +133,7 @@ This will… ...@@ -127,7 +133,7 @@ This will…
[FAIL ] syntax:test.invalid the error output doesn't contain the word "error" [FAIL ] syntax:test.invalid the error output doesn't contain the word "error"
---------------------------------------- ----------------------------------------
Ran 1 tests, of which 1 failed. Ran 1 tests, of which 1 failed.
A full report for each test can be found at mjtest/reports/30.10.16:08:05:10.failed A full report for each test can be found at mjtest/reports/30-10-16_08-05-10.failed
``` ```
- … log that some test cases were executed correctly - … log that some test cases were executed correctly
- … return with an error code of `0` if all tests executed correct - … return with an error code of `0` if all tests executed correct
......
...@@ -38,9 +38,11 @@ if True:#__name__ == '__main__': ...@@ -38,9 +38,11 @@ if True:#__name__ == '__main__':
help="Only run the tests that were incorrect the last run") help="Only run the tests that were incorrect the last run")
parser.add_argument("--produce_no_reports", action="store_true", default=False, parser.add_argument("--produce_no_reports", action="store_true", default=False,
help="Produce no long reports besides the command line output") help="Produce no long reports besides the command line output")
parser.add_argument("--produce_all_reports", action="store_true", default=False,
help="Produce reports also for correct test cases")
parser.add_argument("--parallel", action="store_true", default=False, parser.add_argument("--parallel", action="store_true", default=False,
help="Run the tests in parallel") help="Run the tests in parallel")
parser.add_argument("--output_incorrect_reports", action="store_true", default=True, parser.add_argument("--output_no_incorrect_reports", action="store_true", default=False,
help="Output the long report for every incorrect test case") help="Output the long report for every incorrect test case")
#parser.add_argument("--timeout", action="store_const", default=30, const="timeout", #parser.add_argument("--timeout", action="store_const", default=30, const="timeout",
# help="Abort a program after TIMEOUT seconds") # help="Abort a program after TIMEOUT seconds")
......
...@@ -32,7 +32,8 @@ class Environment: ...@@ -32,7 +32,8 @@ class Environment:
def __init__(self, mode, mj_run: str, tmp_dir: str = "", test_dir: str = "", def __init__(self, mode, mj_run: str, tmp_dir: str = "", test_dir: str = "",
only_incorrect_tests: bool = False, parallel: bool = False, only_incorrect_tests: bool = False, parallel: bool = False,
timeout: int = 30, report_dir: str = "", log_level: str = "warn", timeout: int = 30, report_dir: str = "", log_level: str = "warn",
produce_no_reports: bool = True, output_incorrect_reports: bool = True): produce_no_reports: bool = True, output_no_incorrect_reports: bool = False,
produce_all_reports: bool = False):
self.mode = mode self.mode = mode
self.mj_run_cmd = os.path.realpath(mj_run) self.mj_run_cmd = os.path.realpath(mj_run)
...@@ -60,19 +61,19 @@ class Environment: ...@@ -60,19 +61,19 @@ class Environment:
if not produce_no_reports: if not produce_no_reports:
if report_dir: if report_dir:
self.report_dir = os.path.abspath(os.path.expandvars(report_dir)) self.report_dir = os.path.abspath(os.path.expandvars(report_dir))
if not os.path.exists(report_dir):
os.mkdir(self.report_dir)
else: else:
self.report_dir = os.path.join(get_mjtest_basedir(), "reports") self.report_dir = os.path.join(get_mjtest_basedir(), "reports")
if not os.path.exists(self.report_dir): try:
os.mkdir(self.report_dir) os.mkdir(self.report_dir)
except IOError:
pass
self.report_dir = os.path.join(self.report_dir, datetime.now().strftime("%d-%m-%y_%H-%M-%S")) self.report_dir = os.path.join(self.report_dir, datetime.now().strftime("%d-%m-%y_%H-%M-%S"))
os.mkdir(self.report_dir)
else: else:
self.report_dir = None self.report_dir = None
logging.basicConfig(level=self.LOG_LEVELS[log_level]) logging.basicConfig(level=self.LOG_LEVELS[log_level])
self.produce_reports = not produce_no_reports # type: bool self.produce_reports = not produce_no_reports # type: bool
self.output_incorrect_reports = True self.output_incorrect_reports = not output_no_incorrect_reports
self.produce_all_reports = produce_all_reports
def create_tmpfile(self) -> str: def create_tmpfile(self) -> str:
return os.path.join(self.tmp_dir, str(os.times())) return os.path.join(self.tmp_dir, str(os.times()))
......
...@@ -94,7 +94,7 @@ class TestSuite: ...@@ -94,7 +94,7 @@ class TestSuite:
colored("{} failed.".format(ret.failed), "red", attrs=["bold"])) colored("{} failed.".format(ret.failed), "red", attrs=["bold"]))
else: else:
cprint("All {} run tests succeeded".format(ret.count), "green") cprint("All {} run tests succeeded".format(ret.count), "green")
if self.env.produce_reports: if self.env.produce_reports and (self.env.produce_all_reports or ret.failed > 0):
report_dir = self.env.report_dir + "." + ("successful" if ret.failed == 0 else "failed") report_dir = self.env.report_dir + "." + ("successful" if ret.failed == 0 else "failed")
os.rename(self.env.report_dir, report_dir) os.rename(self.env.report_dir, report_dir)
print("A full report for each test can be found at {}".format( print("A full report for each test can be found at {}".format(
...@@ -139,12 +139,15 @@ class TestSuite: ...@@ -139,12 +139,15 @@ class TestSuite:
tc=test_case.name()), color, attrs=["bold"]) + tc=test_case.name()), color, attrs=["bold"]) +
colored("" if ret.is_correct() else ret.short_message(), color)) colored("" if ret.is_correct() else ret.short_message(), color))
try: try:
if self.env.produce_reports: if self.env.produce_reports and (self.env.produce_all_reports or not ret.is_correct()):
if not exists(self.env.report_dir): if not exists(self.env.report_dir):
os.mkdir(self.env.report_dir) os.mkdir(self.env.report_dir)
rep_dir = join(self.env.report_dir, test_case.type) rep_dir = join(self.env.report_dir, test_case.type)
if not exists(rep_dir): if not exists(rep_dir):
os.mkdir(rep_dir) try:
os.mkdir(rep_dir)
except IOError:
pass
suffix = ".correct" if ret.is_correct() else ".incorrect" suffix = ".correct" if ret.is_correct() else ".incorrect"
ret.store_at(join(rep_dir, test_case.short_name() + suffix)) ret.store_at(join(rep_dir, test_case.short_name() + suffix))
if self.env.output_incorrect_reports and not ret.is_correct(): if self.env.output_incorrect_reports and not ret.is_correct():
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment