Commit 9f96c9d8 authored by Johannes Bechberger's avatar Johannes Bechberger
Browse files

Add aditional options to the test runner

parent fbb7ade3
......@@ -91,7 +91,8 @@ Output of the `./ --help`
usage: [-h] [--only_incorrect_tests] [--produce_no_reports]
[--parallel] [--log_level LOG_LEVEL]
[--produce_all_reports] [--parallel]
[--output_no_incorrect_reports] [--log_level LOG_LEVEL]
{syntax,ast,semantic,exec} MJ_RUN
MiniJava test runner
......@@ -109,8 +110,13 @@ optional arguments:
Only run the tests that were incorrect the last run
--produce_no_reports Produce no long reports besides the command line
Produce reports also for correct test cases
--parallel Run the tests in parallel
--log LOG Logging level (error, warn, info or debug)
Output the long report for every incorrect test case
--log_level LOG_LEVEL
Logging level (error, warn, info or debug)
For Windows users: Using environment variables doesn't work and you have to use prefix `./` with `python`.
......@@ -127,7 +133,7 @@ This will…
[FAIL ] syntax:test.invalid the error output doesn't contain the word "error"
Ran 1 tests, of which 1 failed.
A full report for each test can be found at mjtest/reports/30.10.16:08:05:10.failed
A full report for each test can be found at mjtest/reports/30-10-16_08-05-10.failed
- … log that some test cases were executed correctly
- … return with an error code of `0` if all tests executed correct
......@@ -38,9 +38,11 @@ if True:#__name__ == '__main__':
help="Only run the tests that were incorrect the last run")
parser.add_argument("--produce_no_reports", action="store_true", default=False,
help="Produce no long reports besides the command line output")
parser.add_argument("--produce_all_reports", action="store_true", default=False,
help="Produce reports also for correct test cases")
parser.add_argument("--parallel", action="store_true", default=False,
help="Run the tests in parallel")
parser.add_argument("--output_incorrect_reports", action="store_true", default=True,
parser.add_argument("--output_no_incorrect_reports", action="store_true", default=False,
help="Output the long report for every incorrect test case")
#parser.add_argument("--timeout", action="store_const", default=30, const="timeout",
# help="Abort a program after TIMEOUT seconds")
......@@ -32,7 +32,8 @@ class Environment:
def __init__(self, mode, mj_run: str, tmp_dir: str = "", test_dir: str = "",
only_incorrect_tests: bool = False, parallel: bool = False,
timeout: int = 30, report_dir: str = "", log_level: str = "warn",
produce_no_reports: bool = True, output_incorrect_reports: bool = True):
produce_no_reports: bool = True, output_no_incorrect_reports: bool = False,
produce_all_reports: bool = False):
self.mode = mode
self.mj_run_cmd = os.path.realpath(mj_run)
......@@ -60,19 +61,19 @@ class Environment:
if not produce_no_reports:
if report_dir:
self.report_dir = os.path.abspath(os.path.expandvars(report_dir))
if not os.path.exists(report_dir):
self.report_dir = os.path.join(get_mjtest_basedir(), "reports")
if not os.path.exists(self.report_dir):
except IOError:
self.report_dir = os.path.join(self.report_dir,"%d-%m-%y_%H-%M-%S"))
self.report_dir = None
self.produce_reports = not produce_no_reports # type: bool
self.output_incorrect_reports = True
self.output_incorrect_reports = not output_no_incorrect_reports
self.produce_all_reports = produce_all_reports
def create_tmpfile(self) -> str:
return os.path.join(self.tmp_dir, str(os.times()))
......@@ -94,7 +94,7 @@ class TestSuite:
colored("{} failed.".format(ret.failed), "red", attrs=["bold"]))
cprint("All {} run tests succeeded".format(ret.count), "green")
if self.env.produce_reports:
if self.env.produce_reports and (self.env.produce_all_reports or ret.failed > 0):
report_dir = self.env.report_dir + "." + ("successful" if ret.failed == 0 else "failed")
os.rename(self.env.report_dir, report_dir)
print("A full report for each test can be found at {}".format(
......@@ -139,12 +139,15 @@ class TestSuite:, color, attrs=["bold"]) +
colored("" if ret.is_correct() else ret.short_message(), color))
if self.env.produce_reports:
if self.env.produce_reports and (self.env.produce_all_reports or not ret.is_correct()):
if not exists(self.env.report_dir):
rep_dir = join(self.env.report_dir, test_case.type)
if not exists(rep_dir):
except IOError:
suffix = ".correct" if ret.is_correct() else ".incorrect"
ret.store_at(join(rep_dir, test_case.short_name() + suffix))
if self.env.output_incorrect_reports and not ret.is_correct():
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment