Commit 8e19d593 authored by Johannes Bechberger's avatar Johannes Bechberger

Fix "only_incorrect_testcases" bug

And add "--produce_no_reports" option
parent 4a370c9f
......@@ -68,8 +68,8 @@ To get colored output install the python3 module `termcolor`.
Output of the `./mjt.py --help`
```
usage: mjt.py [-h] [--only_incorrect_tests] [--parallel]
[--log_level LOG_LEVEL]
usage: mjt.py [-h] [--only_incorrect_tests] [--produce_no_reports]
[--parallel] [--log_level LOG_LEVEL]
{syntax,semantic,exec} MJ_RUN
MiniJava test runner
......@@ -85,6 +85,8 @@ optional arguments:
-h, --help show this help message and exit
--only_incorrect_tests
Only run the tests that were incorrect the last run
--produce_no_reports Produce no long reports besides the command line
output
--parallel Run the tests in parallel
--log LOG Logging level (error, warn, info or debug)
```
......
......@@ -36,6 +36,8 @@ if True:#__name__ == '__main__':
# help="Directory that contains all test cases, default is the 'tests' directory")
parser.add_argument("--only_incorrect_tests", action="store_true", default=False,
help="Only run the tests that were incorrect the last run")
parser.add_argument("--produce_no_reports", action="store_true", default=False,
help="Produce no long reports besides the command line output")
parser.add_argument("--parallel", action="store_true", default=False,
help="Run the tests in parallel")
#parser.add_argument("--timeout", action="store_const", default=30, const="timeout",
......@@ -54,7 +56,8 @@ if True:#__name__ == '__main__':
finally:
suite.env.clean_up()
suite.store()
if ret is None or ret:
if ret is None or ret.failed > 0:
sys.exit(1)
else:
sys.exit(0)
......@@ -29,7 +29,8 @@ class Environment:
def __init__(self, mode, mj_run: str, tmp_dir: str = "", test_dir: str = "",
only_incorrect_tests: bool = False, parallel: bool = False,
timeout: int = 30, report_dir: str = "", log_level: str = "warn"):
timeout: int = 30, report_dir: str = "", log_level: str = "warn",
produce_no_reports: bool = True):
self.mode = mode
self.mj_run_cmd = os.path.realpath(mj_run)
......@@ -54,18 +55,21 @@ class Environment:
self.only_incorrect_tests = only_incorrect_tests
self.parallel = parallel
self.timeout = timeout
if report_dir:
self.report_dir = os.path.abspath(os.path.expandvars(report_dir))
if not os.path.exists(report_dir):
os.mkdir(self.report_dir)
if not produce_no_reports:
if report_dir:
self.report_dir = os.path.abspath(os.path.expandvars(report_dir))
if not os.path.exists(report_dir):
os.mkdir(self.report_dir)
else:
self.report_dir = os.path.join(get_mjtest_basedir(), "reports")
if not os.path.exists(self.report_dir):
os.mkdir(self.report_dir)
self.report_dir = os.path.join(self.report_dir, datetime.now().strftime("%d-%m-%y_%H-%M-%S"))
os.mkdir(self.report_dir)
else:
self.report_dir = os.path.join(get_mjtest_basedir(), "reports")
if not os.path.exists(self.report_dir):
os.mkdir(self.report_dir)
self.report_dir = os.path.join(self.report_dir, datetime.now().strftime("%d-%m-%y_%H-%M-%S"))
os.mkdir(self.report_dir)
self.report_dir = None
logging.basicConfig(level=self.LOG_LEVELS[log_level])
self.produce_reports = not produce_no_reports # type: bool
def create_tmpfile(self) -> str:
return os.path.join(self.tmp_dir, str(os.times()))
......
from collections import namedtuple
import shutil
from typing import Optional, List, Tuple, T, Union, Dict
import collections
from mjtest.environment import Environment, TestMode, TEST_MODES
from os.path import join, exists, basename
import logging
......@@ -25,7 +26,7 @@ class TestSuite:
def __init__(self, env: Environment):
self.env = env
self.test_cases = {} # type: Dict[str, List[TestCase]]
self.correct_test_cases = {} # type: Dict[str, List[str]]
self.correct_test_cases = collections.defaultdict(set) # type: Dict[str, Set[str]]
self._load_test_cases()
def _load_test_cases(self):
......@@ -47,7 +48,12 @@ class TestSuite:
log_file = self._log_file_for_type(mode)
if exists(log_file):
with open(log_file) as f:
correct_test_cases = set(f.readlines())
correct_test_cases = set()
for t in f.readlines():
t = t.strip()
if len(t) > 0:
self.correct_test_cases[mode].add(t)
correct_test_cases.add(t)
for file in sorted(os.listdir(dir)):
if not TestCase.has_valid_file_ending(mode, file):
_LOG.debug("Skip file " + file)
......@@ -66,7 +72,7 @@ class TestSuite:
return join(self.env.test_dir, type, ".mjtest_correct_testcases")
def _add_correct_test_case(self, test_case: 'TestCase'):
self.correct_test_cases[test_case.type].append(basename(test_case.file))
self.correct_test_cases[test_case.type].add(basename(test_case.file))
def run(self) -> RunResult:
ret = RunResult(0, 0)
......@@ -87,10 +93,11 @@ class TestSuite:
colored("{} failed.".format(ret.failed), "red", attrs=["bold"]))
else:
cprint("All {} run tests succeeded".format(ret.count), "green")
report_dir = self.env.report_dir + "." + ("successful" if ret.failed == 0 else "failed")
os.rename(self.env.report_dir, report_dir)
print("A full report for each test can be found at {}".format(
os.path.relpath(report_dir)))
if self.env.produce_reports:
report_dir = self.env.report_dir + "." + ("successful" if ret.failed == 0 else "failed")
os.rename(self.env.report_dir, report_dir)
print("A full report for each test can be found at {}".format(
os.path.relpath(report_dir)))
return ret
......@@ -109,13 +116,18 @@ class TestSuite:
def _func(self, test_case: 'TestCase'):
ret = self._run_test_case(test_case)
if ret is not False and ret.is_correct():
return 0, [test_case]
return 1, []
return 0, test_case
return 1, test_case
def _run_parallel(self, mode: str, parallel_jobs: int) -> RunResult:
pool = multiprocessing.Pool(parallel_jobs)
rets = pool.map(self._func, self.test_cases[mode])
return RunResult(len(rets), sum(map(lambda x: x[0], rets)))
result = RunResult(len(rets), sum(map(lambda x: x[0], rets)))
for (suc, test_case) in rets:
if suc == 0:
self._add_correct_test_case(test_case)
return result
def _run_test_case(self, test_case: 'TestCase') -> Optional['TestResult']:
try:
......@@ -126,13 +138,14 @@ class TestSuite:
tc=test_case.name()), color, attrs=["bold"]) +
colored("" if ret.is_correct() else ret.short_message(), color))
try:
if not exists(self.env.report_dir):
os.mkdir(self.env.report_dir)
rep_dir = join(self.env.report_dir, test_case.type)
if not exists(rep_dir):
os.mkdir(rep_dir)
suffix = ".correct" if ret.is_correct() else ".incorrect"
ret.store_at(join(rep_dir, test_case.short_name() + suffix))
if self.env.produce_reports:
if not exists(self.env.report_dir):
os.mkdir(self.env.report_dir)
rep_dir = join(self.env.report_dir, test_case.type)
if not exists(rep_dir):
os.mkdir(rep_dir)
suffix = ".correct" if ret.is_correct() else ".incorrect"
ret.store_at(join(rep_dir, test_case.short_name() + suffix))
return ret
except IOError:
_LOG.exception("Caught i/o error while trying to store the report for '{}'"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment