Commit c1b6f48b authored by Johannes Bechberger's avatar Johannes Bechberger

Add lexer mode and "all" option

parent 18d263ef
......@@ -2,15 +2,18 @@ import logging
import os
import sys
import argparse
from typing import Dict
from mjtest.environment import TestMode, Environment, TEST_MODES
from mjtest.test.tests import TestSuite
# adapted from http://stackoverflow.com/a/8527629
from mjtest.util.utils import cprint, colored
class LogLevelChoices(argparse.Action):
CHOICES = ["error", "warn", "info", "debug"]
def __call__(self, parser, namespace, values, option_string=None):
if values:
for value in values:
def __call__(self, parser, namespace, value, option_string=None):
if value:
if value not in self.CHOICES:
message = ("invalid choice: {0!r} (choose from {1})"
.format(value,
......@@ -18,12 +21,12 @@ class LogLevelChoices(argparse.Action):
for action in self.CHOICES])))
raise argparse.ArgumentError(self, message)
setattr(namespace, self.dest, values)
setattr(namespace, self.dest, value)
if True:#__name__ == '__main__':
parser = argparse.ArgumentParser(description="MiniJava test runner", add_help=True)
parser.add_argument("mode",
choices=TEST_MODES,
choices=["all"] + TEST_MODES,
help="What do you want to test?")
if os.getenv("MJ_RUN", None) is None:
parser.add_argument("mj_run",
......@@ -51,6 +54,12 @@ if True:#__name__ == '__main__':
parser.add_argument("--log_level", action=LogLevelChoices, default="warn", const="log_level",
help="Logging level (error, warn, info or debug)")
args = vars(parser.parse_args())
failed = 0
count = 0
def run(args: Dict[str, str]):
global failed, count
if os.getenv("MJ_RUN", None) is not None:
args["mj_run"] = os.getenv("MJ_RUN")
suite = TestSuite(Environment(**args))
......@@ -60,8 +69,19 @@ if True:#__name__ == '__main__':
finally:
suite.env.clean_up()
suite.store()
if ret is None or ret.failed > 0:
sys.exit(1)
if ret is not None:
failed += ret.failed
count += ret.count
if args["mode"] == "all":
for mode in TEST_MODES:
cprint("Run {} tests".format(mode), attrs=["bold"])
args["mode"] = mode
run(args)
if failed > 0: # some tests failed
print(colored("Ran {} tests, of which ".format(count), "red") +
colored("{} failed.".format(failed), "red", attrs=["bold"]))
else:
cprint("All {} run tests succeeded".format(count), "green")
else:
sys.exit(0)
run(args)
sys.exit(1 if failed > 0 else 0)
......@@ -9,6 +9,9 @@ from typing import Tuple, List
class TestMode:
lexer = "lexer"
syntax = "syntax"
ast = "ast"
......@@ -18,7 +21,7 @@ class TestMode:
exec = "exec"
""" All 'success' tests of the n.th mode can used as 'success' tests for the n-1.th mode"""
TEST_MODES = [TestMode.syntax, TestMode.ast, TestMode.semantic, TestMode.exec]
TEST_MODES = [TestMode.lexer, TestMode.syntax, TestMode.ast, TestMode.semantic, TestMode.exec]
class Environment:
......@@ -95,6 +98,7 @@ class Environment:
:return: (out, err, return code)
"""
mode_flag = {
TestMode.lexer: "--lextest",
TestMode.syntax: "--parsetest",
TestMode.ast: "--parse-ast"
}[mode]
......
......@@ -28,11 +28,17 @@ class ASTDiffTest(TestCase):
out, err, rtcode = self.env.run_mj_command(self.MODE, self.file)
exp_out = ""
if rtcode > 0 and self.should_succeed():
if self._has_expected_output_file:
if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE:
with open(self._expected_output_file, "r") as f:
exp_out = f.read()
else:
_LOG.error("Expected output file for test case {}:{} is missing.".format(self.MODE, self.short_name()))
#else:
# _LOG.error("Expected output file for test case {}:{} is missing.".format(self.MODE, self.short_name()))
return BasicDiffTestResult(self, rtcode, out.decode(), err.decode(), exp_out)
class LexerDiffTest(ASTDiffTest):
MODE = TestMode.lexer
TestCase.TEST_CASE_CLASSES[TestMode.ast].append(ASTDiffTest)
TestCase.TEST_CASE_CLASSES[TestMode.lexer].append(LexerDiffTest)
\ No newline at end of file
......@@ -31,7 +31,7 @@ class TestSuite:
self._load_test_cases()
def _load_test_cases(self):
types = TEST_MODES[TEST_MODES.index(self.env.mode):]
types = TEST_MODES#TEST_MODES[TEST_MODES.index(self.env.mode):]
for type in types:
self._load_test_case_type(type)
......@@ -44,7 +44,6 @@ class TestSuite:
_LOG.info("Test folder {} doesn't exist".format(dir))
def _load_test_case_dir(self, mode: str, dir: str):
self.test_cases[mode] = []
correct_test_cases = set()
log_file = self._log_file_for_type(mode)
if exists(log_file):
......@@ -61,12 +60,14 @@ class TestSuite:
elif self.env.only_incorrect_tests and file in correct_test_cases:
_LOG.info("Skip file {} as its test case was executed correctly the last run")
else:
test_case = TestCase.create_from_file(self.env, self.env.mode, join(dir, file))
test_case = TestCase.create_from_file(self.env, mode, join(dir, file))
if not test_case.can_run():
_LOG.debug("Skip test case '{}' because it isn't suited".format(test_case.name()))
else:
if mode not in self.test_cases:
self.test_cases[mode] = []
self.test_cases[mode].append(test_case)
if len(self.test_cases[mode]) == 0:
if mode in self.test_cases and len(self.test_cases[mode]) == 0:
del self.test_cases[mode]
def _log_file_for_type(self, type: str):
......@@ -96,7 +97,10 @@ class TestSuite:
cprint("All {} run tests succeeded".format(ret.count), "green")
if self.env.produce_reports and (self.env.produce_all_reports or ret.failed > 0):
report_dir = self.env.report_dir + "." + ("successful" if ret.failed == 0 else "failed")
try:
os.rename(self.env.report_dir, report_dir)
except IOError:
pass
print("A full report for each test can be found at {}".format(
os.path.relpath(report_dir)))
......@@ -192,9 +196,10 @@ class TestCase:
def can_run(self, mode: str = "") -> bool:
mode = mode or self.env.mode
types = TEST_MODES[TEST_MODES.index(self.env.mode):]
return self.type == mode or \
(self.type in TEST_MODES[TEST_MODES.index(self.env.mode):] and self.should_succeed())
(self.type in types and self.should_succeed()) or \
(self.type not in types and not self.should_succeed())
def run(self) -> 'TestResult':
raise NotImplementedError()
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment