cli.py 4.9 KB
Newer Older
Johannes Bechberger's avatar
Johannes Bechberger committed
1
import logging
2
import os
Johannes Bechberger's avatar
Johannes Bechberger committed
3 4
import sys
import argparse
5
from typing import Dict
6
from datetime import datetime
7
from mjtest.environment import TestMode, Environment, TEST_MODES
Johannes Bechberger's avatar
Johannes Bechberger committed
8 9 10
from mjtest.test.tests import TestSuite

# adapted from http://stackoverflow.com/a/8527629
11 12 13
from mjtest.util.utils import cprint, colored


Johannes Bechberger's avatar
Johannes Bechberger committed
14 15
class LogLevelChoices(argparse.Action):
    CHOICES = ["error", "warn", "info", "debug"]
16 17 18 19 20 21 22
    def __call__(self, parser, namespace, value, option_string=None):
        if value:
            if value not in self.CHOICES:
                message = ("invalid choice: {0!r} (choose from {1})"
                           .format(value,
                                   ', '.join([repr(action)
                                              for action in self.CHOICES])))
Johannes Bechberger's avatar
Johannes Bechberger committed
23

24 25
                raise argparse.ArgumentError(self, message)
            setattr(namespace, self.dest, value)
Johannes Bechberger's avatar
Johannes Bechberger committed
26 27 28 29

if True:#__name__ == '__main__':
    parser = argparse.ArgumentParser(description="MiniJava test runner", add_help=True)
    parser.add_argument("mode",
30
                        choices=["all"] + TEST_MODES,
Johannes Bechberger's avatar
Johannes Bechberger committed
31
                        help="What do you want to test?")
32 33 34 35 36
    if os.getenv("MJ_RUN", None) is None:
        parser.add_argument("mj_run",
                            metavar="MJ_RUN",
                            help="Command to run your MiniJava implementation, e.g. `mj/run`, "
                                 "can be omitted by assigning the environment variable MJ_RUN")
37 38 39 40
    #parser.add_argument("--tmp_dir", action="store_const", default="", const="tmp_dir",
    #                    help="Used temporary directory")
    #parser.add_argument("--test_dir", action="store_const", default="", const="test_dir",
    #                    help="Directory that contains all test cases, default is the 'tests' directory")
Johannes Bechberger's avatar
Johannes Bechberger committed
41 42
    parser.add_argument("--only_incorrect_tests", action="store_true", default=False,
                        help="Only run the tests that were incorrect the last run")
43 44
    parser.add_argument("--all_exec_tests", action="store_true", default=False,
                        help="Run all exec (compile-firm...) tests, not only the small ones")
45 46
    parser.add_argument("--produce_no_reports", action="store_true", default=False,
                        help="Produce no long reports besides the command line output")
47 48
    parser.add_argument("--produce_all_reports", action="store_true", default=False,
                        help="Produce reports also for correct test cases")
Johannes Bechberger's avatar
Johannes Bechberger committed
49 50
    parser.add_argument("--parallel", action="store_true", default=False,
                        help="Run the tests in parallel")
51
    parser.add_argument("--output_no_incorrect_reports", action="store_true", default=False,
52
                        help="Output the long report for every incorrect test case")
53 54
    parser.add_argument("--color", action="store_true", default=False,
                        help="Output color output even if the output goes to into a file")
55 56 57
    parser.add_argument("--ci_testing", action="store_true", default=False,
                        help="In mode X the succeeding test cases of later modes/phases should also succeed in "
                             "this mode, and failing test cases of prior modes/phases should also fail in this phase.")
58 59 60 61
    #parser.add_argument("--timeout", action="store_const", default=30, const="timeout",
    #                    help="Abort a program after TIMEOUT seconds")
    #parser.add_argument("--report_dir", action="store_const", default="", const="report_dir",
    #                    help="Directory to store the reports in, default is 'reports'")
Johannes Bechberger's avatar
Johannes Bechberger committed
62 63
    parser.add_argument("--log_level", action=LogLevelChoices, default="warn", const="log_level",
                        help="Logging level (error, warn, info or debug)")
64
    args = vars(parser.parse_args())
65

66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83
    failed = 0
    count = 0

    def run(args: Dict[str, str]):
        global failed, count
        if os.getenv("MJ_RUN", None) is not None:
            args["mj_run"] = os.getenv("MJ_RUN")
        suite = TestSuite(Environment(**args))
        ret = None
        try:
            ret = suite.run()
        finally:
            suite.env.clean_up()
            suite.store()
        if ret is not None:
            failed += ret.failed
            count += ret.count
    if args["mode"] == "all":
84
        report_subdir = datetime.now().strftime("%d-%m-%y_%H-%M-%S")
85
        for mode in [TestMode.lexer, TestMode.syntax, TestMode.ast, TestMode.semantic]:
86
            args["report_subdir"] = report_subdir + "_" + mode
87 88 89 90 91 92 93 94
            cprint("Run {} tests".format(mode), attrs=["bold"])
            args["mode"] = mode
            run(args)
        if failed > 0:  # some tests failed
            print(colored("Ran {} tests, of which ".format(count), "red") +
                  colored("{} failed.".format(failed), "red", attrs=["bold"]))
        else:
            cprint("All {} run tests succeeded".format(count), "green")
Johannes Bechberger's avatar
Johannes Bechberger committed
95
    else:
96 97
        run(args)
    sys.exit(1 if failed > 0 else 0)