environment.py 8.75 KB
Newer Older
Johannes Bechberger's avatar
Johannes Bechberger committed
1 2
import logging
import os
Johannes Bechberger's avatar
Johannes Bechberger committed
3
import random
Johannes Bechberger's avatar
Fix #47  
Johannes Bechberger committed
4
import shlex
Johannes Bechberger's avatar
Johannes Bechberger committed
5 6
import shutil
import tempfile
7
import sys
Johannes Bechberger's avatar
Johannes Bechberger committed
8 9
from datetime import datetime
import time
Johannes Bechberger's avatar
Fix #47  
Johannes Bechberger committed
10
from threading import Timer
11
from typing import Dict, Optional
12

Johannes Bechberger's avatar
Fix #47  
Johannes Bechberger committed
13 14
import subprocess

Johannes Bechberger's avatar
Johannes Bechberger committed
15
from mjtest.util.shell import execute
16
from mjtest.util.utils import get_mjtest_basedir, force_colored_output
17
import mjtest.util.utils
Johannes Bechberger's avatar
Johannes Bechberger committed
18
from typing import Tuple, List
19
from preproc.preproc.preprocessor import PreProcessor, PreProcessorError, is_importable_file
Johannes Bechberger's avatar
Johannes Bechberger committed
20

21
_LOG = logging.getLogger("env")
Johannes Bechberger's avatar
Johannes Bechberger committed
22 23

class TestMode:
24 25 26

    lexer = "lexer"

Johannes Bechberger's avatar
Johannes Bechberger committed
27 28
    syntax = "syntax"

Johannes Bechberger's avatar
Johannes Bechberger committed
29 30
    ast = "ast"

Johannes Bechberger's avatar
Johannes Bechberger committed
31 32
    semantic = "semantic"

33
    compile_firm = "compile-firm"
34

Johannes Bechberger's avatar
Johannes Bechberger committed
35 36
    compile = "compile"

37 38 39 40
    compile_firm_only = "compile-firm-only"

    compile_only = "compile-only"

Johannes Bechberger's avatar
Johannes Bechberger committed
41 42
    exec = "exec"

43 44
    bench = "bench"

Johannes Bechberger's avatar
Johannes Bechberger committed
45
    USE_TESTS_OF_OTHER = {
46
        ast: [syntax],
47
        compile_firm: [exec],
Johannes Bechberger's avatar
Johannes Bechberger committed
48
        compile: [exec],
49 50
        compile_firm_only: [semantic, compile_only],
        compile_only: [semantic],
51
        bench: [exec]
Johannes Bechberger's avatar
Johannes Bechberger committed
52 53
    }

Johannes Bechberger's avatar
Johannes Bechberger committed
54
""" All 'success' tests of the n.th mode can used as 'success' tests for the n-1.th mode"""
55 56
TEST_MODES = [TestMode.lexer, TestMode.syntax, TestMode.ast, TestMode.semantic, TestMode.compile_firm_only,
              TestMode.compile_only, TestMode.compile_firm, TestMode.exec, TestMode.bench, TestMode.compile]
57 58 59 60


def get_test_dirname(mode: str) -> str:
    d = {
61
    #    TestMode.compile_firm: "exec"
62 63 64 65 66
    }
    if mode not in d:
        return mode
    return d[mode]

Johannes Bechberger's avatar
Johannes Bechberger committed
67 68 69 70 71 72 73 74 75 76

class Environment:

    LOG_LEVELS = {
        "info": logging.INFO,
        "error": logging.ERROR,
        "warn": logging.WARN,
        "debug": logging.DEBUG
    }

77
    def __init__(self, mode, mj_run: str, tmp_dir: str = "", test_dir: str = "",
Johannes Bechberger's avatar
Johannes Bechberger committed
78
                 only_incorrect_tests: bool = False, parallel: bool = False,
79
                 timeout: int = 30, report_dir: str = "", log_level: str = "warn",
80
                 produce_no_reports: bool = True, output_no_incorrect_reports: bool = False,
81
                 produce_all_reports: bool = False, report_subdir: str = None,
82
                 ci_testing: bool = False, color: bool = False,
83 84
                 all_exec_tests: bool = True, bench_compiler_flags: List[str]=[],
                 bench_runs: int = 10):
85 86
        if color:
            force_colored_output()
Johannes Bechberger's avatar
Johannes Bechberger committed
87
        self.mode = mode
88
        self.mj_run_cmd = os.path.realpath(mj_run)
Johannes Bechberger's avatar
Johannes Bechberger committed
89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104

        if tmp_dir:
            self.own_tmp_dir = True
            self.tmp_dir = os.path.abspath(os.path.expandvars(tmp_dir))
            if not os.path.exists(tmp_dir):
                os.mkdir(self.tmp_dir)
        else:
            self.own_tmp_dir = False
            self.tmp_dir = tempfile.mkdtemp("mjtest")

        if test_dir:
            self.test_dir = os.path.abspath(os.path.realpath(test_dir))
        else:
            self.test_dir = os.path.join(get_mjtest_basedir(), "tests")
        if not os.path.exists(self.test_dir):
            os.mkdir(self.test_dir)
Johannes Bechberger's avatar
Johannes Bechberger committed
105
            for d in TEST_MODES:
Johannes Bechberger's avatar
Johannes Bechberger committed
106 107 108 109
                os.mkdir(os.path.join(self.test_dir, d))

        self.only_incorrect_tests = only_incorrect_tests
        self.parallel = parallel
110 111 112 113 114
        if not produce_no_reports:
            if report_dir:
                self.report_dir = os.path.abspath(os.path.expandvars(report_dir))
            else:
                self.report_dir = os.path.join(get_mjtest_basedir(), "reports")
115 116 117 118
            try:
                os.mkdir(self.report_dir)
            except IOError:
                pass
119
            self.report_dir = os.path.join(self.report_dir, report_subdir or datetime.now().strftime("%d-%m-%y_%H-%M-%S"))
120 121
            if not os.path.exists(self.report_dir):
                os.mkdir(self.report_dir)
Johannes Bechberger's avatar
Johannes Bechberger committed
122
        else:
123
            self.report_dir = None
Johannes Bechberger's avatar
Johannes Bechberger committed
124
        logging.basicConfig(level=self.LOG_LEVELS[log_level])
125
        self.produce_reports = not produce_no_reports # type: bool
126 127
        self.output_incorrect_reports = not output_no_incorrect_reports
        self.produce_all_reports = produce_all_reports
128
        self.ci_testing = ci_testing
Johannes Bechberger's avatar
Johannes Bechberger committed
129
        self._tmp_file_ctr = 0
130
        self._already_preprocessed_files = set()
131
        self._pid_tmpdirs = {}  # type: Dict[int, str]
132
        self.only_small_exec_tests = not all_exec_tests
Johannes Bechberger's avatar
Johannes Bechberger committed
133

134 135
        self.timeout = float(os.getenv("MJ_TIMEOUT", "10"))
        self.big_timeout = float(os.getenv("MJ_BIG_TIMEOUT", "60"))
136

137 138 139
        self.bench_compiler_flags = bench_compiler_flags
        self.bench_runs = bench_runs

Johannes Bechberger's avatar
Johannes Bechberger committed
140
    def create_tmpfile(self) -> str:
Johannes Bechberger's avatar
Johannes Bechberger committed
141 142 143
        self._tmp_file_ctr += 1
        return os.path.join(self.tmp_dir, str(round(time.time() * 100000))
                            + str(random.randrange(0, 10000, 1)) + str(self._tmp_file_ctr))
144 145 146 147 148

    def create_tmpdir(self) -> str:
        dir = self.create_tmpfile()
        os.mkdir(dir)
        return dir
Johannes Bechberger's avatar
Johannes Bechberger committed
149

150 151 152 153 154 155
    def create_pid_local_tmpdir(self) -> str:
        pid = os.getpid()
        if pid not in self._pid_tmpdirs:
            self._pid_tmpdirs[pid] = self.create_tmpdir()
        return self._pid_tmpdirs[pid]

Johannes Bechberger's avatar
Johannes Bechberger committed
156 157 158 159
    def clean_up(self):
        if not self.own_tmp_dir:
            shutil.rmtree(self.tmp_dir)

160
    def run_mj_command(self, mode: str, *args: Tuple[str], timeout: float = None) -> Tuple[bytes, bytes, int]:
Johannes Bechberger's avatar
Johannes Bechberger committed
161 162 163 164 165 166
        """
        Execute the MiniJava `run` script with the given arguments.

        :param args: arguments for the MiniJava `run` script
        :return: (out, err, return code)
        """
Johannes Bechberger's avatar
Johannes Bechberger committed
167
        mode_flag = {
168
            TestMode.lexer: "--lextest",
Johannes Bechberger's avatar
Johannes Bechberger committed
169
            TestMode.syntax: "--parsetest",
Johannes Bechberger's avatar
Johannes Bechberger committed
170
            TestMode.ast: "--print-ast",
171
            TestMode.semantic: "--check",
172 173
            TestMode.compile_only: None,
            TestMode.compile_firm_only: "--compile-firm",
Johannes Bechberger's avatar
Johannes Bechberger committed
174
            TestMode.compile_firm: "--compile-firm",
175
            TestMode.compile: None,
Johannes Bechberger's avatar
Johannes Bechberger committed
176
        }[mode]
Johannes Bechberger's avatar
Johannes Bechberger committed
177 178
        cmd = [self.mj_run_cmd]
        if mode_flag:
Johannes Bechberger's avatar
Johannes Bechberger committed
179
            cmd.append(mode_flag)
Johannes Bechberger's avatar
Johannes Bechberger committed
180
        cmd.extend(list(args))
181
        return execute(cmd, timeout=timeout or self.timeout)
182

183
    def run_command(self, cmd: str, *args: Tuple[str], timeout: float = None, input_bytes: Optional[bytes] = None) -> Tuple[bytes, bytes, int]:
184 185 186 187 188
        """
        Execute the passend command with its arguments

        :return: (out, err, return code)
        """
189
        return execute([cmd] + list(args), timeout=timeout or self.timeout, input_bytes=input_bytes)
190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214

    def has_to_preprocess(self, file: str) -> bool:
        return os.path.relpath(file, self.test_dir).startswith("exec")

    def is_lib_file(self, file: str) -> bool:
        return is_importable_file(file)

    def preprocess(self, file: str) -> str:
        """
        Pre process the passed file if needed and return the resulting file
        """
        if not self.has_to_preprocess(file):
            return file
        if ".preprocessed" in os.path.relpath(file, self.test_dir):
            return file

        import_base_dir = os.path.join(self.test_dir, os.path.relpath(file, self.test_dir).lstrip(os.sep).split(os.sep)[0])
        dst_dir = os.path.join(import_base_dir, ".preprocessed", os.path.split(os.path.relpath(file, import_base_dir))[0])
        dst_file = os.path.join(dst_dir, os.path.basename(file))
        if dst_file.endswith(".mj"):
            dst_file = dst_file.replace(".mj", ".java")
        if dst_file in self._already_preprocessed_files:
            return dst_file
        self._already_preprocessed_files.add(dst_file)

215
        if os.path.exists(dst_file) and os.path.isfile(dst_file) and os.path.getmtime(file) < os.path.getmtime(dst_file) and False:
216 217 218
            _LOG.debug("File '{}' already exists in a pre processed form".format(os.path.relpath(file)))
            return dst_file
        if not os.path.exists(dst_dir):
Johannes Bechberger's avatar
Johannes Bechberger committed
219
            os.makedirs(dst_dir, exist_ok=True)
220 221 222 223 224 225 226
        try:
            PreProcessor(file, import_base_dir, dst_file).preprocess()
        except:
            _LOG.exception("Pre processing file '{}'".format(file))
            raise
        return dst_file

Johannes Bechberger's avatar
Fix #47  
Johannes Bechberger committed
227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251
    def run_command_till_timeout(self, cmd: str, *args: Tuple[str], timeout: float = 0) -> Tuple[bytes, bytes, int, bool]:
        """
        Much slower than `run_command` but captures the command output even when a
        timeout occurs.

        Inspired by http://stackoverflow.com/a/10768774
        """
        cmds = [cmd] + list(args)
        cmds = [shlex.quote(c) for c in cmds]
        proc = subprocess.Popen(cmds, stdout=subprocess.PIPE,
                                stderr=subprocess.PIPE)
        timeouted = False

        def kill_proc(proc):
            timeouted = True
            proc.kill()

        stdout, stderr = None, None
        if timeout >= 0.0:
            timer = Timer(timeout, kill_proc, [proc])
            timer.start()
            stdout, stderr = proc.communicate()
            timer.cancel()
        else:
            stdout, stderr = proc.communicate()
252
        return stdout, stderr, proc.returncode, timeouted