...
 
Commits (19)
......@@ -23,8 +23,10 @@ The test cases are divided in 6 'modes':
them.
- __ast__: Test cases that check the generated ast by using the pretty printing functionality.
- __semantic__: Test cases that check semantic checking of MiniJava programs
- __compile-firm__: Test cases that check the correct compilation and execution of MiniJava programs with the libfirm backend.
- __compile__: Test cases that check the correct compilation and execution of MiniJava programs with the self implemented backend.
- __compile-firm-only__: Test cases that check the compilation of MiniJava programs with the libfirm backend.
- __compile-firm__ (alias `exec-firm`): Test cases that check the correct compilation and execution of MiniJava programs with the libfirm backend.
- __compile-only__: Test cases that check the compilation MiniJava programs with the self implemented backend.
- __compile__ (alias `exec`): Test cases that check the correct compilation and execution of MiniJava programs with the self implemented backend.
The test different test cases for each mode are located in a folder with the same name.
Except the compile-firm test cases which are located in the `exec` folder.
......@@ -103,6 +105,19 @@ Test types for the semantic mode
__All semantic mode tests have to be syntactically correct__
Test types for the compile-firm-only and compile-only mode
----------------------------------------------------------
<table>
<tr><th>File ending(s) of test cases</th><th>Expected behaviour to complete a test of this type</th></tr>
<tr>
<td><code>.valid.mj</code> <code>.mj</code> <code>.valid.java</code> <code>.java</code></td>
<td>Return code is <code>0</code> and the compiler produces an `a.out` file, i.e. the MiniJava code can be compiled</td>
</tr>
</table>
__All valid semantic test-cases are also used, the non-semantic compile-(firm)-only
should be placed into the `compile-only` folder__
Test types for the compile-firm and compile mode
------------------------------------------------
......@@ -112,7 +127,7 @@ Test types for the compile-firm and compile mode
<tr>
<td><code>.java</code> <code>.mj</code></td>
<td>The test file should compile. The resulting binary should be runnable and its output should be:<br/>
if a <code>[test file].out</code> exists: the content of this file<br/>
if a <code>[input file].out</code> (for test cases with input) or <code>[test file].out</code> (else) exists: the content of this file<br/>
if no such file exists: the same as the execution of the file with <code>java</code></td>
</tr>
<tr>
......@@ -171,12 +186,13 @@ usage: mjt.py [-h] [--only_incorrect_tests] [--all_exec_tests]
[--produce_no_reports] [--produce_all_reports] [--parallel]
[--output_no_incorrect_reports] [--color] [--ci_testing]
[--log_level LOG_LEVEL]
{all,lexer,syntax,ast,semantic,compile-firm,compile} MJ_RUN
{all,lexer,syntax,ast,semantic,compile-firm-only,compile-only,
compile-firm,compile,exec,exec-firm} MJ_RUN
MiniJava test runner
positional arguments:
{all,lexer,syntax,ast,semantic,compile-firm,exec}
{all,lexer,syntax,ast,semantic,compile-firm-only,compile-only,compile-firm,exec,exec-firm}
What do you want to test?
MJ_RUN Command to run your MiniJava implementation, e.g.
`mj/run`, can be omitted by assigning the environment
......
......@@ -27,7 +27,7 @@ class LogLevelChoices(argparse.Action):
if True:#__name__ == '__main__':
parser = argparse.ArgumentParser(description="MiniJava test runner", add_help=True)
parser.add_argument("mode",
choices=["all"] + TEST_MODES,
choices=["all"] + TEST_MODES + ["exec-firm"],
help="What do you want to test?")
if os.getenv("MJ_RUN", None) is None:
parser.add_argument("mj_run",
......@@ -84,9 +84,16 @@ if True:#__name__ == '__main__':
if ret is not None:
failed += ret.failed
count += ret.count
if args["mode"] in ["exec", "exec-firm"]:
new_mode = args["mode"].replace("exec", "compile")
print(colored("\"{}\" is just an alias for \"{}\", using the latter instead"
.format(args["mode"], new_mode),
"yellow"))
args["mode"] = new_mode
if args["mode"] == "all":
report_subdir = datetime.now().strftime("%d-%m-%y_%H-%M-%S")
for mode in [TestMode.lexer, TestMode.syntax, TestMode.ast, TestMode.semantic,
TestMode.compile_firm_only, TestMode.compile_only,
TestMode.compile_firm, TestMode.compile]:
args["all_exec_tests"] = True
args["report_subdir"] = report_subdir + "_" + mode
......
......@@ -34,6 +34,10 @@ class TestMode:
compile = "compile"
compile_firm_only = "compile-firm-only"
compile_only = "compile-only"
exec = "exec"
bench = "bench"
......@@ -42,12 +46,14 @@ class TestMode:
ast: [syntax],
compile_firm: [exec],
compile: [exec],
compile_firm_only: [semantic, compile_only],
compile_only: [semantic],
bench: [exec]
}
""" All 'success' tests of the n.th mode can used as 'success' tests for the n-1.th mode"""
TEST_MODES = [TestMode.lexer, TestMode.syntax, TestMode.ast, TestMode.semantic, TestMode.compile_firm,
TestMode.exec, TestMode.bench, TestMode.compile]
TEST_MODES = [TestMode.lexer, TestMode.syntax, TestMode.ast, TestMode.semantic, TestMode.compile_firm_only,
TestMode.compile_only, TestMode.compile_firm, TestMode.exec, TestMode.bench, TestMode.compile]
def get_test_dirname(mode: str) -> str:
......@@ -163,8 +169,10 @@ class Environment:
TestMode.syntax: "--parsetest",
TestMode.ast: "--print-ast",
TestMode.semantic: "--check",
TestMode.compile_only: None,
TestMode.compile_firm_only: "--compile-firm",
TestMode.compile_firm: "--compile-firm",
TestMode.compile: None
TestMode.compile: None,
}[mode]
cmd = [self.mj_run_cmd]
if mode_flag:
......@@ -172,13 +180,13 @@ class Environment:
cmd.extend(list(args))
return execute(cmd, timeout=timeout or self.timeout)
def run_command(self, cmd: str, *args: Tuple[str], timeout: float = None, input_str: Optional[str] = None) -> Tuple[bytes, bytes, int]:
def run_command(self, cmd: str, *args: Tuple[str], timeout: float = None, input_bytes: Optional[bytes] = None) -> Tuple[bytes, bytes, int]:
"""
Execute the passend command with its arguments
:return: (out, err, return code)
"""
return execute([cmd] + list(args), timeout=timeout or self.timeout, input_str=input_str)
return execute([cmd] + list(args), timeout=timeout or self.timeout, input_bytes=input_bytes)
def has_to_preprocess(self, file: str) -> bool:
return os.path.relpath(file, self.test_dir).startswith("exec")
......@@ -241,4 +249,4 @@ class Environment:
timer.cancel()
else:
stdout, stderr = proc.communicate()
return stdout, stderr, proc.returncode, timeouted
\ No newline at end of file
return stdout, stderr, proc.returncode, timeouted
......@@ -7,6 +7,8 @@ from mjtest.test.syntax_tests import BasicSyntaxTest
from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult
from os import path
from mjtest.util.utils import decode
_LOG = logging.getLogger("tests")
......@@ -59,6 +61,6 @@ class ASTPrettyPrintTest(BasicSyntaxTest):
with open(output_file, "wb") as f:
f.write(out)
f.flush()
return rtcode, out.decode(), err.decode()
return rtcode, decode(out), decode(err)
TestCase.TEST_CASE_CLASSES["ast"].append(ASTPrettyPrintTest)
......@@ -16,7 +16,7 @@ from mjtest.environment import TestMode, Environment
from mjtest.test.syntax_tests import BasicSyntaxTest
from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult
from mjtest.util.shell import SigKill
from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict
from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict, decode
_LOG = logging.getLogger("bench_tests")
......@@ -49,8 +49,8 @@ class BenchExecTest(BasicSyntaxTest):
INVALID_FILE_ENDINGS = [".inf.java", ".inf.mj"]
MODE = TestMode.compile_firm
def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str):
super().__init__(env, type, file, preprocessed_file)
def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str = ""):
super().__init__(env, type, file, preprocessed_file, log_file_mode)
self._should_succeed = True
def _bench_command(self, cmd: str, *args: Tuple[str]) -> _RunResult:
......@@ -85,7 +85,7 @@ class BenchExecTest(BasicSyntaxTest):
_LOG.error("File \"{}\" isn't valid Java".format(self.preprocessed_file))
test_result.incorrect_msg = "invalid java code, but output file missing"
test_result.set_error_code(javac_rtcode)
test_result.add_long_text("Javac error message", err.decode())
test_result.add_long_text("Javac error message", decode(err))
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
......@@ -102,8 +102,8 @@ class BenchExecTest(BasicSyntaxTest):
if rtcode != 0:
test_result.incorrect_msg = "file can't be compiled"
test_result.set_error_code(rtcode)
test_result.add_long_text("Error output", err.decode())
test_result.add_long_text("Output", out.decode())
test_result.add_long_text("Error output", decode(err))
test_result.add_long_text("Output", decode(out))
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
......
This diff is collapsed.
......@@ -3,6 +3,8 @@ from mjtest.environment import Environment, TestMode, TEST_MODES, get_test_dirna
from mjtest.test.tests import TestCase, BasicTestResult
from os import path
from mjtest.util.utils import decode
class BasicSyntaxTest(TestCase):
......@@ -35,7 +37,7 @@ class BasicSyntaxTest(TestCase):
def run(self) -> BasicTestResult:
out, err, rtcode = self.env.run_mj_command(self.MODE, self.preprocessed_file)
return BasicTestResult(self, rtcode, out.decode(), err.decode())
return BasicTestResult(self, rtcode, decode(out), decode(err))
TestCase.TEST_CASE_CLASSES[TestMode.syntax].append(BasicSyntaxTest)
......@@ -53,7 +55,7 @@ class JavaCompileTest(BasicSyntaxTest):
tmp_dir = self.env.create_pid_local_tmpdir()
_, self.javac_err, self.javac_rtcode = \
self.env.run_command("javac", path.relpath(preprocessed_file), "-d", tmp_dir, "-verbose")
self.javac_err = self.javac_err.decode() # type: str
self.javac_err = decode(self.javac_err) # type: str
self._should_succeed = self._is_file_syntactically_correct() if self.SYNTAX_TEST else self.javac_rtcode == 0
def _is_file_syntactically_correct(self):
......
from collections import namedtuple
import shutil
from typing import Optional, List, Tuple, T, Union, Dict
from typing import Optional, List, Tuple, Union, Dict
import collections
from mjtest.environment import Environment, TestMode, TEST_MODES, get_test_dirname
from os.path import join, exists, basename
......@@ -8,7 +8,7 @@ import logging
import os
import multiprocessing
from mjtest.util.parallelism import available_cpu_count
from mjtest.util.utils import cprint, colored
from mjtest.util.utils import cprint, colored, decode
from pprint import pprint
import shutil
import difflib
......@@ -50,7 +50,7 @@ class TestSuite:
correct_test_cases = set()
log_file = self._log_file_for_type(mode)
if exists(log_file):
with open(log_file) as f:
with open(log_file, errors="backslashreplace") as f:
correct_test_cases = set()
for t in f.readlines():
t = t.strip()
......@@ -223,7 +223,7 @@ class TestSuite:
os.mkdir(os.path.dirname(log_file))
except IOError:
pass
with open(log_file, "w") as f:
with open(log_file, "w", errors="backslashreplace") as f:
f.write("\n".join(self.correct_test_cases[mode]))
except IOError as e:
_LOG.exception("Caught i/o error while storing {}".format(log_file))
......@@ -314,7 +314,7 @@ class TestResult:
def store_at(self, file: str):
os.makedirs(os.path.dirname(file), exist_ok=True)
with open(file, "w") as f:
with open(file, "w", errors="backslashreplace") as f:
print(self.long_message(), file=f)
def short_message(self) -> str:
......@@ -348,8 +348,8 @@ class ExtensibleTestResult(TestResult):
self.messages.append(TestResultMessage(title, content, multiline=False, with_line_numbers=False))
def add_file(self, title: str, file_name: str, with_line_numbers: bool = True):
with open(file_name, "r") as f:
file_content = os.linesep.join([line.rstrip() for line in f])
with open(file_name, "r", errors="backslashreplace") as f:
file_content = os.linesep.join([line.rstrip() for line in f.readlines()])
self.add_long_text(title, file_content, with_line_numbers)
def succeeded(self):
......@@ -461,7 +461,7 @@ class BasicTestResult(TestResult):
def long_message(self) -> str:
file_content = []
with open(self.test_case.preprocessed_file, "r") as f:
with open(self.test_case.preprocessed_file, "r", errors="backslashreplace") as f:
file_content = [line.rstrip() for line in f]
others = []
for title, content, long_text in self.other_texts:
......@@ -563,13 +563,13 @@ class DiffTest(TestCase):
exp_out = ""
if rtcode == 0 and self.should_succeed():
if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE:
with open(self._expected_output_file, "r") as f:
with open(self._expected_output_file, "r", errors="backslashreplace") as f:
exp_out = f.read()
#else:
# _LOG.error("Expected output file for test case {}:{} is missing.".format(self.MODE, self.short_name()))
if self.type == self.MODE and self.env.mode == self.MODE:
return BasicDiffTestResult(self, rtcode, out.decode(), err.decode(), exp_out)
return BasicTestResult(self, rtcode, out.decode(), err.decode())
return BasicDiffTestResult(self, rtcode, decode(out), decode(err), exp_out)
return BasicTestResult(self, rtcode, decode(out), decode(err))
class LexerDiffTest(DiffTest):
......@@ -582,4 +582,4 @@ import mjtest.test.syntax_tests
import mjtest.test.ast_tests
import mjtest.test.semantic_tests
import mjtest.test.exec_tests
import mjtest.test.bench
\ No newline at end of file
import mjtest.test.bench
......@@ -16,7 +16,7 @@ try:
except ImportError:
has_resource_module = False
pass
import sys
import signal
import threading
......@@ -46,14 +46,14 @@ def _lower_rlimit(res, limit):
class _Execute(object):
def __init__(self, cmd, timeout, env, rlimit, input_str):
def __init__(self, cmd, timeout, env, rlimit, input_bytes):
self.cmd = cmd
self.timeout = timeout
self.env = env
self.proc = None
self.exception = None
self.rlimit = rlimit
self.input_str = input_str
self.input_bytes = input_bytes
MB = 1024 * 1024
if not 'RLIMIT_CORE' in rlimit:
rlimit['RLIMIT_CORE'] = 0
......@@ -80,14 +80,13 @@ class _Execute(object):
self.proc = subprocess.Popen(self.cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE if self.input_str else None,
stdin=subprocess.PIPE if self.input_bytes else None,
env=self.env,
shell=True,
**prexec_args)
#if self.input_str:
# self.proc.stdin.write(self.input_str.decode)
input_bytes = self.input_str.encode() if self.input_str else None
x = self.proc.communicate(input=input_bytes, timeout=self.timeout if self.timeout > 0.0 else None)
#if self.input_bytes:
# self.proc.stdin.write(self.input_bytes.decode)
x = self.proc.communicate(input=self.input_bytes, timeout=self.timeout if self.timeout > 0.0 else None)
self.out, self.err = x
self.returncode = self.proc.returncode
except subprocess.TimeoutExpired as t:
......@@ -110,7 +109,7 @@ class _Execute(object):
raise SigKill(-self.returncode, _EXIT_CODES[self.returncode] + ": " + os.strerror(-self.returncode))
return (self.out, self.err, self.returncode)
def execute(cmd, env=None, timeout=0, rlimit=None, propagate_sigint=True, input_str = None):
def execute(cmd, env=None, timeout=0, rlimit=None, propagate_sigint=True, input_bytes = None):
"""Execute a command and return stderr and stdout data"""
if not rlimit:
rlimit = dict()
......@@ -120,7 +119,7 @@ def execute(cmd, env=None, timeout=0, rlimit=None, propagate_sigint=True, input_
else:
#cmd = shlex.split(cmd[0]) + cmd[1:]
cmd = " ".join(shlex.quote(c) for c in cmd)
exc = _Execute(cmd, timeout, env, rlimit, input_str)
exc = _Execute(cmd, timeout, env, rlimit, input_bytes)
(out, err, returncode) = exc.run()
if returncode == -signal.SIGINT:
raise KeyboardInterrupt
......
......@@ -47,7 +47,7 @@ def cprint(text: str, *args, **kwargs):
def get_main_class_name(file: str) -> Optional[str]:
current_class = None
with open(file, "r") as f:
with open(file, "r", errors="backslashreplace") as f:
for line in f:
if line.startswith("class ") or line.startswith("/*public*/ class "):
match = re.search("[A-Za-z_0-9]+", line.replace("class ", "").replace("/*public*/", ""))
......@@ -113,4 +113,11 @@ class InsertionTimeOrderedDict:
ret = InsertionTimeOrderedDict()
for item in items:
ret[key_func(item)] = item
return ret
\ No newline at end of file
return ret
def decode(arr: bytes) -> str:
"""
Decodes the passed byte array as UTF8 and handles invalid characters
"""
return arr.decode("utf-8", "backslashreplace")
\ No newline at end of file
......@@ -48,7 +48,7 @@ class PreProcessor:
def add_commented(line: str):
middle_lines.append("/*{}*/".format(line))
with open(file, "r") as f:
with open(file, "r", errors="backslashreplace") as f:
for line in f:
line = line.rstrip()
if self._import_regexp.match(line):
......@@ -108,7 +108,7 @@ class PreProcessor:
print()
print(text)
else:
with open(self.dst_file, "w") as f:
with open(self.dst_file, "w", errors="backslashreplace") as f:
for text in reversed(self.imported_strs):
f.write(text)
f.write("\n\n")
......@@ -122,7 +122,7 @@ def is_importable_file(file: str) -> bool:
has_package = False
has_public_class = False
has_main_method = False
with open(file, "r") as f:
with open(file, "r", errors="backslashreplace") as f:
for line in f:
if line.startswith("package "):
has_package = True
......
Subproject commit 541910c3730523949b1b309e457f9cfdff7808b3
Subproject commit 9e7b9291db05fa7049738b435b2f818ed2aa29e1