Commit fbbd2749 authored by Johannes Bucher's avatar Johannes Bucher

Update mjtest and mjtest-tests

Merge remote-tracking branch 'gitlab-original/master'
parents d771772b 529180eb
...@@ -23,8 +23,10 @@ The test cases are divided in 6 'modes': ...@@ -23,8 +23,10 @@ The test cases are divided in 6 'modes':
them. them.
- __ast__: Test cases that check the generated ast by using the pretty printing functionality. - __ast__: Test cases that check the generated ast by using the pretty printing functionality.
- __semantic__: Test cases that check semantic checking of MiniJava programs - __semantic__: Test cases that check semantic checking of MiniJava programs
- __compile-firm__: Test cases that check the correct compilation and execution of MiniJava programs with the libfirm backend. - __compile-firm-only__: Test cases that check the compilation of MiniJava programs with the libfirm backend.
- __compile__: Test cases that check the correct compilation and execution of MiniJava programs with the self implemented backend. - __compile-firm__ (alias `exec-firm`): Test cases that check the correct compilation and execution of MiniJava programs with the libfirm backend.
- __compile-only__: Test cases that check the compilation MiniJava programs with the self implemented backend.
- __compile__ (alias `exec`): Test cases that check the correct compilation and execution of MiniJava programs with the self implemented backend.
The test different test cases for each mode are located in a folder with the same name. The test different test cases for each mode are located in a folder with the same name.
Except the compile-firm test cases which are located in the `exec` folder. Except the compile-firm test cases which are located in the `exec` folder.
...@@ -103,6 +105,19 @@ Test types for the semantic mode ...@@ -103,6 +105,19 @@ Test types for the semantic mode
__All semantic mode tests have to be syntactically correct__ __All semantic mode tests have to be syntactically correct__
Test types for the compile-firm-only and compile-only mode
----------------------------------------------------------
<table>
<tr><th>File ending(s) of test cases</th><th>Expected behaviour to complete a test of this type</th></tr>
<tr>
<td><code>.valid.mj</code> <code>.mj</code> <code>.valid.java</code> <code>.java</code></td>
<td>Return code is <code>0</code> and the compiler produces an `a.out` file, i.e. the MiniJava code can be compiled</td>
</tr>
</table>
__All valid semantic test-cases are also used, the non-semantic compile-(firm)-only
should be placed into the `compile-only` folder__
Test types for the compile-firm and compile mode Test types for the compile-firm and compile mode
------------------------------------------------ ------------------------------------------------
...@@ -112,7 +127,7 @@ Test types for the compile-firm and compile mode ...@@ -112,7 +127,7 @@ Test types for the compile-firm and compile mode
<tr> <tr>
<td><code>.java</code> <code>.mj</code></td> <td><code>.java</code> <code>.mj</code></td>
<td>The test file should compile. The resulting binary should be runnable and its output should be:<br/> <td>The test file should compile. The resulting binary should be runnable and its output should be:<br/>
if a <code>[test file].out</code> exists: the content of this file<br/> if a <code>[input file].out</code> (for test cases with input) or <code>[test file].out</code> (else) exists: the content of this file<br/>
if no such file exists: the same as the execution of the file with <code>java</code></td> if no such file exists: the same as the execution of the file with <code>java</code></td>
</tr> </tr>
<tr> <tr>
...@@ -171,12 +186,13 @@ usage: mjt.py [-h] [--only_incorrect_tests] [--all_exec_tests] ...@@ -171,12 +186,13 @@ usage: mjt.py [-h] [--only_incorrect_tests] [--all_exec_tests]
[--produce_no_reports] [--produce_all_reports] [--parallel] [--produce_no_reports] [--produce_all_reports] [--parallel]
[--output_no_incorrect_reports] [--color] [--ci_testing] [--output_no_incorrect_reports] [--color] [--ci_testing]
[--log_level LOG_LEVEL] [--log_level LOG_LEVEL]
{all,lexer,syntax,ast,semantic,compile-firm,compile} MJ_RUN {all,lexer,syntax,ast,semantic,compile-firm-only,compile-only,
compile-firm,compile,exec,exec-firm} MJ_RUN
MiniJava test runner MiniJava test runner
positional arguments: positional arguments:
{all,lexer,syntax,ast,semantic,compile-firm,exec} {all,lexer,syntax,ast,semantic,compile-firm-only,compile-only,compile-firm,exec,exec-firm}
What do you want to test? What do you want to test?
MJ_RUN Command to run your MiniJava implementation, e.g. MJ_RUN Command to run your MiniJava implementation, e.g.
`mj/run`, can be omitted by assigning the environment `mj/run`, can be omitted by assigning the environment
......
...@@ -27,7 +27,7 @@ class LogLevelChoices(argparse.Action): ...@@ -27,7 +27,7 @@ class LogLevelChoices(argparse.Action):
if True:#__name__ == '__main__': if True:#__name__ == '__main__':
parser = argparse.ArgumentParser(description="MiniJava test runner", add_help=True) parser = argparse.ArgumentParser(description="MiniJava test runner", add_help=True)
parser.add_argument("mode", parser.add_argument("mode",
choices=["all"] + TEST_MODES, choices=["all"] + TEST_MODES + ["exec-firm"],
help="What do you want to test?") help="What do you want to test?")
if os.getenv("MJ_RUN", None) is None: if os.getenv("MJ_RUN", None) is None:
parser.add_argument("mj_run", parser.add_argument("mj_run",
...@@ -84,9 +84,16 @@ if True:#__name__ == '__main__': ...@@ -84,9 +84,16 @@ if True:#__name__ == '__main__':
if ret is not None: if ret is not None:
failed += ret.failed failed += ret.failed
count += ret.count count += ret.count
if args["mode"] in ["exec", "exec-firm"]:
new_mode = args["mode"].replace("exec", "compile")
print(colored("\"{}\" is just an alias for \"{}\", using the latter instead"
.format(args["mode"], new_mode),
"yellow"))
args["mode"] = new_mode
if args["mode"] == "all": if args["mode"] == "all":
report_subdir = datetime.now().strftime("%d-%m-%y_%H-%M-%S") report_subdir = datetime.now().strftime("%d-%m-%y_%H-%M-%S")
for mode in [TestMode.lexer, TestMode.syntax, TestMode.ast, TestMode.semantic, for mode in [TestMode.lexer, TestMode.syntax, TestMode.ast, TestMode.semantic,
TestMode.compile_firm_only, TestMode.compile_only,
TestMode.compile_firm, TestMode.compile]: TestMode.compile_firm, TestMode.compile]:
args["all_exec_tests"] = True args["all_exec_tests"] = True
args["report_subdir"] = report_subdir + "_" + mode args["report_subdir"] = report_subdir + "_" + mode
......
...@@ -34,6 +34,10 @@ class TestMode: ...@@ -34,6 +34,10 @@ class TestMode:
compile = "compile" compile = "compile"
compile_firm_only = "compile-firm-only"
compile_only = "compile-only"
exec = "exec" exec = "exec"
bench = "bench" bench = "bench"
...@@ -42,12 +46,14 @@ class TestMode: ...@@ -42,12 +46,14 @@ class TestMode:
ast: [syntax], ast: [syntax],
compile_firm: [exec], compile_firm: [exec],
compile: [exec], compile: [exec],
compile_firm_only: [semantic, compile_only],
compile_only: [semantic],
bench: [exec] bench: [exec]
} }
""" All 'success' tests of the n.th mode can used as 'success' tests for the n-1.th mode""" """ All 'success' tests of the n.th mode can used as 'success' tests for the n-1.th mode"""
TEST_MODES = [TestMode.lexer, TestMode.syntax, TestMode.ast, TestMode.semantic, TestMode.compile_firm, TEST_MODES = [TestMode.lexer, TestMode.syntax, TestMode.ast, TestMode.semantic, TestMode.compile_firm_only,
TestMode.exec, TestMode.bench, TestMode.compile] TestMode.compile_only, TestMode.compile_firm, TestMode.exec, TestMode.bench, TestMode.compile]
def get_test_dirname(mode: str) -> str: def get_test_dirname(mode: str) -> str:
...@@ -163,8 +169,10 @@ class Environment: ...@@ -163,8 +169,10 @@ class Environment:
TestMode.syntax: "--parsetest", TestMode.syntax: "--parsetest",
TestMode.ast: "--print-ast", TestMode.ast: "--print-ast",
TestMode.semantic: "--check", TestMode.semantic: "--check",
TestMode.compile_only: None,
TestMode.compile_firm_only: "--compile-firm",
TestMode.compile_firm: "--compile-firm", TestMode.compile_firm: "--compile-firm",
TestMode.compile: None TestMode.compile: None,
}[mode] }[mode]
cmd = [self.mj_run_cmd] cmd = [self.mj_run_cmd]
if mode_flag: if mode_flag:
...@@ -172,13 +180,13 @@ class Environment: ...@@ -172,13 +180,13 @@ class Environment:
cmd.extend(list(args)) cmd.extend(list(args))
return execute(cmd, timeout=timeout or self.timeout) return execute(cmd, timeout=timeout or self.timeout)
def run_command(self, cmd: str, *args: Tuple[str], timeout: float = None, input_str: Optional[str] = None) -> Tuple[bytes, bytes, int]: def run_command(self, cmd: str, *args: Tuple[str], timeout: float = None, input_bytes: Optional[bytes] = None) -> Tuple[bytes, bytes, int]:
""" """
Execute the passend command with its arguments Execute the passend command with its arguments
:return: (out, err, return code) :return: (out, err, return code)
""" """
return execute([cmd] + list(args), timeout=timeout or self.timeout, input_str=input_str) return execute([cmd] + list(args), timeout=timeout or self.timeout, input_bytes=input_bytes)
def has_to_preprocess(self, file: str) -> bool: def has_to_preprocess(self, file: str) -> bool:
return os.path.relpath(file, self.test_dir).startswith("exec") return os.path.relpath(file, self.test_dir).startswith("exec")
...@@ -241,4 +249,4 @@ class Environment: ...@@ -241,4 +249,4 @@ class Environment:
timer.cancel() timer.cancel()
else: else:
stdout, stderr = proc.communicate() stdout, stderr = proc.communicate()
return stdout, stderr, proc.returncode, timeouted return stdout, stderr, proc.returncode, timeouted
\ No newline at end of file
...@@ -7,6 +7,8 @@ from mjtest.test.syntax_tests import BasicSyntaxTest ...@@ -7,6 +7,8 @@ from mjtest.test.syntax_tests import BasicSyntaxTest
from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult
from os import path from os import path
from mjtest.util.utils import decode
_LOG = logging.getLogger("tests") _LOG = logging.getLogger("tests")
...@@ -59,6 +61,6 @@ class ASTPrettyPrintTest(BasicSyntaxTest): ...@@ -59,6 +61,6 @@ class ASTPrettyPrintTest(BasicSyntaxTest):
with open(output_file, "wb") as f: with open(output_file, "wb") as f:
f.write(out) f.write(out)
f.flush() f.flush()
return rtcode, out.decode(), err.decode() return rtcode, decode(out), decode(err)
TestCase.TEST_CASE_CLASSES["ast"].append(ASTPrettyPrintTest) TestCase.TEST_CASE_CLASSES["ast"].append(ASTPrettyPrintTest)
...@@ -16,7 +16,7 @@ from mjtest.environment import TestMode, Environment ...@@ -16,7 +16,7 @@ from mjtest.environment import TestMode, Environment
from mjtest.test.syntax_tests import BasicSyntaxTest from mjtest.test.syntax_tests import BasicSyntaxTest
from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult
from mjtest.util.shell import SigKill from mjtest.util.shell import SigKill
from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict, decode
_LOG = logging.getLogger("bench_tests") _LOG = logging.getLogger("bench_tests")
...@@ -49,8 +49,8 @@ class BenchExecTest(BasicSyntaxTest): ...@@ -49,8 +49,8 @@ class BenchExecTest(BasicSyntaxTest):
INVALID_FILE_ENDINGS = [".inf.java", ".inf.mj"] INVALID_FILE_ENDINGS = [".inf.java", ".inf.mj"]
MODE = TestMode.compile_firm MODE = TestMode.compile_firm
def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str): def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str = ""):
super().__init__(env, type, file, preprocessed_file) super().__init__(env, type, file, preprocessed_file, log_file_mode)
self._should_succeed = True self._should_succeed = True
def _bench_command(self, cmd: str, *args: Tuple[str]) -> _RunResult: def _bench_command(self, cmd: str, *args: Tuple[str]) -> _RunResult:
...@@ -85,7 +85,7 @@ class BenchExecTest(BasicSyntaxTest): ...@@ -85,7 +85,7 @@ class BenchExecTest(BasicSyntaxTest):
_LOG.error("File \"{}\" isn't valid Java".format(self.preprocessed_file)) _LOG.error("File \"{}\" isn't valid Java".format(self.preprocessed_file))
test_result.incorrect_msg = "invalid java code, but output file missing" test_result.incorrect_msg = "invalid java code, but output file missing"
test_result.set_error_code(javac_rtcode) test_result.set_error_code(javac_rtcode)
test_result.add_long_text("Javac error message", err.decode()) test_result.add_long_text("Javac error message", decode(err))
test_result.add_file("Source file", self.preprocessed_file) test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd) os.chdir(cwd)
return test_result return test_result
...@@ -102,8 +102,8 @@ class BenchExecTest(BasicSyntaxTest): ...@@ -102,8 +102,8 @@ class BenchExecTest(BasicSyntaxTest):
if rtcode != 0: if rtcode != 0:
test_result.incorrect_msg = "file can't be compiled" test_result.incorrect_msg = "file can't be compiled"
test_result.set_error_code(rtcode) test_result.set_error_code(rtcode)
test_result.add_long_text("Error output", err.decode()) test_result.add_long_text("Error output", decode(err))
test_result.add_long_text("Output", out.decode()) test_result.add_long_text("Output", decode(out))
test_result.add_file("Source file", self.preprocessed_file) test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd) os.chdir(cwd)
return test_result return test_result
......
...@@ -14,7 +14,7 @@ from mjtest.environment import TestMode, Environment ...@@ -14,7 +14,7 @@ from mjtest.environment import TestMode, Environment
from mjtest.test.syntax_tests import BasicSyntaxTest from mjtest.test.syntax_tests import BasicSyntaxTest
from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult
from mjtest.util.shell import SigKill from mjtest.util.shell import SigKill
from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict, decode
_LOG = logging.getLogger("exec_tests") _LOG = logging.getLogger("exec_tests")
...@@ -48,6 +48,7 @@ class JavaExecTest(BasicSyntaxTest): ...@@ -48,6 +48,7 @@ class JavaExecTest(BasicSyntaxTest):
self._expected_output_file = self._prev_out_file self._expected_output_file = self._prev_out_file
_LOG.info("Reuse old java output file \"{}\"".format(path.relpath(self._prev_out_file))) _LOG.info("Reuse old java output file \"{}\"".format(path.relpath(self._prev_out_file)))
self._should_succeed = True self._should_succeed = True
self._only_compile = self.MODE.endswith("-only")
def run(self) -> BasicDiffTestResult: def run(self) -> BasicDiffTestResult:
is_big_testcase = "big" in self.file is_big_testcase = "big" in self.file
...@@ -58,33 +59,32 @@ class JavaExecTest(BasicSyntaxTest): ...@@ -58,33 +59,32 @@ class JavaExecTest(BasicSyntaxTest):
cwd = os.getcwd() cwd = os.getcwd()
os.chdir(tmp_dir) os.chdir(tmp_dir)
exp_out = None exp_out = None
input_str = None input_bytes = None
#print(base_filename, get_main_class_name(base_filename + ".java")) #print(base_filename, get_main_class_name(base_filename + ".java"))
test_result = ExtensibleTestResult(self) test_result = ExtensibleTestResult(self)
if self._has_input_file: if self._has_input_file:
if self._has_character_input: if self._has_character_input:
with open(self._input_file, "r") as f: with open(self._input_file, "rb") as f:
input_str = f.read() input_bytes = f.read()
else: else:
with open(self._input_file, "r") as f: with open(self._input_file, "r", errors="backslashreplace") as f:
chars = bytearray('ascii', 'ignore') # type: bytearray input_bytes = bytearray(encoding='ascii', errors='ignore') # type: bytearray
for line in f.readlines(): for line in f.readlines():
for part in line.split(" "): for part in line.split(" "):
part = part.strip() part = part.strip()
if len(part) > 0: if len(part) > 0:
chars.append(int(part)) chars.append(int(part))
input_str = chars.decode()
if not self._has_expected_output_file: if not self._has_expected_output_file and not self._only_compile:
_, err, javac_rtcode = \ _, err, javac_rtcode = \
self.env.run_command("javac", base_filename + ".java", timeout=timeout) self.env.run_command("javac", base_filename + ".java", timeout=timeout)
if javac_rtcode != 0: if javac_rtcode != 0:
_LOG.error("File \"{}\" isn't valid Java".format(self.preprocessed_file)) _LOG.error("File \"{}\" isn't valid Java".format(self.preprocessed_file))
test_result.incorrect_msg = "invalid java code, but output file missing" test_result.incorrect_msg = "invalid java code, but output file missing"
test_result.set_error_code(javac_rtcode) test_result.set_error_code(javac_rtcode)
test_result.add_long_text("Javac error message", err.decode()) test_result.add_long_text("Javac error message", decode(err))
test_result.add_file("Source file", self.preprocessed_file) test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd) os.chdir(cwd)
return test_result return test_result
...@@ -92,30 +92,31 @@ class JavaExecTest(BasicSyntaxTest): ...@@ -92,30 +92,31 @@ class JavaExecTest(BasicSyntaxTest):
if not main_class: if not main_class:
_LOG.debug("Can't find a main class, using the file name instead") _LOG.debug("Can't find a main class, using the file name instead")
main_class = base_filename main_class = base_filename
exp_out, err, java_rtcode = \ if not self._only_compile:
self.env.run_command("java", get_main_class_name(base_filename + ".java"), timeout=timeout, input_str=input_str) exp_out, err, java_rtcode = \
if javac_rtcode != 0: self.env.run_command("java", get_main_class_name(base_filename + ".java"), timeout=timeout, input_bytes=input_bytes)
test_result.add_long_text("Java output: ", exp_out.decode()) if javac_rtcode != 0:
test_result.incorrect_msg = "java runtime error" test_result.add_long_text("Java output: ", decode(exp_out))
test_result.set_error_code(java_rtcode) test_result.incorrect_msg = "java runtime error"
test_result.add_long_text("Java error message", err.decode()) test_result.set_error_code(java_rtcode)
test_result.add_file("Source file", self.preprocessed_file) test_result.add_long_text("Java error message", decode(err))
if self._input_file: test_result.add_file("Source file", self.preprocessed_file)
test_result.add_file("Input file", self.file) if self._input_file:
os.chdir(cwd) test_result.add_file("Input file", self.file)
return test_result os.chdir(cwd)
exp_out = exp_out.decode() return test_result
exp_out = decode(exp_out)
with open(self._prev_out_file, "w") as f:
f.write(exp_out) with open(self._prev_out_file, "w", errors="backslashreplace") as f:
f.flush() f.write(exp_out)
with open(self._prev_out_hash_file, "w") as f: f.flush()
f.write(self._hash_sum_for_file(base_filename + ".java")) with open(self._prev_out_hash_file, "w", errors="backslashreplace") as f:
f.flush() f.write(self._hash_sum_for_file(base_filename + ".java"))
if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE: f.flush()
with open(self._expected_output_file, "r") as f: if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE \
and not self._only_compile:
with open(self._expected_output_file, "r", errors="backslashreplace") as f:
exp_out = f.read() exp_out = f.read()
try: try:
out, err, rtcode = None, None, None out, err, rtcode = None, None, None
try: try:
...@@ -123,8 +124,8 @@ class JavaExecTest(BasicSyntaxTest): ...@@ -123,8 +124,8 @@ class JavaExecTest(BasicSyntaxTest):
if rtcode != 0: if rtcode != 0:
test_result.incorrect_msg = "file can't be compiled" test_result.incorrect_msg = "file can't be compiled"
test_result.set_error_code(rtcode) test_result.set_error_code(rtcode)
test_result.add_long_text("Error output", err.decode()) test_result.add_long_text("Error output", decode(err))
test_result.add_long_text("Output", out.decode()) test_result.add_long_text("Output", decode(out))
test_result.add_file("Source file", self.preprocessed_file) test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd) os.chdir(cwd)
return test_result return test_result
...@@ -137,13 +138,19 @@ class JavaExecTest(BasicSyntaxTest): ...@@ -137,13 +138,19 @@ class JavaExecTest(BasicSyntaxTest):
except: except:
os.chdir(cwd) os.chdir(cwd)
raise raise
if self._only_compile:
if not os.path.exists("a.out"):
test_result.incorrect_msg = "binary cannot be found"
test_result.set_error_code(1)
test_result.add_file("Source file", self.preprocessed_file)
return test_result
try: try:
out, err, rtcode = self.env.run_command("./a.out", timeout=timeout, input_str=input_str) out, err, rtcode = self.env.run_command("./a.out", timeout=timeout, input_bytes=input_bytes)
if rtcode != 0: if rtcode != 0:
test_result.incorrect_msg = "binary can't be run, non zero error code" test_result.incorrect_msg = "binary can't be run, non zero error code"
test_result.set_error_code(rtcode) test_result.set_error_code(rtcode)
test_result.add_long_text("Error output", err.decode()) test_result.add_long_text("Error output", decode(err))
test_result.add_long_text("Output", out.decode()) test_result.add_long_text("Output", decode(out))
test_result.add_file("Source file", self.preprocessed_file) test_result.add_file("Source file", self.preprocessed_file)
if self._input_file: if self._input_file:
test_result.add_file("Input file", self.file) test_result.add_file("Input file", self.file)
...@@ -160,7 +167,7 @@ class JavaExecTest(BasicSyntaxTest): ...@@ -160,7 +167,7 @@ class JavaExecTest(BasicSyntaxTest):
except: except:
os.chdir(cwd) os.chdir(cwd)
raise raise
out = out.decode() out = decode(out)
if self.type == self.MODE and self.env.mode == self.MODE: if self.type == self.MODE and self.env.mode == self.MODE:
if exp_out.strip() != out.strip(): if exp_out.strip() != out.strip():
test_result.incorrect_msg = "incorrect output" test_result.incorrect_msg = "incorrect output"
...@@ -176,7 +183,7 @@ class JavaExecTest(BasicSyntaxTest): ...@@ -176,7 +183,7 @@ class JavaExecTest(BasicSyntaxTest):
test_result.add_long_text("Output", out) test_result.add_long_text("Output", out)
os.chdir(cwd) os.chdir(cwd)
return test_result return test_result
return BasicTestResult(self, rtcode, out, err.decode()) return BasicTestResult(self, rtcode, out, decode(err))
except SigKill as sig: except SigKill as sig:
os.chdir(cwd) os.chdir(cwd)
assert False assert False
...@@ -186,7 +193,7 @@ class JavaExecTest(BasicSyntaxTest): ...@@ -186,7 +193,7 @@ class JavaExecTest(BasicSyntaxTest):
def _check_hash_sum(self, file: str, hash_sum_file: str) -> bool: def _check_hash_sum(self, file: str, hash_sum_file: str) -> bool:
old_hash = "" old_hash = ""
with open(hash_sum_file, "r") as f: with open(hash_sum_file, "r", errors="backslashreplace") as f:
try: try:
old_hash = f.readline().strip() old_hash = f.readline().strip()
except UnicodeDecodeError: except UnicodeDecodeError:
...@@ -195,7 +202,7 @@ class JavaExecTest(BasicSyntaxTest): ...@@ -195,7 +202,7 @@ class JavaExecTest(BasicSyntaxTest):
@classmethod @classmethod
def _hash_sum_for_file(self, file: str) -> str: def _hash_sum_for_file(self, file: str) -> str:
with open(file, "r") as f: with open(file, "r", errors="backslashreplace") as f:
return hashlib.sha256(f.read().encode()).hexdigest() return hashlib.sha256(f.read().encode()).hexdigest()
@classmethod @classmethod
...@@ -228,6 +235,16 @@ class JavaCompileExecTest(JavaExecTest): ...@@ -228,6 +235,16 @@ class JavaCompileExecTest(JavaExecTest):
TestCase.TEST_CASE_CLASSES[TestMode.compile].append(JavaCompileExecTest) TestCase.TEST_CASE_CLASSES[TestMode.compile].append(JavaCompileExecTest)
class JavaCompileOnlyTest(JavaExecTest):
MODE = TestMode.compile_only
TestCase.TEST_CASE_CLASSES[TestMode.compile_only].append(JavaCompileOnlyTest)
class JavaCompileFirmOnlyTest(JavaExecTest):
MODE = TestMode.compile_firm_only
TestCase.TEST_CASE_CLASSES[TestMode.compile_firm_only].append(JavaCompileFirmOnlyTest)
class JavaInfiniteLoopTest(BasicSyntaxTest): class JavaInfiniteLoopTest(BasicSyntaxTest):
FILE_ENDINGS = [".inf.java", ".inf.mj"] FILE_ENDINGS = [".inf.java", ".inf.mj"]
...@@ -255,8 +272,8 @@ class JavaInfiniteLoopTest(BasicSyntaxTest): ...@@ -255,8 +272,8 @@ class JavaInfiniteLoopTest(BasicSyntaxTest):
if rtcode != 0: if rtcode != 0:
test_result.incorrect_msg = "file can't be compiled" test_result.incorrect_msg = "file can't be compiled"
test_result.set_error_code(rtcode) test_result.set_error_code(rtcode)
test_result.add_long_text("Error output", err.decode()) test_result.add_long_text("Error output", decode(err))
test_result.add_long_text("Output", out.decode()) test_result.add_long_text("Output", decode(out))
test_result.add_file("Source file", self.preprocessed_file) test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd) os.chdir(cwd)
return test_result return test_result
...@@ -264,19 +281,19 @@ class JavaInfiniteLoopTest(BasicSyntaxTest): ...@@ -264,19 +281,19 @@ class JavaInfiniteLoopTest(BasicSyntaxTest):
if rtcode != 0: if rtcode != 0:
test_result.incorrect_msg = "binary can't be run, non zero error code" test_result.incorrect_msg = "binary can't be run, non zero error code"
test_result.set_error_code(rtcode) test_result.set_error_code(rtcode)
test_result.add_long_text("Error output", err.decode()) test_result.add_long_text("Error output", decode(err))
test_result.add_long_text("Output", out.decode()) test_result.add_long_text("Output", decode(out))
test_result.add_file("Source file", self.preprocessed_file) test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd) os.chdir(cwd)
return test_result return test_result
except SigKill as sig: except SigKill as sig:
if sig.retcode == signal.SIGXCPU: if sig.retcode == signal.SIGXCPU:
out, _, _, _ = self.env.run_command_till_timeout("./a.out", timeout=1) out, _, _, _ = self.env.run_command_till_timeout("./a.out", timeout=1)
test_result.add_long_text("Output", out.decode()) test_result.add_long_text("Output", decode(out))
if self._has_output_file: if self._has_output_file:
out = out.decode().strip() out = decode(out).strip()
exp_out = "" exp_out = ""
with open(self._output_file, "r") as f: with open(self._output_file, "r", errors="backslashreplace") as f:
exp_out = f.read().strip() exp_out = f.read().strip()
test_result.add_long_text("Expected output start", exp_out) test_result.add_long_text("Expected output start", exp_out)
if not out.startswith(exp_out): if not out.startswith(exp_out):
...@@ -342,8 +359,8 @@ class InvalidJavaExecTest(BasicSyntaxTest): ...@@ -342,8 +359,8 @@ class InvalidJavaExecTest(BasicSyntaxTest):
if rtcode != 0: if rtcode != 0:
test_result.incorrect_msg = "file can't be compiled" test_result.incorrect_msg = "file can't be compiled"
test_result.set_error_code(-rtcode) test_result.set_error_code(-rtcode)
test_result.add_long_text("Error output", err.decode()) test_result.add_long_text("Error output", decode(err))
test_result.add_long_text("Output", out.decode()) test_result.add_long_text("Output", decode(out))
test_result.add_file("Source file", self.preprocessed_file) test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd) os.chdir(cwd)
return test_result return test_result
...@@ -361,8 +378,8 @@ class InvalidJavaExecTest(BasicSyntaxTest): ...@@ -361,8 +378,8 @@ class InvalidJavaExecTest(BasicSyntaxTest):
if rtcode != 0: if rtcode != 0: