Commit 63ba046a authored by Johannes Bechberger's avatar Johannes Bechberger

Fix bugs in compile-firm mode test runners

parent d67903a5
...@@ -6,9 +6,9 @@ import signal ...@@ -6,9 +6,9 @@ import signal
from os import path from os import path
from mjtest.environment import TestMode, Environment from mjtest.environment import TestMode, Environment
from mjtest.test.syntax_tests import BasicSyntaxTest from mjtest.test.syntax_tests import BasicSyntaxTest
from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult
from mjtest.util.shell import SigKill from mjtest.util.shell import SigKill
from mjtest.util.utils import get_main_class_name from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict
_LOG = logging.getLogger("exec_tests") _LOG = logging.getLogger("exec_tests")
...@@ -41,22 +41,38 @@ class JavaExecTest(BasicSyntaxTest): ...@@ -41,22 +41,38 @@ class JavaExecTest(BasicSyntaxTest):
def run(self) -> BasicDiffTestResult: def run(self) -> BasicDiffTestResult:
base_filename = path.basename(self.file).split(".")[0] base_filename = path.basename(self.file).split(".")[0]
tmp_dir = self.env.create_pid_local_tmpdir() tmp_dir = self.env.create_tmpdir()
shutil.copy(self.preprocessed_file, path.join(tmp_dir, base_filename + ".java")) shutil.copy(self.preprocessed_file, path.join(tmp_dir, base_filename + ".java"))
cwd = os.getcwd() cwd = os.getcwd()
os.chdir(tmp_dir) os.chdir(tmp_dir)
exp_out = None exp_out = None
#print(base_filename, get_main_class_name(base_filename + ".java")) #print(base_filename, get_main_class_name(base_filename + ".java"))
test_result = ExtensibleTestResult(self)
if not self._has_expected_output_file: if not self._has_expected_output_file:
_, _, javac_rtcode = \ _, err, javac_rtcode = \
self.env.run_command("javac", base_filename + ".java") self.env.run_command("javac", base_filename + ".java")
if javac_rtcode != 0: if javac_rtcode != 0:
_LOG.error("File \"{}\" isn't valid Java".format(self.preprocessed_file)) _LOG.error("File \"{}\" isn't valid Java".format(self.preprocessed_file))
test_result.incorrect_msg = "invalid java code, but output file missing"
test_result.error_code = javac_rtcode
test_result.add_long_text("Javac error message", err.decode())
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd) os.chdir(cwd)
raise InterruptedError() return test_result
exp_out, _, _ = \ exp_out, err, java_rtcode = \
self.env.run_command("java", get_main_class_name(base_filename + ".java")) self.env.run_command("java", get_main_class_name(base_filename + ".java"))
test_result.add_long_text("Java output: ", exp_out.decode())
if javac_rtcode != 0:
test_result.incorrect_msg = "java runtime error"
test_result.error_code = java_rtcode
test_result.add_long_text("Java error message", err.decode())
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
exp_out = exp_out.decode().strip() exp_out = exp_out.decode().strip()
with open(self._prev_out_file, "w") as f: with open(self._prev_out_file, "w") as f:
f.write(exp_out) f.write(exp_out)
f.flush() f.flush()
...@@ -66,20 +82,64 @@ class JavaExecTest(BasicSyntaxTest): ...@@ -66,20 +82,64 @@ class JavaExecTest(BasicSyntaxTest):
if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE: if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE:
with open(self._expected_output_file, "r") as f: with open(self._expected_output_file, "r") as f:
exp_out = f.read() exp_out = f.read()
test_result.add_short_text("Expected output file", self._expected_output_file)
test_result.add_long_text("Expected output", exp_out)
try: try:
_, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java") out, err, rtcode = None, None, None
out, _, _ = self.env.run_command("./" + base_filename) try:
out, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java")
if rtcode != 0:
test_result.incorrect_msg = "file can't be compiled"
test_result.error_code = rtcode
test_result.add_long_text("Error output", err.decode())
test_result.add_long_text("Output", out.decode())
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
except SigKill as sig:
test_result.incorrect_msg = "file can't be compiled: " + sig.name
test_result.error_code = sig.retcode
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
except:
os.chdir(cwd)
raise
try:
out, err, rtcode = self.env.run_command("./" + base_filename)
if rtcode != 0:
test_result.incorrect_msg = "file can't be run"
test_result.error_code = rtcode
test_result.add_long_text("Error output", err.decode())
test_result.add_long_text("Output", out.decode())
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
except SigKill as sig:
test_result.incorrect_msg = "binary can't be run: " + sig.name
test_result.error_code = sig.retcode
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
except:
os.chdir(cwd)
raise
out = out.decode().strip() out = out.decode().strip()
os.chdir(cwd)
if self.type == self.MODE and self.env.mode == self.MODE: if self.type == self.MODE and self.env.mode == self.MODE:
return BasicDiffTestResult(self, rtcode, out, err.decode(), exp_out) test_result.add_long_text("Output", out.decode())
if exp_out.strip() != out.strip():
test_result.incorrect_msg = "incorrect output"
test_result.add_diff("Output diff [expected <-> actual]", exp_out, out)
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
return BasicTestResult(self, rtcode, out, err.decode()) return BasicTestResult(self, rtcode, out, err.decode())
except SigKill as sig: except SigKill as sig:
os.chdir(cwd) os.chdir(cwd)
return BasicTestResult(self, sig.retcode, "", exp_out, sig.name) assert False
except: except:
os.chdir(cwd) os.chdir(cwd)
raise assert False
def _check_hash_sum(self, file: str, hash_sum_file: str) -> bool: def _check_hash_sum(self, file: str, hash_sum_file: str) -> bool:
old_hash = "" old_hash = ""
......
...@@ -296,6 +296,99 @@ class TestResult: ...@@ -296,6 +296,99 @@ class TestResult:
raise NotImplementedError() raise NotImplementedError()
class ExtensibleTestResult(TestResult):
def __init__(self, test_case: TestCase):
super().__init__(test_case, None)
self.messages = [] # type: List[TestResultMessage]
self.incorrect_msg = None # type: Optional[str]
self.has_succeeded = True # type: bool
self._contains_error_str = True # type: bool
def add_error_output(self, title: str, error_output: str):
"""
Checks for "error" string
"""
self._contains_error_str = self._contains_error_str and error_output is not None and "error" in error_output
self.messages.append(TestResultMessage(title, c))
def add_long_text(self, title: str, content: str, with_line_numbers: bool = True):
self.messages.append(TestResultMessage(title, content, multiline=True, with_line_numbers=with_line_numbers))
def add_short_text(self, title, content: str):
self.messages.append(TestResultMessage(title, content, multiline=False, with_line_numbers=False))
def add_file(self, title: str, file_name: str, with_line_numbers: bool = True):
with open(file_name, "r") as f:
file_content = os.linesep.join([line.rstrip() for line in f])
self.add_long_text(title, file_content, with_line_numbers)
def succeeded(self):
return self.has_succeeded
def is_correct(self):
if self.succeeded():
return super().is_correct()
else:
return super().is_correct() and self._contains_error_str
def short_message(self) -> str:
if self.is_correct():
return "correct"
else:
if not self.succeeded() and not self.test_case.should_succeed() and not self._contains_error_str:
return "the error output doesn't contain the word \"error\""
return self.incorrect_msg
def long_message(self) -> str:
texts = [self.short_message().capitalize()]
if self.error_code is not None:
texts.append("Error code: {}".format(self.error_code))
for msg in self.messages:
if msg.multiline:
texts.append(msg.title + ":")
texts.append("")
if msg.with_line_numbers:
texts.append(self._ident(msg.content))
else:
texts.append(msg.content)
else:
texts.append("{}: {}".format(msg.title, msg.content))
def _ident(self, text: Union[str,List[str]]) -> str:
arr = text if isinstance(text, list) else text.split("\n")
if len(arr) == 0 or text == "":
return ""
arr = ["[{:04d}] {:s}".format(i + 1, l) for (i, l) in enumerate(arr)]
return "\n".join(arr)
def add_diff(self, title: str, first: str, second: str, with_line_numbers: bool):
self.add_long_text(title, "".join(difflib.Differ().compare(first.splitlines(True), second.splitlines(True))),
with_line_numbers=with_line_numbers)
class TestResultMessage:
def __init__(self, title: str, content: str, multiline: bool, with_line_numbers: bool):
self.title = title
self.content = content
self.multiline = multiline
self.with_line_numbers = with_line_numbers
class TestResultFactory:
def __init__(self):
self._texts = [] # type: [str, str, bool]
self.return_code = 0
self.short_error_message = None # type: str
def add_short_texts(self, title: str, content: str):
self._texts.append((title, content, True))
def add_long_message(self, title: str, content: str):
self._texts.append((title, content, True))
class BasicTestResult(TestResult): class BasicTestResult(TestResult):
def __init__(self, test_case: TestCase, error_code: int, output: str = None, error_output: str = None, def __init__(self, test_case: TestCase, error_code: int, output: str = None, error_output: str = None,
...@@ -310,6 +403,10 @@ class BasicTestResult(TestResult): ...@@ -310,6 +403,10 @@ class BasicTestResult(TestResult):
self.add_additional_text("Output", output) self.add_additional_text("Output", output)
if error_output: if error_output:
self.add_additional_text("Error output", error_output) self.add_additional_text("Error output", error_output)
self.has_succeeded = error_code == 0
def succeeded(self):
return self.has_succeeded
def is_correct(self): def is_correct(self):
if self.succeeded(): if self.succeeded():
......
import logging import logging
from os import path from os import path
import sys import sys
from typing import Tuple, Optional from typing import Tuple, Optional, Any, List, Callable
import re import re
COLOR_OUTPUT_IF_POSSIBLE = False COLOR_OUTPUT_IF_POSSIBLE = False
...@@ -57,3 +57,60 @@ def get_main_class_name(file: str) -> Optional[str]: ...@@ -57,3 +57,60 @@ def get_main_class_name(file: str) -> Optional[str]:
elif "String[]" in line and "main" in line and "void" in line and "static" in line and "public" in line: elif "String[]" in line and "main" in line and "void" in line and "static" in line and "public" in line:
return current_class return current_class
return None return None
class InsertionTimeOrderedDict:
"""
A dictionary which's elements are sorted by their insertion time.
"""
def __init__(self):
self._dict = {}
self._keys = []
dict()
def __delitem__(self, key):
""" Remove the entry with the passed key """
del(self._dict[key])
del(self._keys[self._keys.index(key)])
def __getitem__(self, key):
""" Get the entry with the passed key """
return self._dict[key]
def __setitem__(self, key, value):
""" Set the value of the item with the passed key """
if key not in self._dict:
self._keys.append(key)
self._dict[key] = value
def __iter__(self):
""" Iterate over all keys """
return self._keys.__iter__()
def values(self) -> List:
""" Rerturns all values of this dictionary. They are sorted by their insertion time. """
return [self._dict[key] for key in self._keys]
def keys(self) -> List:
""" Returns all keys of this dictionary. They are sorted by their insertion time. """
return self._keys
def __len__(self):
""" Returns the number of items in this dictionary """
return len(self._keys)
@classmethod
def from_list(cls, items: Optional[list], key_func: Callable[[Any], Any]) -> 'InsertionTimeOrderedDict':
"""
Creates an ordered dict out of a list of elements.
:param items: list of elements
:param key_func: function that returns a key for each passed list element
:return: created ordered dict with the elements in the same order as in the passed list
"""
if items is None:
return InsertionTimeOrderedDict()
ret = InsertionTimeOrderedDict()
for item in items:
ret[key_func(item)] = item
return ret
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment