Commit 63ba046a authored by Johannes Bechberger's avatar Johannes Bechberger

Fix bugs in compile-firm mode test runners

parent d67903a5
......@@ -6,9 +6,9 @@ import signal
from os import path
from mjtest.environment import TestMode, Environment
from mjtest.test.syntax_tests import BasicSyntaxTest
from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult
from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult
from mjtest.util.shell import SigKill
from mjtest.util.utils import get_main_class_name
from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict
_LOG = logging.getLogger("exec_tests")
......@@ -41,22 +41,38 @@ class JavaExecTest(BasicSyntaxTest):
def run(self) -> BasicDiffTestResult:
base_filename = path.basename(self.file).split(".")[0]
tmp_dir = self.env.create_pid_local_tmpdir()
tmp_dir = self.env.create_tmpdir()
shutil.copy(self.preprocessed_file, path.join(tmp_dir, base_filename + ".java"))
cwd = os.getcwd()
os.chdir(tmp_dir)
exp_out = None
#print(base_filename, get_main_class_name(base_filename + ".java"))
test_result = ExtensibleTestResult(self)
if not self._has_expected_output_file:
_, _, javac_rtcode = \
_, err, javac_rtcode = \
self.env.run_command("javac", base_filename + ".java")
if javac_rtcode != 0:
_LOG.error("File \"{}\" isn't valid Java".format(self.preprocessed_file))
test_result.incorrect_msg = "invalid java code, but output file missing"
test_result.error_code = javac_rtcode
test_result.add_long_text("Javac error message", err.decode())
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
raise InterruptedError()
exp_out, _, _ = \
return test_result
exp_out, err, java_rtcode = \
self.env.run_command("java", get_main_class_name(base_filename + ".java"))
test_result.add_long_text("Java output: ", exp_out.decode())
if javac_rtcode != 0:
test_result.incorrect_msg = "java runtime error"
test_result.error_code = java_rtcode
test_result.add_long_text("Java error message", err.decode())
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
exp_out = exp_out.decode().strip()
with open(self._prev_out_file, "w") as f:
f.write(exp_out)
f.flush()
......@@ -66,20 +82,64 @@ class JavaExecTest(BasicSyntaxTest):
if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE:
with open(self._expected_output_file, "r") as f:
exp_out = f.read()
test_result.add_short_text("Expected output file", self._expected_output_file)
test_result.add_long_text("Expected output", exp_out)
try:
_, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java")
out, _, _ = self.env.run_command("./" + base_filename)
out, err, rtcode = None, None, None
try:
out, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java")
if rtcode != 0:
test_result.incorrect_msg = "file can't be compiled"
test_result.error_code = rtcode
test_result.add_long_text("Error output", err.decode())
test_result.add_long_text("Output", out.decode())
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
except SigKill as sig:
test_result.incorrect_msg = "file can't be compiled: " + sig.name
test_result.error_code = sig.retcode
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
except:
os.chdir(cwd)
raise
try:
out, err, rtcode = self.env.run_command("./" + base_filename)
if rtcode != 0:
test_result.incorrect_msg = "file can't be run"
test_result.error_code = rtcode
test_result.add_long_text("Error output", err.decode())
test_result.add_long_text("Output", out.decode())
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
except SigKill as sig:
test_result.incorrect_msg = "binary can't be run: " + sig.name
test_result.error_code = sig.retcode
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
except:
os.chdir(cwd)
raise
out = out.decode().strip()
os.chdir(cwd)
if self.type == self.MODE and self.env.mode == self.MODE:
return BasicDiffTestResult(self, rtcode, out, err.decode(), exp_out)
test_result.add_long_text("Output", out.decode())
if exp_out.strip() != out.strip():
test_result.incorrect_msg = "incorrect output"
test_result.add_diff("Output diff [expected <-> actual]", exp_out, out)
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
return BasicTestResult(self, rtcode, out, err.decode())
except SigKill as sig:
os.chdir(cwd)
return BasicTestResult(self, sig.retcode, "", exp_out, sig.name)
assert False
except:
os.chdir(cwd)
raise
assert False
def _check_hash_sum(self, file: str, hash_sum_file: str) -> bool:
old_hash = ""
......
......@@ -296,6 +296,99 @@ class TestResult:
raise NotImplementedError()
class ExtensibleTestResult(TestResult):
def __init__(self, test_case: TestCase):
super().__init__(test_case, None)
self.messages = [] # type: List[TestResultMessage]
self.incorrect_msg = None # type: Optional[str]
self.has_succeeded = True # type: bool
self._contains_error_str = True # type: bool
def add_error_output(self, title: str, error_output: str):
"""
Checks for "error" string
"""
self._contains_error_str = self._contains_error_str and error_output is not None and "error" in error_output
self.messages.append(TestResultMessage(title, c))
def add_long_text(self, title: str, content: str, with_line_numbers: bool = True):
self.messages.append(TestResultMessage(title, content, multiline=True, with_line_numbers=with_line_numbers))
def add_short_text(self, title, content: str):
self.messages.append(TestResultMessage(title, content, multiline=False, with_line_numbers=False))
def add_file(self, title: str, file_name: str, with_line_numbers: bool = True):
with open(file_name, "r") as f:
file_content = os.linesep.join([line.rstrip() for line in f])
self.add_long_text(title, file_content, with_line_numbers)
def succeeded(self):
return self.has_succeeded
def is_correct(self):
if self.succeeded():
return super().is_correct()
else:
return super().is_correct() and self._contains_error_str
def short_message(self) -> str:
if self.is_correct():
return "correct"
else:
if not self.succeeded() and not self.test_case.should_succeed() and not self._contains_error_str:
return "the error output doesn't contain the word \"error\""
return self.incorrect_msg
def long_message(self) -> str:
texts = [self.short_message().capitalize()]
if self.error_code is not None:
texts.append("Error code: {}".format(self.error_code))
for msg in self.messages:
if msg.multiline:
texts.append(msg.title + ":")
texts.append("")
if msg.with_line_numbers:
texts.append(self._ident(msg.content))
else:
texts.append(msg.content)
else:
texts.append("{}: {}".format(msg.title, msg.content))
def _ident(self, text: Union[str,List[str]]) -> str:
arr = text if isinstance(text, list) else text.split("\n")
if len(arr) == 0 or text == "":
return ""
arr = ["[{:04d}] {:s}".format(i + 1, l) for (i, l) in enumerate(arr)]
return "\n".join(arr)
def add_diff(self, title: str, first: str, second: str, with_line_numbers: bool):
self.add_long_text(title, "".join(difflib.Differ().compare(first.splitlines(True), second.splitlines(True))),
with_line_numbers=with_line_numbers)
class TestResultMessage:
def __init__(self, title: str, content: str, multiline: bool, with_line_numbers: bool):
self.title = title
self.content = content
self.multiline = multiline
self.with_line_numbers = with_line_numbers
class TestResultFactory:
def __init__(self):
self._texts = [] # type: [str, str, bool]
self.return_code = 0
self.short_error_message = None # type: str
def add_short_texts(self, title: str, content: str):
self._texts.append((title, content, True))
def add_long_message(self, title: str, content: str):
self._texts.append((title, content, True))
class BasicTestResult(TestResult):
def __init__(self, test_case: TestCase, error_code: int, output: str = None, error_output: str = None,
......@@ -310,6 +403,10 @@ class BasicTestResult(TestResult):
self.add_additional_text("Output", output)
if error_output:
self.add_additional_text("Error output", error_output)
self.has_succeeded = error_code == 0
def succeeded(self):
return self.has_succeeded
def is_correct(self):
if self.succeeded():
......
import logging
from os import path
import sys
from typing import Tuple, Optional
from typing import Tuple, Optional, Any, List, Callable
import re
COLOR_OUTPUT_IF_POSSIBLE = False
......@@ -57,3 +57,60 @@ def get_main_class_name(file: str) -> Optional[str]:
elif "String[]" in line and "main" in line and "void" in line and "static" in line and "public" in line:
return current_class
return None
class InsertionTimeOrderedDict:
"""
A dictionary which's elements are sorted by their insertion time.
"""
def __init__(self):
self._dict = {}
self._keys = []
dict()
def __delitem__(self, key):
""" Remove the entry with the passed key """
del(self._dict[key])
del(self._keys[self._keys.index(key)])
def __getitem__(self, key):
""" Get the entry with the passed key """
return self._dict[key]
def __setitem__(self, key, value):
""" Set the value of the item with the passed key """
if key not in self._dict:
self._keys.append(key)
self._dict[key] = value
def __iter__(self):
""" Iterate over all keys """
return self._keys.__iter__()
def values(self) -> List:
""" Rerturns all values of this dictionary. They are sorted by their insertion time. """
return [self._dict[key] for key in self._keys]
def keys(self) -> List:
""" Returns all keys of this dictionary. They are sorted by their insertion time. """
return self._keys
def __len__(self):
""" Returns the number of items in this dictionary """
return len(self._keys)
@classmethod
def from_list(cls, items: Optional[list], key_func: Callable[[Any], Any]) -> 'InsertionTimeOrderedDict':
"""
Creates an ordered dict out of a list of elements.
:param items: list of elements
:param key_func: function that returns a key for each passed list element
:return: created ordered dict with the elements in the same order as in the passed list
"""
if items is None:
return InsertionTimeOrderedDict()
ret = InsertionTimeOrderedDict()
for item in items:
ret[key_func(item)] = item
return ret
\ No newline at end of file
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment