Commit fcaf1778 authored by Johannes Bechberger's avatar Johannes Bechberger

Add tests with input files

And fix test timeouts.
parent 4a6e643d
...@@ -8,7 +8,7 @@ import sys ...@@ -8,7 +8,7 @@ import sys
from datetime import datetime from datetime import datetime
import time import time
from threading import Timer from threading import Timer
from typing import Dict from typing import Dict, Optional
import subprocess import subprocess
...@@ -165,13 +165,13 @@ class Environment: ...@@ -165,13 +165,13 @@ class Environment:
cmd = [self.mj_run_cmd, mode_flag] + list(args) cmd = [self.mj_run_cmd, mode_flag] + list(args)
return execute(cmd, timeout=timeout or self.timeout) return execute(cmd, timeout=timeout or self.timeout)
def run_command(self, cmd: str, *args: Tuple[str], timeout: float = None) -> Tuple[bytes, bytes, int]: def run_command(self, cmd: str, *args: Tuple[str], timeout: float = None, input_str: Optional[str] = None) -> Tuple[bytes, bytes, int]:
""" """
Execute the passend command with its arguments Execute the passend command with its arguments
:return: (out, err, return code) :return: (out, err, return code)
""" """
return execute([cmd] + list(args), timeout=timeout or self.timeout) return execute([cmd] + list(args), timeout=timeout or self.timeout, input_str=input_str)
def has_to_preprocess(self, file: str) -> bool: def has_to_preprocess(self, file: str) -> bool:
return os.path.relpath(file, self.test_dir).startswith("exec") return os.path.relpath(file, self.test_dir).startswith("exec")
......
...@@ -4,6 +4,10 @@ import os ...@@ -4,6 +4,10 @@ import os
import shutil import shutil
import signal import signal
from os import path from os import path
import re
from typing import Optional
from mjtest.environment import TestMode, Environment from mjtest.environment import TestMode, Environment
from mjtest.test.syntax_tests import BasicSyntaxTest from mjtest.test.syntax_tests import BasicSyntaxTest
from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult
...@@ -18,12 +22,16 @@ class JavaExecTest(BasicSyntaxTest): ...@@ -18,12 +22,16 @@ class JavaExecTest(BasicSyntaxTest):
""" """
FILE_ENDINGS = [".java", ".mj"] FILE_ENDINGS = [".java", ".mj"]
INVALID_FILE_ENDINGS = [".inf.java", ".inf.mj"] INVALID_FILE_ENDINGS = [".inf.java", ".inf.mj", ".input.mj", ".input.java"]
OUTPUT_FILE_ENDING = ".out" OUTPUT_FILE_ENDING = ".out"
MODE = TestMode.compile_firm MODE = TestMode.compile_firm
INPUT_FILE_REGEXP = r'(\.[0-9]+)\.input(c?)$'
def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str): def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str):
super().__init__(env, type, file, preprocessed_file) super().__init__(env, type, file, preprocessed_file)
self._has_input_file = bool(re.search(self.INPUT_FILE_REGEXP, file))
self._input_file = file if self._has_input_file else None
self._has_character_input = file.endswith("c") if self._has_input_file else False
self._expected_output_file = file + self.OUTPUT_FILE_ENDING self._expected_output_file = file + self.OUTPUT_FILE_ENDING
prev_out_dir = path.join(path.dirname(file), ".java_output") prev_out_dir = path.join(path.dirname(file), ".java_output")
if not path.exists(prev_out_dir): if not path.exists(prev_out_dir):
...@@ -48,10 +56,25 @@ class JavaExecTest(BasicSyntaxTest): ...@@ -48,10 +56,25 @@ class JavaExecTest(BasicSyntaxTest):
cwd = os.getcwd() cwd = os.getcwd()
os.chdir(tmp_dir) os.chdir(tmp_dir)
exp_out = None exp_out = None
input_str = None
#print(base_filename, get_main_class_name(base_filename + ".java")) #print(base_filename, get_main_class_name(base_filename + ".java"))
test_result = ExtensibleTestResult(self) test_result = ExtensibleTestResult(self)
if self._has_input_file:
if self._has_character_input:
with open(self._input_file, "r") as f:
input_str = f.read()
else:
with open(self._input_file, "r") as f:
chars = bytearray('ascii', 'ignore') # type: bytearray
for line in f.readlines():
for part in line.split(" "):
part = part.strip()
if len(part) > 0:
chars.append(int(part))
input_str = chars.decode()
if not self._has_expected_output_file: if not self._has_expected_output_file:
_, err, javac_rtcode = \ _, err, javac_rtcode = \
self.env.run_command("javac", base_filename + ".java", timeout=timeout) self.env.run_command("javac", base_filename + ".java", timeout=timeout)
...@@ -68,13 +91,15 @@ class JavaExecTest(BasicSyntaxTest): ...@@ -68,13 +91,15 @@ class JavaExecTest(BasicSyntaxTest):
_LOG.debug("Can't find a main class, using the file name instead") _LOG.debug("Can't find a main class, using the file name instead")
main_class = base_filename main_class = base_filename
exp_out, err, java_rtcode = \ exp_out, err, java_rtcode = \
self.env.run_command("java", get_main_class_name(base_filename + ".java"), timeout=timeout) self.env.run_command("java", get_main_class_name(base_filename + ".java"), timeout=timeout, input_str=input_str)
if javac_rtcode != 0: if javac_rtcode != 0:
test_result.add_long_text("Java output: ", exp_out.decode()) test_result.add_long_text("Java output: ", exp_out.decode())
test_result.incorrect_msg = "java runtime error" test_result.incorrect_msg = "java runtime error"
test_result.set_error_code(java_rtcode) test_result.set_error_code(java_rtcode)
test_result.add_long_text("Java error message", err.decode()) test_result.add_long_text("Java error message", err.decode())
test_result.add_file("Source file", self.preprocessed_file) test_result.add_file("Source file", self.preprocessed_file)
if self._input_file:
test_result.add_file("Input file", self.file)
os.chdir(cwd) os.chdir(cwd)
return test_result return test_result
exp_out = exp_out.decode() exp_out = exp_out.decode()
...@@ -111,19 +136,23 @@ class JavaExecTest(BasicSyntaxTest): ...@@ -111,19 +136,23 @@ class JavaExecTest(BasicSyntaxTest):
os.chdir(cwd) os.chdir(cwd)
raise raise
try: try:
out, err, rtcode = self.env.run_command("./a.out", timeout=timeout) out, err, rtcode = self.env.run_command("./a.out", timeout=timeout, input_str=input_str)
if rtcode != 0: if rtcode != 0:
test_result.incorrect_msg = "binary can't be run, non zero error code" test_result.incorrect_msg = "binary can't be run, non zero error code"
test_result.set_error_code(rtcode) test_result.set_error_code(rtcode)
test_result.add_long_text("Error output", err.decode()) test_result.add_long_text("Error output", err.decode())
test_result.add_long_text("Output", out.decode()) test_result.add_long_text("Output", out.decode())
test_result.add_file("Source file", self.preprocessed_file) test_result.add_file("Source file", self.preprocessed_file)
if self._input_file:
test_result.add_file("Input file", self.file)
os.chdir(cwd) os.chdir(cwd)
return test_result return test_result
except SigKill as sig: except SigKill as sig:
test_result.incorrect_msg = "binary can't be run: " + sig.name.strip() test_result.incorrect_msg = "binary can't be run: " + sig.name.strip()
test_result.set_error_code(sig.retcode) test_result.set_error_code(sig.retcode)
test_result.add_file("Source file", self.preprocessed_file) test_result.add_file("Source file", self.preprocessed_file)
if self._input_file:
test_result.add_file("Input file", self.file)
os.chdir(cwd) os.chdir(cwd)
return test_result return test_result
except: except:
...@@ -140,6 +169,8 @@ class JavaExecTest(BasicSyntaxTest): ...@@ -140,6 +169,8 @@ class JavaExecTest(BasicSyntaxTest):
test_result.add_long_text("Actual output", out) test_result.add_long_text("Actual output", out)
test_result.add_diff("Output diff [expected <-> actual]", exp_out, out, with_line_numbers=True) test_result.add_diff("Output diff [expected <-> actual]", exp_out, out, with_line_numbers=True)
test_result.add_file("Source file", self.preprocessed_file) test_result.add_file("Source file", self.preprocessed_file)
if self._input_file:
test_result.add_file("Input file", self.file)
os.chdir(cwd) os.chdir(cwd)
return test_result return test_result
return BasicTestResult(self, rtcode, out, err.decode()) return BasicTestResult(self, rtcode, out, err.decode())
...@@ -159,10 +190,34 @@ class JavaExecTest(BasicSyntaxTest): ...@@ -159,10 +190,34 @@ class JavaExecTest(BasicSyntaxTest):
_LOG.exception("Decoding a hash sum for java output caching failed") _LOG.exception("Decoding a hash sum for java output caching failed")
return self._hash_sum_for_file(file) == old_hash return self._hash_sum_for_file(file) == old_hash
@classmethod
def _hash_sum_for_file(self, file: str) -> str: def _hash_sum_for_file(self, file: str) -> str:
with open(file, "r") as f: with open(file, "r") as f:
return hashlib.sha256(f.read().encode()).hexdigest() return hashlib.sha256(f.read().encode()).hexdigest()
@classmethod
def _test_case_file_for_input_file(cls, input_file: str) -> Optional[str]:
base = re.sub(cls.INPUT_FILE_REGEXP, "", input_file)
if path.exists(base + ".input.mj"):
return base + ".input.mj"
if path.exists(base + ".input.java"):
return base + ".input.java"
return None
@staticmethod
def is_file_ending_valid(cls, file: str) -> bool:
if re.search(cls.INPUT_FILE_REGEXP, file) and ".java_output" not in file and "precprocessor" not in file:
if not JavaExecTest._test_case_file_for_input_file(file):
_LOG.error("Skip {} that hasn't a corresponding test case file".format(file))
return True
return super().is_file_ending_valid(cls, file)
@staticmethod
def _get_test_case_file(cls, file: str) -> Optional[str]:
if re.search(JavaExecTest.INPUT_FILE_REGEXP, file):
return JavaExecTest._test_case_file_for_input_file(file)
return file
TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(JavaExecTest) TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(JavaExecTest)
......
...@@ -78,7 +78,7 @@ class TestSuite: ...@@ -78,7 +78,7 @@ class TestSuite:
if not p.startswith("."): if not p.startswith("."):
file_names.append(p) file_names.append(p)
for file in sorted(file_names): for file in sorted(file_names):
if not TestCase.has_valid_file_ending(self.env.mode, file): if not TestCase.has_valid_file_ending(self.env.mode, join(dir, file)):
_LOG.debug("Skip file " + file) _LOG.debug("Skip file " + file)
elif self.env.only_incorrect_tests and file in correct_test_cases: elif self.env.only_incorrect_tests and file in correct_test_cases:
_LOG.info("Skip file {} as its test case was executed correctly the last run") _LOG.info("Skip file {} as its test case was executed correctly the last run")
...@@ -87,7 +87,7 @@ class TestSuite: ...@@ -87,7 +87,7 @@ class TestSuite:
if self.env.has_to_preprocess(file_path) and self.env.is_lib_file(file_path): if self.env.has_to_preprocess(file_path) and self.env.is_lib_file(file_path):
_LOG.debug("Skip lib file '{}'".format(file)) _LOG.debug("Skip lib file '{}'".format(file))
continue continue
preprocessed = self.env.preprocess(join(dir, file)) preprocessed = self.env.preprocess(join(dir, TestCase.get_test_case_file(self.env, join(dir, file))))
test_case = TestCase.create_from_file(self.env, m, join(dir, file), preprocessed) test_case = TestCase.create_from_file(self.env, m, join(dir, file), preprocessed)
if not test_case: if not test_case:
pass pass
...@@ -261,8 +261,7 @@ class TestCase: ...@@ -261,8 +261,7 @@ class TestCase:
@classmethod @classmethod
def _test_case_class_for_file(cls, type: str, file: str): def _test_case_class_for_file(cls, type: str, file: str):
for t in cls.TEST_CASE_CLASSES[type]: for t in cls.TEST_CASE_CLASSES[type]:
if any(file.endswith(e) for e in t.FILE_ENDINGS) and \ if t.is_file_ending_valid(t, file):
not any(file.endswith(e) for e in t.INVALID_FILE_ENDINGS):
return t return t
return False return False
...@@ -271,6 +270,19 @@ class TestCase: ...@@ -271,6 +270,19 @@ class TestCase:
def has_valid_file_ending(cls, type: str, file: str): def has_valid_file_ending(cls, type: str, file: str):
return cls._test_case_class_for_file(type, file) != False return cls._test_case_class_for_file(type, file) != False
@staticmethod
def is_file_ending_valid(cls, file: str):
return any(file.endswith(e) for e in cls.FILE_ENDINGS) and \
not any(file.endswith(e) for e in cls.INVALID_FILE_ENDINGS)
@staticmethod
def _get_test_case_file(cls, file: str) -> Optional[str]:
return file
@classmethod
def get_test_case_file(cls, env: Environment, file: str) -> Optional[str]:
return cls._test_case_class_for_file(env.mode, file)._get_test_case_file(cls, file)
class TestResult: class TestResult:
...@@ -373,6 +385,7 @@ class ExtensibleTestResult(TestResult): ...@@ -373,6 +385,7 @@ class ExtensibleTestResult(TestResult):
self.has_succeeded = error_code == 0 self.has_succeeded = error_code == 0
self.error_code = error_code self.error_code = error_code
class TestResultMessage: class TestResultMessage:
def __init__(self, title: str, content: str, multiline: bool, with_line_numbers: bool): def __init__(self, title: str, content: str, multiline: bool, with_line_numbers: bool):
......
...@@ -43,13 +43,14 @@ def _lower_rlimit(res, limit): ...@@ -43,13 +43,14 @@ def _lower_rlimit(res, limit):
class _Execute(object): class _Execute(object):
def __init__(self, cmd, timeout, env, rlimit): def __init__(self, cmd, timeout, env, rlimit, input_str):
self.cmd = cmd self.cmd = cmd
self.timeout = timeout self.timeout = timeout
self.env = env self.env = env
self.proc = None self.proc = None
self.exception = None self.exception = None
self.rlimit = rlimit self.rlimit = rlimit
self.input_str = input_str
MB = 1024 * 1024 MB = 1024 * 1024
if not 'RLIMIT_CORE' in rlimit: if not 'RLIMIT_CORE' in rlimit:
rlimit['RLIMIT_CORE'] = 0 rlimit['RLIMIT_CORE'] = 0
...@@ -76,35 +77,25 @@ class _Execute(object): ...@@ -76,35 +77,25 @@ class _Execute(object):
self.proc = subprocess.Popen(self.cmd, self.proc = subprocess.Popen(self.cmd,
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, stderr=subprocess.PIPE,
stdin=subprocess.PIPE if self.input_str else None,
env=self.env, env=self.env,
shell=True, shell=True,
**prexec_args) **prexec_args)
x = self.proc.communicate() #if self.input_str:
# self.proc.stdin.write(self.input_str.decode)
input_bytes = self.input_str.encode() if self.input_str else None
x = self.proc.communicate(input=input_bytes, timeout=self.timeout if self.timeout > 0.0 else None)
self.out, self.err = x self.out, self.err = x
self.returncode = self.proc.returncode self.returncode = self.proc.returncode
except Exception as e: except subprocess.TimeoutExpired as t:
self.exception = e self.returncode = -signal.SIGXCPU
self.out = bytearray()
self.err = bytearray()
#except Exception as e:
# self.exception = e
def run(self): def run(self):
if self.timeout > 0.0: self._run_process()
_LOG.debug("run with timeout %.1f: %s" % (float(self.timeout), self.cmd))
thread = threading.Thread(target=self._run_process)
thread.start()
thread.join(float(self.timeout))
if self.exception is not None:
raise self.exception
if thread.is_alive():
_LOG.debug("timeout %.1f reached, terminate process" % self.timeout)
if self.proc is not None:
self.proc.terminate()
thread.join(1.0)
if thread.is_alive():
_LOG.debug("termination ignored, now kill process")
self.proc.kill()
thread.join()
raise SigKill(signal.SIGXCPU, "SIGXCPU")
else:
self._run_process()
# Usually python can recognize application terminations triggered by # Usually python can recognize application terminations triggered by
# signals, but somehow it doesn't do this for java (I suspect, that java # signals, but somehow it doesn't do this for java (I suspect, that java
...@@ -117,7 +108,7 @@ class _Execute(object): ...@@ -117,7 +108,7 @@ class _Execute(object):
raise SigKill(-self.returncode, _EXIT_CODES[self.returncode] + ": " + os.strerror(-self.returncode)) raise SigKill(-self.returncode, _EXIT_CODES[self.returncode] + ": " + os.strerror(-self.returncode))
return (self.out, self.err, self.returncode) return (self.out, self.err, self.returncode)
def execute(cmd, env=None, timeout=0, rlimit=None, propagate_sigint=True): def execute(cmd, env=None, timeout=0, rlimit=None, propagate_sigint=True, input_str = None):
"""Execute a command and return stderr and stdout data""" """Execute a command and return stderr and stdout data"""
if not rlimit: if not rlimit:
rlimit = dict() rlimit = dict()
...@@ -127,7 +118,7 @@ def execute(cmd, env=None, timeout=0, rlimit=None, propagate_sigint=True): ...@@ -127,7 +118,7 @@ def execute(cmd, env=None, timeout=0, rlimit=None, propagate_sigint=True):
else: else:
#cmd = shlex.split(cmd[0]) + cmd[1:] #cmd = shlex.split(cmd[0]) + cmd[1:]
cmd = " ".join(shlex.quote(c) for c in cmd) cmd = " ".join(shlex.quote(c) for c in cmd)
exc = _Execute(cmd, timeout, env, rlimit) exc = _Execute(cmd, timeout, env, rlimit, input_str)
(out, err, returncode) = exc.run() (out, err, returncode) = exc.run()
if returncode == -signal.SIGINT: if returncode == -signal.SIGINT:
raise KeyboardInterrupt raise KeyboardInterrupt
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment