Commit fcaf1778 authored by Johannes Bechberger's avatar Johannes Bechberger

Add tests with input files

And fix test timeouts.
parent 4a6e643d
......@@ -8,7 +8,7 @@ import sys
from datetime import datetime
import time
from threading import Timer
from typing import Dict
from typing import Dict, Optional
import subprocess
......@@ -165,13 +165,13 @@ class Environment:
cmd = [self.mj_run_cmd, mode_flag] + list(args)
return execute(cmd, timeout=timeout or self.timeout)
def run_command(self, cmd: str, *args: Tuple[str], timeout: float = None) -> Tuple[bytes, bytes, int]:
def run_command(self, cmd: str, *args: Tuple[str], timeout: float = None, input_str: Optional[str] = None) -> Tuple[bytes, bytes, int]:
"""
Execute the passend command with its arguments
:return: (out, err, return code)
"""
return execute([cmd] + list(args), timeout=timeout or self.timeout)
return execute([cmd] + list(args), timeout=timeout or self.timeout, input_str=input_str)
def has_to_preprocess(self, file: str) -> bool:
return os.path.relpath(file, self.test_dir).startswith("exec")
......
......@@ -4,6 +4,10 @@ import os
import shutil
import signal
from os import path
import re
from typing import Optional
from mjtest.environment import TestMode, Environment
from mjtest.test.syntax_tests import BasicSyntaxTest
from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult
......@@ -18,12 +22,16 @@ class JavaExecTest(BasicSyntaxTest):
"""
FILE_ENDINGS = [".java", ".mj"]
INVALID_FILE_ENDINGS = [".inf.java", ".inf.mj"]
INVALID_FILE_ENDINGS = [".inf.java", ".inf.mj", ".input.mj", ".input.java"]
OUTPUT_FILE_ENDING = ".out"
MODE = TestMode.compile_firm
INPUT_FILE_REGEXP = r'(\.[0-9]+)\.input(c?)$'
def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str):
super().__init__(env, type, file, preprocessed_file)
self._has_input_file = bool(re.search(self.INPUT_FILE_REGEXP, file))
self._input_file = file if self._has_input_file else None
self._has_character_input = file.endswith("c") if self._has_input_file else False
self._expected_output_file = file + self.OUTPUT_FILE_ENDING
prev_out_dir = path.join(path.dirname(file), ".java_output")
if not path.exists(prev_out_dir):
......@@ -48,10 +56,25 @@ class JavaExecTest(BasicSyntaxTest):
cwd = os.getcwd()
os.chdir(tmp_dir)
exp_out = None
input_str = None
#print(base_filename, get_main_class_name(base_filename + ".java"))
test_result = ExtensibleTestResult(self)
if self._has_input_file:
if self._has_character_input:
with open(self._input_file, "r") as f:
input_str = f.read()
else:
with open(self._input_file, "r") as f:
chars = bytearray('ascii', 'ignore') # type: bytearray
for line in f.readlines():
for part in line.split(" "):
part = part.strip()
if len(part) > 0:
chars.append(int(part))
input_str = chars.decode()
if not self._has_expected_output_file:
_, err, javac_rtcode = \
self.env.run_command("javac", base_filename + ".java", timeout=timeout)
......@@ -68,13 +91,15 @@ class JavaExecTest(BasicSyntaxTest):
_LOG.debug("Can't find a main class, using the file name instead")
main_class = base_filename
exp_out, err, java_rtcode = \
self.env.run_command("java", get_main_class_name(base_filename + ".java"), timeout=timeout)
self.env.run_command("java", get_main_class_name(base_filename + ".java"), timeout=timeout, input_str=input_str)
if javac_rtcode != 0:
test_result.add_long_text("Java output: ", exp_out.decode())
test_result.incorrect_msg = "java runtime error"
test_result.set_error_code(java_rtcode)
test_result.add_long_text("Java error message", err.decode())
test_result.add_file("Source file", self.preprocessed_file)
if self._input_file:
test_result.add_file("Input file", self.file)
os.chdir(cwd)
return test_result
exp_out = exp_out.decode()
......@@ -111,19 +136,23 @@ class JavaExecTest(BasicSyntaxTest):
os.chdir(cwd)
raise
try:
out, err, rtcode = self.env.run_command("./a.out", timeout=timeout)
out, err, rtcode = self.env.run_command("./a.out", timeout=timeout, input_str=input_str)
if rtcode != 0:
test_result.incorrect_msg = "binary can't be run, non zero error code"
test_result.set_error_code(rtcode)
test_result.add_long_text("Error output", err.decode())
test_result.add_long_text("Output", out.decode())
test_result.add_file("Source file", self.preprocessed_file)
if self._input_file:
test_result.add_file("Input file", self.file)
os.chdir(cwd)
return test_result
except SigKill as sig:
test_result.incorrect_msg = "binary can't be run: " + sig.name.strip()
test_result.set_error_code(sig.retcode)
test_result.add_file("Source file", self.preprocessed_file)
if self._input_file:
test_result.add_file("Input file", self.file)
os.chdir(cwd)
return test_result
except:
......@@ -140,6 +169,8 @@ class JavaExecTest(BasicSyntaxTest):
test_result.add_long_text("Actual output", out)
test_result.add_diff("Output diff [expected <-> actual]", exp_out, out, with_line_numbers=True)
test_result.add_file("Source file", self.preprocessed_file)
if self._input_file:
test_result.add_file("Input file", self.file)
os.chdir(cwd)
return test_result
return BasicTestResult(self, rtcode, out, err.decode())
......@@ -159,10 +190,34 @@ class JavaExecTest(BasicSyntaxTest):
_LOG.exception("Decoding a hash sum for java output caching failed")
return self._hash_sum_for_file(file) == old_hash
@classmethod
def _hash_sum_for_file(self, file: str) -> str:
with open(file, "r") as f:
return hashlib.sha256(f.read().encode()).hexdigest()
@classmethod
def _test_case_file_for_input_file(cls, input_file: str) -> Optional[str]:
base = re.sub(cls.INPUT_FILE_REGEXP, "", input_file)
if path.exists(base + ".input.mj"):
return base + ".input.mj"
if path.exists(base + ".input.java"):
return base + ".input.java"
return None
@staticmethod
def is_file_ending_valid(cls, file: str) -> bool:
if re.search(cls.INPUT_FILE_REGEXP, file) and ".java_output" not in file and "precprocessor" not in file:
if not JavaExecTest._test_case_file_for_input_file(file):
_LOG.error("Skip {} that hasn't a corresponding test case file".format(file))
return True
return super().is_file_ending_valid(cls, file)
@staticmethod
def _get_test_case_file(cls, file: str) -> Optional[str]:
if re.search(JavaExecTest.INPUT_FILE_REGEXP, file):
return JavaExecTest._test_case_file_for_input_file(file)
return file
TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(JavaExecTest)
......
......@@ -78,7 +78,7 @@ class TestSuite:
if not p.startswith("."):
file_names.append(p)
for file in sorted(file_names):
if not TestCase.has_valid_file_ending(self.env.mode, file):
if not TestCase.has_valid_file_ending(self.env.mode, join(dir, file)):
_LOG.debug("Skip file " + file)
elif self.env.only_incorrect_tests and file in correct_test_cases:
_LOG.info("Skip file {} as its test case was executed correctly the last run")
......@@ -87,7 +87,7 @@ class TestSuite:
if self.env.has_to_preprocess(file_path) and self.env.is_lib_file(file_path):
_LOG.debug("Skip lib file '{}'".format(file))
continue
preprocessed = self.env.preprocess(join(dir, file))
preprocessed = self.env.preprocess(join(dir, TestCase.get_test_case_file(self.env, join(dir, file))))
test_case = TestCase.create_from_file(self.env, m, join(dir, file), preprocessed)
if not test_case:
pass
......@@ -261,8 +261,7 @@ class TestCase:
@classmethod
def _test_case_class_for_file(cls, type: str, file: str):
for t in cls.TEST_CASE_CLASSES[type]:
if any(file.endswith(e) for e in t.FILE_ENDINGS) and \
not any(file.endswith(e) for e in t.INVALID_FILE_ENDINGS):
if t.is_file_ending_valid(t, file):
return t
return False
......@@ -271,6 +270,19 @@ class TestCase:
def has_valid_file_ending(cls, type: str, file: str):
return cls._test_case_class_for_file(type, file) != False
@staticmethod
def is_file_ending_valid(cls, file: str):
return any(file.endswith(e) for e in cls.FILE_ENDINGS) and \
not any(file.endswith(e) for e in cls.INVALID_FILE_ENDINGS)
@staticmethod
def _get_test_case_file(cls, file: str) -> Optional[str]:
return file
@classmethod
def get_test_case_file(cls, env: Environment, file: str) -> Optional[str]:
return cls._test_case_class_for_file(env.mode, file)._get_test_case_file(cls, file)
class TestResult:
......@@ -373,6 +385,7 @@ class ExtensibleTestResult(TestResult):
self.has_succeeded = error_code == 0
self.error_code = error_code
class TestResultMessage:
def __init__(self, title: str, content: str, multiline: bool, with_line_numbers: bool):
......
......@@ -43,13 +43,14 @@ def _lower_rlimit(res, limit):
class _Execute(object):
def __init__(self, cmd, timeout, env, rlimit):
def __init__(self, cmd, timeout, env, rlimit, input_str):
self.cmd = cmd
self.timeout = timeout
self.env = env
self.proc = None
self.exception = None
self.rlimit = rlimit
self.input_str = input_str
MB = 1024 * 1024
if not 'RLIMIT_CORE' in rlimit:
rlimit['RLIMIT_CORE'] = 0
......@@ -76,35 +77,25 @@ class _Execute(object):
self.proc = subprocess.Popen(self.cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE if self.input_str else None,
env=self.env,
shell=True,
**prexec_args)
x = self.proc.communicate()
#if self.input_str:
# self.proc.stdin.write(self.input_str.decode)
input_bytes = self.input_str.encode() if self.input_str else None
x = self.proc.communicate(input=input_bytes, timeout=self.timeout if self.timeout > 0.0 else None)
self.out, self.err = x
self.returncode = self.proc.returncode
except Exception as e:
self.exception = e
except subprocess.TimeoutExpired as t:
self.returncode = -signal.SIGXCPU
self.out = bytearray()
self.err = bytearray()
#except Exception as e:
# self.exception = e
def run(self):
if self.timeout > 0.0:
_LOG.debug("run with timeout %.1f: %s" % (float(self.timeout), self.cmd))
thread = threading.Thread(target=self._run_process)
thread.start()
thread.join(float(self.timeout))
if self.exception is not None:
raise self.exception
if thread.is_alive():
_LOG.debug("timeout %.1f reached, terminate process" % self.timeout)
if self.proc is not None:
self.proc.terminate()
thread.join(1.0)
if thread.is_alive():
_LOG.debug("termination ignored, now kill process")
self.proc.kill()
thread.join()
raise SigKill(signal.SIGXCPU, "SIGXCPU")
else:
self._run_process()
self._run_process()
# Usually python can recognize application terminations triggered by
# signals, but somehow it doesn't do this for java (I suspect, that java
......@@ -117,7 +108,7 @@ class _Execute(object):
raise SigKill(-self.returncode, _EXIT_CODES[self.returncode] + ": " + os.strerror(-self.returncode))
return (self.out, self.err, self.returncode)
def execute(cmd, env=None, timeout=0, rlimit=None, propagate_sigint=True):
def execute(cmd, env=None, timeout=0, rlimit=None, propagate_sigint=True, input_str = None):
"""Execute a command and return stderr and stdout data"""
if not rlimit:
rlimit = dict()
......@@ -127,7 +118,7 @@ def execute(cmd, env=None, timeout=0, rlimit=None, propagate_sigint=True):
else:
#cmd = shlex.split(cmd[0]) + cmd[1:]
cmd = " ".join(shlex.quote(c) for c in cmd)
exc = _Execute(cmd, timeout, env, rlimit)
exc = _Execute(cmd, timeout, env, rlimit, input_str)
(out, err, returncode) = exc.run()
if returncode == -signal.SIGINT:
raise KeyboardInterrupt
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment