Commit 89dbd74f authored by Johannes Bechberger's avatar Johannes Bechberger

Fix some bugs and remove the infinite loop test cases

The infinite loop test cases are hard to maintain and
are therefore removed.
parent fcaf1778
...@@ -112,11 +112,6 @@ Test types for the compile-firm mode ...@@ -112,11 +112,6 @@ Test types for the compile-firm mode
if a <code>[test file].out</code> exists: the content of this file<br/> if a <code>[test file].out</code> exists: the content of this file<br/>
if no such file exists: the same as the execution of the file with <code>java</code></td> if no such file exists: the same as the execution of the file with <code>java</code></td>
</tr> </tr>
<tr>
<td><code>.inf.java</code> <code>.inf.mj</code></td>
<td>Tests for infinite loops. The test file should compile. The binary should run at least 1 second.
The output should start with the content of the <code>[test file].out</code> file, if such file exists.</td>
</tr>
</table> </table>
__All compile-firm mode tests have to be semantically correct___ __All compile-firm mode tests have to be semantically correct___
......
...@@ -15,8 +15,8 @@ class ASTPrettyPrintTest(BasicSyntaxTest): ...@@ -15,8 +15,8 @@ class ASTPrettyPrintTest(BasicSyntaxTest):
FILE_ENDINGS = [".mj", ".valid.mj"] FILE_ENDINGS = [".mj", ".valid.mj"]
INVALID_FILE_ENDINGS = [".invalid.mj"] INVALID_FILE_ENDINGS = [".invalid.mj"]
def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str): def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str):
super().__init__(env, type, file, preprocessed_file) super().__init__(env, type, file, preprocessed_file, log_file_mode)
if type != TestMode.ast and TEST_MODES.index(TestMode.ast) < TEST_MODES.index(type): if type != TestMode.ast and TEST_MODES.index(TestMode.ast) < TEST_MODES.index(type):
self._should_succeed = True self._should_succeed = True
......
...@@ -8,6 +8,8 @@ from os import path ...@@ -8,6 +8,8 @@ from os import path
import re import re
from typing import Optional from typing import Optional
import errno
from mjtest.environment import TestMode, Environment from mjtest.environment import TestMode, Environment
from mjtest.test.syntax_tests import BasicSyntaxTest from mjtest.test.syntax_tests import BasicSyntaxTest
from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult
...@@ -27,8 +29,8 @@ class JavaExecTest(BasicSyntaxTest): ...@@ -27,8 +29,8 @@ class JavaExecTest(BasicSyntaxTest):
MODE = TestMode.compile_firm MODE = TestMode.compile_firm
INPUT_FILE_REGEXP = r'(\.[0-9]+)\.input(c?)$' INPUT_FILE_REGEXP = r'(\.[0-9]+)\.input(c?)$'
def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str): def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str):
super().__init__(env, type, file, preprocessed_file) super().__init__(env, type, file, preprocessed_file, log_file_mode)
self._has_input_file = bool(re.search(self.INPUT_FILE_REGEXP, file)) self._has_input_file = bool(re.search(self.INPUT_FILE_REGEXP, file))
self._input_file = file if self._has_input_file else None self._input_file = file if self._has_input_file else None
self._has_character_input = file.endswith("c") if self._has_input_file else False self._has_character_input = file.endswith("c") if self._has_input_file else False
...@@ -227,8 +229,8 @@ class JavaInfiniteLoopTest(BasicSyntaxTest): ...@@ -227,8 +229,8 @@ class JavaInfiniteLoopTest(BasicSyntaxTest):
OUTPUT_FILE_ENDING = ".out" OUTPUT_FILE_ENDING = ".out"
MODE = TestMode.compile_firm MODE = TestMode.compile_firm
def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str): def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str):
super().__init__(env, type, file, preprocessed_file) super().__init__(env, type, file, preprocessed_file, log_file_mode)
self._output_file = self.file + self.OUTPUT_FILE_ENDING self._output_file = self.file + self.OUTPUT_FILE_ENDING
self._has_output_file = path.exists(self._output_file) self._has_output_file = path.exists(self._output_file)
...@@ -283,13 +285,14 @@ class JavaInfiniteLoopTest(BasicSyntaxTest): ...@@ -283,13 +285,14 @@ class JavaInfiniteLoopTest(BasicSyntaxTest):
os.chdir(cwd) os.chdir(cwd)
return test_result return test_result
else: else:
test_result.incorrect_msg = "binary can't be run: " + sig.name test_result.incorrect_msg = "sdfbinary can't be run: " + sig.name
test_result.set_error_code(sig.retcode) test_result.set_error_code(sig.retcode)
test_result.add_file("Source file", self.preprocessed_file) test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd) os.chdir(cwd)
return test_result return test_result
except: except BaseException as e:
os.chdir(cwd) os.chdir(cwd)
print(e)
raise raise
os.chdir(cwd) os.chdir(cwd)
test_result.incorrect_msg = "run shorter than one second" test_result.incorrect_msg = "run shorter than one second"
......
...@@ -9,8 +9,8 @@ class BasicSyntaxTest(TestCase): ...@@ -9,8 +9,8 @@ class BasicSyntaxTest(TestCase):
FILE_ENDINGS = [".invalid.mj", ".valid.mj", ".mj", ".invalid.java", ".java"] FILE_ENDINGS = [".invalid.mj", ".valid.mj", ".mj", ".invalid.java", ".java"]
MODE = TestMode.syntax MODE = TestMode.syntax
def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str): def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str):
super().__init__(env, type, file, preprocessed_file) super().__init__(env, type, file, preprocessed_file, log_file_mode)
if type != self.MODE and TEST_MODES.index(type) > TEST_MODES.index(self.MODE): if type != self.MODE and TEST_MODES.index(type) > TEST_MODES.index(self.MODE):
self._should_succeed = True self._should_succeed = True
else: else:
...@@ -48,8 +48,8 @@ class JavaCompileTest(BasicSyntaxTest): ...@@ -48,8 +48,8 @@ class JavaCompileTest(BasicSyntaxTest):
FILE_ENDINGS = [".java"] FILE_ENDINGS = [".java"]
SYNTAX_TEST = True SYNTAX_TEST = True
def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str): def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str):
super().__init__(env, type, file, preprocessed_file) super().__init__(env, type, file, preprocessed_file, log_file_mode)
tmp_dir = self.env.create_pid_local_tmpdir() tmp_dir = self.env.create_pid_local_tmpdir()
_, self.javac_err, self.javac_rtcode = \ _, self.javac_err, self.javac_rtcode = \
self.env.run_command("javac", path.relpath(preprocessed_file), "-d", tmp_dir, "-verbose") self.env.run_command("javac", path.relpath(preprocessed_file), "-d", tmp_dir, "-verbose")
......
...@@ -65,13 +65,15 @@ class TestSuite: ...@@ -65,13 +65,15 @@ class TestSuite:
for root, dirs, files in os.walk(dir): for root, dirs, files in os.walk(dir):
base = os.path.relpath(root, dir) base = os.path.relpath(root, dir)
dirname = os.path.dirname(base) dirname = os.path.dirname(base)
if root.startswith("."):
continue
if dir == root: if dir == root:
file_names.extend(files) file_names.extend(files)
elif ".preprocessed" in base or (dirname.startswith(".") and dirname != "."): elif ".preprocessed" in base or (dirname.startswith(".") and dirname != "."):
continue continue
elif base.startswith("big") and get_test_dirname(mode) == "exec" and self.env.only_small_exec_tests: elif base.startswith("big") and get_test_dirname(mode) == "exec" and self.env.only_small_exec_tests:
continue continue
elif base == ".": elif base.startswith("."):
continue continue
for file in files: for file in files:
p = join(base, file) p = join(base, file)
...@@ -88,23 +90,36 @@ class TestSuite: ...@@ -88,23 +90,36 @@ class TestSuite:
_LOG.debug("Skip lib file '{}'".format(file)) _LOG.debug("Skip lib file '{}'".format(file))
continue continue
preprocessed = self.env.preprocess(join(dir, TestCase.get_test_case_file(self.env, join(dir, file)))) preprocessed = self.env.preprocess(join(dir, TestCase.get_test_case_file(self.env, join(dir, file))))
test_case = TestCase.create_from_file(self.env, m, join(dir, file), preprocessed) test_case = TestCase.create_from_file(self.env, m, join(dir, file), preprocessed, mode)
if not test_case: if not test_case:
pass pass
elif not test_case.can_run(): elif not test_case.can_run():
_LOG.debug("Skip test case '{}' because it isn't suited".format(test_case.name())) _LOG.debug("Skip test case '{}' because it isn't suited".format(test_case.name()))
else: else:
if m not in self.test_cases: if mode not in self.test_cases:
self.test_cases[m] = [] self.test_cases[mode] = []
self.test_cases[m].append(test_case) self.test_cases[mode].append(test_case)
if m in self.test_cases and len(self.test_cases[m]) == 0: if mode in self.test_cases and len(self.test_cases[mode]) == 0:
del self.test_cases[m] del self.test_cases[mode]
def _log_file_for_type(self, type: str): def _log_file_for_type(self, type: str):
return join(self.env.test_dir, get_test_dirname(type), ".mjtest_correct_testcases_" + self.env.mode) return join(self.env.test_dir, get_test_dirname(type), ".mjtest_correct_testcases_" + self.env.mode)
def _add_correct_test_case(self, test_case: 'TestCase'): def _add_correct_test_case(self, test_case: 'TestCase'):
self.correct_test_cases[test_case.type].add(basename(test_case.file)) test_dir = None
modes = [test_case.MODE]
if test_case.MODE in TestMode.USE_TESTS_OF_OTHER:
modes += TestMode.USE_TESTS_OF_OTHER[test_case.MODE]
for mode in modes:
subdir = os.path.join(self.env.test_dir, get_test_dirname(mode))
if test_case.file.startswith(subdir):
test_dir = subdir
break
basefile = None
if test_dir is None:
basefile = os.path.relpath(test_case.file, self.env.test_dir)
basefile = os.path.relpath(test_case.file, test_dir)
self.correct_test_cases[test_case.log_file_mode].add(basefile)
def run(self) -> RunResult: def run(self) -> RunResult:
ret = RunResult(0, 0) ret = RunResult(0, 0)
...@@ -223,11 +238,12 @@ class TestCase: ...@@ -223,11 +238,12 @@ class TestCase:
FILE_ENDINGS = [] FILE_ENDINGS = []
INVALID_FILE_ENDINGS = [] INVALID_FILE_ENDINGS = []
def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str): def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str):
self.env = env self.env = env
self.type = type self.type = type
self.file = file self.file = file
self.preprocessed_file = preprocessed_file self.preprocessed_file = preprocessed_file
self.log_file_mode = log_file_mode
def should_succeed(self) -> bool: def should_succeed(self) -> bool:
raise NotImplementedError() raise NotImplementedError()
...@@ -247,9 +263,9 @@ class TestCase: ...@@ -247,9 +263,9 @@ class TestCase:
raise NotImplementedError() raise NotImplementedError()
@classmethod @classmethod
def create_from_file(cls, env: Environment, mode: str, file: str, preprocessed_file: str) -> Optional['TestCase']: def create_from_file(cls, env: Environment, mode: str, file: str, preprocessed_file: str, log_file_mode: str) -> Optional['TestCase']:
if cls.has_valid_file_ending(env.mode, file): if cls.has_valid_file_ending(env.mode, file):
return cls._test_case_class_for_file(env.mode, file)(env, mode, file, preprocessed_file) return cls._test_case_class_for_file(env.mode, file)(env, mode, file, preprocessed_file, log_file_mode)
return None return None
def name(self): def name(self):
......
...@@ -7,6 +7,9 @@ Alternative to subprocess and os.system ...@@ -7,6 +7,9 @@ Alternative to subprocess and os.system
import os import os
import shlex import shlex
import subprocess import subprocess
import errno
has_resource_module = True has_resource_module = True
try: try:
import resource import resource
...@@ -88,9 +91,8 @@ class _Execute(object): ...@@ -88,9 +91,8 @@ class _Execute(object):
self.out, self.err = x self.out, self.err = x
self.returncode = self.proc.returncode self.returncode = self.proc.returncode
except subprocess.TimeoutExpired as t: except subprocess.TimeoutExpired as t:
self.returncode = -signal.SIGXCPU self.returncode = -errno.ETIMEDOUT
self.out = bytearray() raise SigKill(signal.SIGXCPU, "SIGXCPU")
self.err = bytearray()
#except Exception as e: #except Exception as e:
# self.exception = e # self.exception = e
...@@ -103,7 +105,7 @@ class _Execute(object): ...@@ -103,7 +105,7 @@ class _Execute(object):
# should. Calculate to python convention returncode # should. Calculate to python convention returncode
if self.returncode > 127: if self.returncode > 127:
self.returncode = 128 - self.returncode self.returncode = 128 - self.returncode
if self.returncode in _EXIT_CODES: if self.returncode in _EXIT_CODES or self.returncode == -errno.ETIMEDOUT:
_LOG.debug("returncode %d recognized as '%s'" % (self.returncode, _EXIT_CODES[self.returncode])) _LOG.debug("returncode %d recognized as '%s'" % (self.returncode, _EXIT_CODES[self.returncode]))
raise SigKill(-self.returncode, _EXIT_CODES[self.returncode] + ": " + os.strerror(-self.returncode)) raise SigKill(-self.returncode, _EXIT_CODES[self.returncode] + ": " + os.strerror(-self.returncode))
return (self.out, self.err, self.returncode) return (self.out, self.err, self.returncode)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment