Commit caa946ba authored by Johannes Bechberger's avatar Johannes Bechberger

Improve README and add invalid exec test case handler

parent 7d3a9a85
......@@ -113,9 +113,23 @@ Test types for the compile-firm and compile mode
if a <code>[test file].out</code> exists: the content of this file<br/>
if no such file exists: the same as the execution of the file with <code>java</code></td>
</tr>
<tr>
<td><code>.[number].inputc</code></td>
<td>This a test case with an input. The input is interpreted as ASCII chars. It requires a corresponding
<code>.input.java</code> or <code>.input.mj</code> file that is executed with this input.
The rest is equivalent to normal test cases from the row above</td>
</tr>
<tr>
<td><code>.[number].input</code></td>
<td>This is an input test case whichs input file contains space seperated decimal numbers that are used as input data.</td>
</tr>
<tr>
<td><code>.invalid.mj</code> <code>.invalid.java</code></td>
<td>The test file should compile. The execution of the resulting binary should abort width a return code <code>&gt; 0</code></td>
</tr>
</table>
__All compile-firm and compile mode tests have to be semantically correct___
__All compile-firm and compile mode tests have to be semantically correct__
__All bigger test cases (e.g. for benchmarking) should be placed into the `big` folder__
......
......@@ -24,7 +24,7 @@ class JavaExecTest(BasicSyntaxTest):
"""
FILE_ENDINGS = [".java", ".mj"]
INVALID_FILE_ENDINGS = [".inf.java", ".inf.mj", ".input.mj", ".input.java"]
INVALID_FILE_ENDINGS = [".inf.java", ".inf.mj", ".input.mj", ".input.java", ".invalid.java", ".invalid.mj"]
OUTPUT_FILE_ENDING = ".out"
MODE = TestMode.compile_firm
INPUT_FILE_REGEXP = r'(\.[0-9]+)\.input(c?)$'
......@@ -289,7 +289,7 @@ class JavaInfiniteLoopTest(BasicSyntaxTest):
os.chdir(cwd)
return test_result
else:
test_result.incorrect_msg = "sdfbinary can't be run: " + sig.name
test_result.incorrect_msg = "binary can't be run: " + sig.name
test_result.set_error_code(sig.retcode)
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
......@@ -308,3 +308,86 @@ class JavaInfiniteLoopTest(BasicSyntaxTest):
TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(JavaInfiniteLoopTest)
class InvalidJavaExecTest(BasicSyntaxTest):
"""
The compiled binary should fail to execute properly
"""
FILE_ENDINGS = [".invalid.java", ".invalid.mj"]
MODE = TestMode.compile_firm
def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str):
super().__init__(env, type, file, preprocessed_file, log_file_mode)
self._should_succeed = False
def run(self) -> BasicDiffTestResult:
is_big_testcase = "big" in self.file
timeout = self.env.big_timeout if is_big_testcase else self.env.timeout
base_filename = path.basename(self.file).split(".")[0]
tmp_dir = self.env.create_pid_local_tmpdir()
shutil.copy(self.preprocessed_file, path.join(tmp_dir, base_filename + ".java"))
cwd = os.getcwd()
os.chdir(tmp_dir)
#print(base_filename, get_main_class_name(base_filename + ".java"))
test_result = ExtensibleTestResult(self)
test_result.require_error_string_in_error_case = False
try:
out, err, rtcode = None, None, None
try:
out, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java", timeout=-1)
if rtcode != 0:
test_result.incorrect_msg = "file can't be compiled"
test_result.set_error_code(-rtcode)
test_result.add_long_text("Error output", err.decode())
test_result.add_long_text("Output", out.decode())
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
except SigKill as sig:
test_result.incorrect_msg = "file can't be compiled: " + sig.name
test_result.set_error_code(sig.retcode)
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
except:
os.chdir(cwd)
raise
try:
out, err, rtcode = self.env.run_command("./a.out", timeout=timeout)
if rtcode != 0:
test_result.incorrect_msg = "binary can't be run, non zero error code"
test_result.set_error_code(rtcode)
test_result.add_long_text("Error output", err.decode())
test_result.add_long_text("Output", out.decode())
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
except SigKill as sig:
test_result.incorrect_msg = "binary can't be run: " + sig.name.strip()
test_result.set_error_code(sig.retcode)
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
except:
os.chdir(cwd)
raise
test_result = BasicTestResult(self, rtcode, out.decode(), err.decode())
test_result.require_error_string_in_error_case = False
return test_result
except SigKill as sig:
os.chdir(cwd)
assert False
except BaseException:
os.chdir(cwd)
raise
TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(InvalidJavaExecTest)
class InvalidJavaCompileExecTest(InvalidJavaExecTest):
MODE = TestMode.compile
TestCase.TEST_CASE_CLASSES[TestMode.compile].append(InvalidJavaExecTest)
......@@ -439,6 +439,7 @@ class BasicTestResult(TestResult):
if error_output:
self.add_additional_text("Error output", error_output)
self.has_succeeded = error_code == 0
self.require_error_string_in_error_case = True
def succeeded(self):
return self.has_succeeded
......@@ -447,13 +448,14 @@ class BasicTestResult(TestResult):
if self.succeeded():
return super().is_correct()
else:
return super().is_correct() and self._contains_error_str
return super().is_correct() and (self._contains_error_str or not self.require_error_string_in_error_case)
def short_message(self) -> str:
if self.is_correct():
return "correct"
else:
if not self.succeeded() and not self.test_case.should_succeed() and not self._contains_error_str:
if not self.succeeded() and not self.test_case.should_succeed() and not self._contains_error_str and \
not self.require_error_string_in_error_case:
return "the error output doesn't contain the word \"error\""
return self._incorrect_msg
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment