Commit e843d83b authored by Johannes Bechberger's avatar Johannes Bechberger

Improve infinite loop test case runner

parent 0706a7a6
......@@ -129,6 +129,7 @@ class JavaExecTest(BasicSyntaxTest):
test_result.add_long_text("Output", out.decode())
if exp_out.strip() != out.strip():
test_result.incorrect_msg = "incorrect output"
test_result.has_succeeded = False
test_result.add_diff("Output diff [expected <-> actual]", exp_out, out)
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
......@@ -172,31 +173,64 @@ class JavaInfiniteLoopTest(BasicSyntaxTest):
cwd = os.getcwd()
os.chdir(tmp_dir)
timeout = 1
err = ""
out = ""
out2 = ""
err = None
out = None
rtcode = None
test_result = ExtensibleTestResult(self)
try:
out2, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java")
out, _, _ = self.env.run_command("./" + base_filename, timeout=timeout)
except SigKill as kill:
os.chdir(cwd)
if kill.retcode == signal.SIGXCPU:
out, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java")
if rtcode != 0:
test_result.incorrect_msg = "file can't be compiled"
test_result.error_code = rtcode
test_result.add_long_text("Error output", err.decode())
test_result.add_long_text("Output", out.decode())
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
out, err, rtcode = self.env.run_command("./" + base_filename, timeout=timeout)
if rtcode != 0:
test_result.incorrect_msg = "file can't be run"
test_result.error_code = rtcode
test_result.add_long_text("Error output", err.decode())
test_result.add_long_text("Output", out.decode())
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
except SigKill as sig:
if sig.retcode == signal.SIGXCPU:
test_result.add_long_text("Output", out.decode())
if self._has_output_file:
out = out.decode().strip()
exp_out = ""
with open(self._output_file, "r") as f:
exp_out = f.read().strip()
test_result.add_long_text("Expected output start", exp_out)
if not out.startswith(exp_out):
return BasicDiffTestResult(self, 1, out, err.decode(), exp_out,
"Output doesn't start like expected")
return BasicTestResult(self, 0, err.decode(), out.decode())
test_result.incorrect_msg = "incorrect output start"
test_result.has_succeeded = False
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
else:
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
else:
return BasicTestResult(self, kill.retcode, "", err.decode(), "No timeout, got " + kill.name)
test_result.incorrect_msg = "binary can't be run: " + sig.name
test_result.error_code = sig.retcode
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
except:
os.chdir(cwd)
raise
os.chdir(cwd)
return BasicTestResult(self, 1, "", err.decode(), "No timeout")
test_result.incorrect_msg = "run shorter than one second"
test_result.add_long_text("Output", out)
test_result.has_succeeded = False
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(JavaInfiniteLoopTest)
\ No newline at end of file
......@@ -354,6 +354,7 @@ class ExtensibleTestResult(TestResult):
texts.append(msg.content)
else:
texts.append("{}: {}".format(msg.title, msg.content))
return "\n".join(texts)
def _ident(self, text: Union[str,List[str]]) -> str:
arr = text if isinstance(text, list) else text.split("\n")
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment