Improve infinite loop test case runner

mjtest/test/exec_tests.py

@@ -129,6 +129,7 @@ class JavaExecTest(BasicSyntaxTest):
                 test_result.add_long_text("Output", out.decode())
                 if exp_out.strip() != out.strip():
                     test_result.incorrect_msg = "incorrect output"
+                    test_result.has_succeeded = False
                     test_result.add_diff("Output diff [expected <-> actual]", exp_out, out)
                     test_result.add_file("Source file", self.preprocessed_file)
                 os.chdir(cwd)
@@ -172,31 +173,64 @@ class JavaInfiniteLoopTest(BasicSyntaxTest):
         cwd = os.getcwd()
         os.chdir(tmp_dir)
         timeout = 1
-        err = ""
-        out = ""
-        out2 = ""
+        err = None
+        out = None
+        rtcode = None
+        test_result = ExtensibleTestResult(self)
         try:
-            out2, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java")
-            out, _, _ = self.env.run_command("./" + base_filename, timeout=timeout)
-        except SigKill as kill:
-            os.chdir(cwd)
-            if kill.retcode == signal.SIGXCPU:
+            out, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java")
+            if rtcode != 0:
+                test_result.incorrect_msg = "file can't be compiled"
+                test_result.error_code = rtcode
+                test_result.add_long_text("Error output", err.decode())
+                test_result.add_long_text("Output", out.decode())
+                test_result.add_file("Source file", self.preprocessed_file)
+                os.chdir(cwd)
+                return test_result
+            out, err, rtcode = self.env.run_command("./" + base_filename, timeout=timeout)
+            if rtcode != 0:
+                test_result.incorrect_msg = "file can't be run"
+                test_result.error_code = rtcode
+                test_result.add_long_text("Error output", err.decode())
+                test_result.add_long_text("Output", out.decode())
+                test_result.add_file("Source file", self.preprocessed_file)
+                os.chdir(cwd)
+                return test_result
+        except SigKill as sig:
+            if sig.retcode == signal.SIGXCPU:
+                test_result.add_long_text("Output", out.decode())
                 if self._has_output_file:
                     out = out.decode().strip()
                     exp_out = ""
                     with open(self._output_file, "r") as f:
                         exp_out = f.read().strip()
+                    test_result.add_long_text("Expected output start", exp_out)
                     if not out.startswith(exp_out):
-                        return BasicDiffTestResult(self, 1, out, err.decode(), exp_out,
-                                                   "Output doesn't start like expected")
-                return BasicTestResult(self, 0, err.decode(), out.decode())
+                        test_result.incorrect_msg = "incorrect output start"
+                        test_result.has_succeeded = False
+                    test_result.add_file("Source file", self.preprocessed_file)
+                    os.chdir(cwd)
+                    return test_result
+                else:
+                    test_result.add_file("Source file", self.preprocessed_file)
+                    os.chdir(cwd)
+                    return test_result
             else:
-                return BasicTestResult(self, kill.retcode, "", err.decode(), "No timeout, got " + kill.name)
+                test_result.incorrect_msg = "binary can't be run: " + sig.name
+                test_result.error_code = sig.retcode
+                test_result.add_file("Source file", self.preprocessed_file)
+                os.chdir(cwd)
+                return test_result
         except:
             os.chdir(cwd)
             raise
         os.chdir(cwd)
-        return BasicTestResult(self, 1, "", err.decode(), "No timeout")
+        test_result.incorrect_msg = "run shorter than one second"
+        test_result.add_long_text("Output", out)
+        test_result.has_succeeded = False
+        test_result.add_file("Source file", self.preprocessed_file)
+        os.chdir(cwd)
+        return test_result
 
 
 TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(JavaInfiniteLoopTest)
\ No newline at end of file

mjtest/test/tests.py

@@ -354,6 +354,7 @@ class ExtensibleTestResult(TestResult):
                     texts.append(msg.content)
             else:
                 texts.append("{}: {}".format(msg.title, msg.content))
+        return "\n".join(texts)
 
     def _ident(self, text: Union[str,List[str]]) -> str:
         arr = text if isinstance(text, list) else text.split("\n")