Improve README and add invalid exec test case handler

README.mdwn

@@ -113,9 +113,23 @@ Test types for the compile-firm and compile mode
     if a <code>[test file].out</code> exists: the content of this file<br/>
     if no such file exists: the same as the execution of the file with <code>java</code></td>
 </tr>
+<tr>
+    <td><code>.[number].inputc</code></td>
+    <td>This a test case with an input. The input is interpreted as ASCII chars. It requires a corresponding 
+    <code>.input.java</code> or <code>.input.mj</code> file that is executed with this input.
+    The rest is equivalent to normal test cases from the row above</td>
+</tr>
+<tr>
+    <td><code>.[number].input</code></td>
+    <td>This is an input test case whichs input file contains space seperated decimal numbers that are used as input data.</td>
+</tr>
+<tr>
+    <td><code>.invalid.mj</code> <code>.invalid.java</code></td>
+    <td>The test file should compile. The execution of the resulting binary should abort width a return code <code>&gt; 0</code></td>
+</tr>
 </table>
 
-__All compile-firm and compile mode tests have to be semantically correct___
+__All compile-firm and compile mode tests have to be semantically correct__
 
 __All bigger test cases (e.g. for benchmarking) should be placed into the `big` folder__
 

mjtest/test/exec_tests.py

@@ -24,7 +24,7 @@ class JavaExecTest(BasicSyntaxTest):
     """
 
     FILE_ENDINGS = [".java", ".mj"]
-    INVALID_FILE_ENDINGS = [".inf.java", ".inf.mj", ".input.mj", ".input.java"]
+    INVALID_FILE_ENDINGS = [".inf.java", ".inf.mj", ".input.mj", ".input.java", ".invalid.java", ".invalid.mj"]
     OUTPUT_FILE_ENDING = ".out"
     MODE = TestMode.compile_firm
     INPUT_FILE_REGEXP = r'(\.[0-9]+)\.input(c?)$'
@@ -289,7 +289,7 @@ class JavaInfiniteLoopTest(BasicSyntaxTest):
                     os.chdir(cwd)
                     return test_result
             else:
-                test_result.incorrect_msg = "sdfbinary can't be run: " + sig.name
+                test_result.incorrect_msg = "binary can't be run: " + sig.name
                 test_result.set_error_code(sig.retcode)
                 test_result.add_file("Source file", self.preprocessed_file)
                 os.chdir(cwd)
@@ -308,3 +308,86 @@ class JavaInfiniteLoopTest(BasicSyntaxTest):
 
 
 TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(JavaInfiniteLoopTest)
+
+
+class InvalidJavaExecTest(BasicSyntaxTest):
+    """
+    The compiled binary should fail to execute properly
+    """
+
+    FILE_ENDINGS = [".invalid.java", ".invalid.mj"]
+    MODE = TestMode.compile_firm
+
+    def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str):
+        super().__init__(env, type, file, preprocessed_file, log_file_mode)
+        self._should_succeed = False
+
+    def run(self) -> BasicDiffTestResult:
+        is_big_testcase = "big" in self.file
+        timeout = self.env.big_timeout if is_big_testcase else self.env.timeout
+        base_filename = path.basename(self.file).split(".")[0]
+        tmp_dir = self.env.create_pid_local_tmpdir()
+        shutil.copy(self.preprocessed_file, path.join(tmp_dir, base_filename + ".java"))
+        cwd = os.getcwd()
+        os.chdir(tmp_dir)
+        #print(base_filename, get_main_class_name(base_filename + ".java"))
+
+        test_result = ExtensibleTestResult(self)
+        test_result.require_error_string_in_error_case = False
+        try:
+            out, err, rtcode = None, None, None
+            try:
+                out, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java", timeout=-1)
+                if rtcode != 0:
+                    test_result.incorrect_msg = "file can't be compiled"
+                    test_result.set_error_code(-rtcode)
+                    test_result.add_long_text("Error output", err.decode())
+                    test_result.add_long_text("Output", out.decode())
+                    test_result.add_file("Source file", self.preprocessed_file)
+                    os.chdir(cwd)
+                    return test_result
+            except SigKill as sig:
+                test_result.incorrect_msg = "file can't be compiled: " + sig.name
+                test_result.set_error_code(sig.retcode)
+                test_result.add_file("Source file", self.preprocessed_file)
+                os.chdir(cwd)
+                return test_result
+            except:
+                os.chdir(cwd)
+                raise
+            try:
+                out, err, rtcode = self.env.run_command("./a.out", timeout=timeout)
+                if rtcode != 0:
+                    test_result.incorrect_msg = "binary can't be run, non zero error code"
+                    test_result.set_error_code(rtcode)
+                    test_result.add_long_text("Error output", err.decode())
+                    test_result.add_long_text("Output", out.decode())
+                    test_result.add_file("Source file", self.preprocessed_file)
+                    os.chdir(cwd)
+                    return test_result
+            except SigKill as sig:
+                test_result.incorrect_msg = "binary can't be run: " + sig.name.strip()
+                test_result.set_error_code(sig.retcode)
+                test_result.add_file("Source file", self.preprocessed_file)
+                os.chdir(cwd)
+                return test_result
+            except:
+                os.chdir(cwd)
+                raise
+            test_result = BasicTestResult(self, rtcode, out.decode(), err.decode())
+            test_result.require_error_string_in_error_case = False
+            return test_result
+        except SigKill as sig:
+            os.chdir(cwd)
+            assert False
+        except BaseException:
+            os.chdir(cwd)
+            raise
+
+
+TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(InvalidJavaExecTest)
+
+class InvalidJavaCompileExecTest(InvalidJavaExecTest):
+    MODE = TestMode.compile
+
+TestCase.TEST_CASE_CLASSES[TestMode.compile].append(InvalidJavaExecTest)

mjtest/test/tests.py

@@ -439,6 +439,7 @@ class BasicTestResult(TestResult):
         if error_output:
             self.add_additional_text("Error output", error_output)
         self.has_succeeded = error_code == 0
+        self.require_error_string_in_error_case = True
 
     def succeeded(self):
         return self.has_succeeded
@@ -447,13 +448,14 @@ class BasicTestResult(TestResult):
         if self.succeeded():
             return super().is_correct()
         else:
-            return super().is_correct() and self._contains_error_str
+            return super().is_correct() and (self._contains_error_str or not self.require_error_string_in_error_case)
 
     def short_message(self) -> str:
         if self.is_correct():
             return "correct"
         else:
-            if not self.succeeded() and not self.test_case.should_succeed() and not self._contains_error_str:
+            if not self.succeeded() and not self.test_case.should_succeed() and not self._contains_error_str and \
+               not self.require_error_string_in_error_case:
                 return "the error output doesn't contain the word \"error\""
             return self._incorrect_msg