Add `--compile-(firm)-only` to compile successful semantic tests

Fix #2 and #1

README.md

@@ -23,7 +23,9 @@ The test cases are divided in 6 'modes':
    them.
 - __ast__: Test cases that check the generated ast by using the pretty printing functionality.
 - __semantic__: Test cases that check semantic checking of MiniJava programs
+- __compile-firm-only__: Test cases that check the compilation of MiniJava programs with the libfirm backend.
 - __compile-firm__: Test cases that check the correct compilation and execution of MiniJava programs with the libfirm backend.
+- __compile-only__: Test cases that check the compilation MiniJava programs with the self implemented backend.
 - __compile__: Test cases that check the correct compilation and execution of MiniJava programs with the self implemented backend.
 
 The test different test cases for each mode are located in a folder with the same name. 
@@ -103,6 +105,19 @@ Test types for the semantic mode
 
 __All semantic mode tests have to be syntactically correct__
 
+Test types for the compile-firm-only and compile-only mode
+----------------------------------------------------------
+
+<table>
+<tr><th>File ending(s) of test cases</th><th>Expected behaviour to complete a test of this type</th></tr>
+<tr>
+    <td><code>.valid.mj</code> <code>.mj</code> <code>.valid.java</code> <code>.java</code></td>
+    <td>Return code is <code>0</code> and the compiler produces an `a.out` file, i.e. the MiniJava code can be compiled</td>
+</tr>
+</table>
+
+__All valid semantic test-cases are also used, the non-semantic compile-(firm)-only
+should be placed into the `compile-only` folder__
 
 Test types for the compile-firm and compile mode
 ------------------------------------------------
@@ -171,12 +186,13 @@ usage: mjt.py [-h] [--only_incorrect_tests] [--all_exec_tests]
               [--produce_no_reports] [--produce_all_reports] [--parallel]
               [--output_no_incorrect_reports] [--color] [--ci_testing]
               [--log_level LOG_LEVEL]
-              {all,lexer,syntax,ast,semantic,compile-firm,compile} MJ_RUN
+              {all,lexer,syntax,ast,semantic,compile-firm-only,compile-only,
+               compile-firm,compile} MJ_RUN
 
 MiniJava test runner
 
 positional arguments:
-  {all,lexer,syntax,ast,semantic,compile-firm,exec}
+  {all,lexer,syntax,ast,semantic,compile-firm-only,compile-only,compile-firm,exec}
                         What do you want to test?
   MJ_RUN                Command to run your MiniJava implementation, e.g.
                         `mj/run`, can be omitted by assigning the environment

mjtest/cli.py

@@ -87,6 +87,7 @@ if True:#__name__ == '__main__':
     if args["mode"] == "all":
         report_subdir = datetime.now().strftime("%d-%m-%y_%H-%M-%S")
         for mode in [TestMode.lexer, TestMode.syntax, TestMode.ast, TestMode.semantic,
+                     TestMode.compile_firm_only, TestMode.compile_only,
                      TestMode.compile_firm, TestMode.compile]:
             args["all_exec_tests"] = True
             args["report_subdir"] = report_subdir + "_" + mode

mjtest/environment.py

@@ -34,6 +34,10 @@ class TestMode:
 
     compile = "compile"
 
+    compile_firm_only = "compile-firm-only"
+
+    compile_only = "compile-only"
+
     exec = "exec"
 
     bench = "bench"
@@ -42,12 +46,14 @@ class TestMode:
         ast: [syntax],
         compile_firm: [exec],
         compile: [exec],
+        compile_firm_only: [semantic, compile_only],
+        compile_only: [semantic],
         bench: [exec]
     }
 
 """ All 'success' tests of the n.th mode can used as 'success' tests for the n-1.th mode"""
-TEST_MODES = [TestMode.lexer, TestMode.syntax, TestMode.ast, TestMode.semantic, TestMode.compile_firm,
-              TestMode.exec, TestMode.bench, TestMode.compile]
+TEST_MODES = [TestMode.lexer, TestMode.syntax, TestMode.ast, TestMode.semantic, TestMode.compile_firm_only,
+              TestMode.compile_only, TestMode.compile_firm, TestMode.exec, TestMode.bench, TestMode.compile]
 
 
 def get_test_dirname(mode: str) -> str:
@@ -163,8 +169,10 @@ class Environment:
             TestMode.syntax: "--parsetest",
             TestMode.ast: "--print-ast",
             TestMode.semantic: "--check",
+            TestMode.compile_only: None,
+            TestMode.compile_firm_only: "--compile-firm",
             TestMode.compile_firm: "--compile-firm",
-            TestMode.compile: None
+            TestMode.compile: None,
         }[mode]
         cmd = [self.mj_run_cmd]
         if mode_flag:

mjtest/test/exec_tests.py

@@ -48,6 +48,7 @@ class JavaExecTest(BasicSyntaxTest):
                 self._expected_output_file = self._prev_out_file
                 _LOG.info("Reuse old java output file \"{}\"".format(path.relpath(self._prev_out_file)))
         self._should_succeed = True
+        self._only_compile = self.MODE.endswith("-only")
 
     def run(self) -> BasicDiffTestResult:
         is_big_testcase = "big" in self.file
@@ -77,7 +78,7 @@ class JavaExecTest(BasicSyntaxTest):
                                 chars.append(int(part))
                 input_str = chars.decode()
 
-        if not self._has_expected_output_file:
+        if not self._has_expected_output_file and not self._only_compile:
             _, err, javac_rtcode = \
                 self.env.run_command("javac", base_filename + ".java", timeout=timeout)
             if javac_rtcode != 0:
@@ -92,27 +93,29 @@ class JavaExecTest(BasicSyntaxTest):
             if not main_class:
                 _LOG.debug("Can't find a main class, using the file name instead")
                 main_class = base_filename
-            exp_out, err, java_rtcode = \
-                self.env.run_command("java", get_main_class_name(base_filename + ".java"), timeout=timeout, input_str=input_str)
-            if javac_rtcode != 0:
-                test_result.add_long_text("Java output: ", exp_out.decode())
-                test_result.incorrect_msg = "java runtime error"
-                test_result.set_error_code(java_rtcode)
-                test_result.add_long_text("Java error message", err.decode())
-                test_result.add_file("Source file", self.preprocessed_file)
-                if self._input_file:
-                    test_result.add_file("Input file", self.file)
-                os.chdir(cwd)
-                return test_result
-            exp_out = exp_out.decode()
-
-            with open(self._prev_out_file, "w") as f:
-                f.write(exp_out)
-                f.flush()
-            with open(self._prev_out_hash_file, "w") as f:
-                f.write(self._hash_sum_for_file(base_filename + ".java"))
-                f.flush()
-        if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE:
+            if not self._only_compile:
+                exp_out, err, java_rtcode = \
+                    self.env.run_command("java", get_main_class_name(base_filename + ".java"), timeout=timeout, input_str=input_str)
+                if javac_rtcode != 0:
+                    test_result.add_long_text("Java output: ", exp_out.decode())
+                    test_result.incorrect_msg = "java runtime error"
+                    test_result.set_error_code(java_rtcode)
+                    test_result.add_long_text("Java error message", err.decode())
+                    test_result.add_file("Source file", self.preprocessed_file)
+                    if self._input_file:
+                        test_result.add_file("Input file", self.file)
+                    os.chdir(cwd)
+                    return test_result
+                exp_out = exp_out.decode()
+
+                with open(self._prev_out_file, "w") as f:
+                    f.write(exp_out)
+                    f.flush()
+                with open(self._prev_out_hash_file, "w") as f:
+                    f.write(self._hash_sum_for_file(base_filename + ".java"))
+                    f.flush()
+        if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE \
+                and not self._only_compile:
             with open(self._expected_output_file, "r") as f:
                 exp_out = f.read()
 
@@ -137,6 +140,12 @@ class JavaExecTest(BasicSyntaxTest):
             except:
                 os.chdir(cwd)
                 raise
+            if self._only_compile:
+                if not os.path.exists("a.out"):
+                    test_result.incorrect_msg = "binary cannot be found"
+                    test_result.set_error_code(1)
+                    test_result.add_file("Source file", self.preprocessed_file)
+                return test_result
             try:
                 out, err, rtcode = self.env.run_command("./a.out", timeout=timeout, input_str=input_str)
                 if rtcode != 0:
@@ -228,6 +237,16 @@ class JavaCompileExecTest(JavaExecTest):
 
 TestCase.TEST_CASE_CLASSES[TestMode.compile].append(JavaCompileExecTest)
 
+class JavaCompileOnlyTest(JavaExecTest):
+    MODE = TestMode.compile_only
+
+TestCase.TEST_CASE_CLASSES[TestMode.compile_only].append(JavaCompileOnlyTest)
+
+class JavaCompileFirmOnlyTest(JavaExecTest):
+    MODE = TestMode.compile_firm_only
+
+TestCase.TEST_CASE_CLASSES[TestMode.compile_firm_only].append(JavaCompileFirmOnlyTest)
+
 class JavaInfiniteLoopTest(BasicSyntaxTest):
 
     FILE_ENDINGS = [".inf.java", ".inf.mj"]