Add to big exec test cases and improve test runner

README.mdwn

@@ -25,9 +25,11 @@ _Only the lexer, syntax and ast mode is currently usable, but the others will fo
 The test different test cases for each mode are located in a folder with the same name.
 The default directory that contains all test folders is `tests`.
 
+Sub folders in side the test case folders are allowed.
+
 The different types a test cases are differentiated by their file endings.
 
-Side note: An error code greater than 0 should result in an errror message on error output containing the word `error`.
+Side note: An error code greater than 0 should result in an error message on error output containing the word `error`.
 
 
 Test types for the lexer mode
@@ -36,7 +38,7 @@ Test types for the lexer mode
 <table>
 <tr><th>File ending(s) of test cases</th><th>Expected behaviour to complete a test of this type</th></tr>
 <tr>
-    <td><code>.valid.mj</code> <code>.mj</code>
+    <td><code>.valid.mj</code> <code>.mj</code></td>
     <td>Return code is <code>0</code> and the output matches the expected output (located in the file `[test file].out`</td>
 </tr>
 <tr>
@@ -52,20 +54,16 @@ Test types for the syntax mode
 <table>
 <tr><th>File ending(s) of test cases</th><th>Expected behaviour to complete a test of this type</th></tr>
 <tr>
-    <td><code>.valid.mj</code> <code>.mj</code>
+    <td><code>.valid.mj</code> <code>.mj</code> <code>.valid.java</code> <code>.java</code>
     <td>Return code is <code>0</code>, i.e. the MiniJava is accepted as syntactically correct</td>
 </tr>
 <tr>
-    <td><code>.invalid.mj</code>
+    <td><code>.invalid.mj</code> <code>.invalid.java</code></td>
     <td>Return code is <code>&gt; 0</code> and the error output contains the word <code>error</code></td>
 </tr>
-<tr>
-    <td><code>.java</code>
-    <td>If <code>javac</code> accepts the syntax of the file then the MiniJava compiler
-    should accept its syntax too.</td>
-</tr>
 </table>
 
+It uses all semantic mode tests implicitly if `--ci_testing` is given.
 
 Test types for the ast mode
 ------------------------------
@@ -73,7 +71,7 @@ Test types for the ast mode
 <table>
 <tr><th>File ending(s) of test cases</th><th>Expected behaviour to complete a test of this type</th></tr>
 <tr>
-    <td><code>.valid.mj</code> <code>.mj</code>
+    <td><code>.valid.mj</code> <code>.mj</code> <code>.valid.java</code> <code>.java</code></td>
     <td>Pretty printing the source file should result in the same output as pretty printing the already pretty printed file.
     All lines in the lexer output for the source file should be present in the lexer output of the pretty printed file.</td>
 </tr>
@@ -82,13 +80,31 @@ Test types for the ast mode
 It uses all syntax mode tests implicitly.
 
 
+Test types for the semantic mode
+------------------------------
+
+<table>
+<tr><th>File ending(s) of test cases</th><th>Expected behaviour to complete a test of this type</th></tr>
+<tr>
+    <td><code>.valid.mj</code> <code>.mj</code> <code>.valid.java</code> <code>.java</code>
+    <td>Return code is <code>0</code>, i.e. the MiniJava is accepted as semantically correct</td>
+</tr>
+<tr>
+    <td><code>.invalid.mj</code> <code>.invalid.java</code></td>
+    <td>Return code is <code>&gt; 0</code> and the error output contains the word <code>error</code></td>
+</tr>
+</table>
+
+__All semantic mode tests have to syntactically correct___
+
+
 Test runner
 -----------
 
 ### Requirements
 The following programs are required (and executable by simply calling their names).
-- `python3` (at least Python3.3)
-- `javac` and `java` (for `.java` test cases)three
+- `python3` (at least Python3.4)
+- `javac` and `java` (for `.java` test cases)
 
 ### Installation
 

mjtest/test/syntax_tests.py

@@ -6,7 +6,7 @@ from os import path
 
 class BasicSyntaxTest(TestCase):
 
-    FILE_ENDINGS = [".invalid.mj", ".valid.mj", ".mj"]
+    FILE_ENDINGS = [".invalid.mj", ".valid.mj", ".mj", ".invalid.java", ".java"]
     MODE = TestMode.syntax
 
     def __init__(self, env: Environment, type: str, file: str):
@@ -14,7 +14,7 @@ class BasicSyntaxTest(TestCase):
         if type != self.MODE and TEST_MODES.index(type) > TEST_MODES.index(self.MODE):
             self._should_succeed = True
         else:
-            self._should_succeed = not file.endswith(".invalid.mj")
+            self._should_succeed = not file.endswith(".invalid.mj") and not file.endswith(".invalid.java")
     def should_succeed(self) -> bool:
         return self._should_succeed
 
@@ -58,4 +58,4 @@ class JavaCompileTest(BasicSyntaxTest):
         ret.add_additional_text_line("Is syntax correct? ", str(self._is_file_syntactically_correct()))
         return ret
 
-TestCase.TEST_CASE_CLASSES[TestMode.syntax].append(JavaCompileTest)
+#TestCase.TEST_CASE_CLASSES[TestMode.syntax].append(JavaCompileTest)

mjtest/test/tests.py

@@ -62,7 +62,14 @@ class TestSuite:
         if m != self.env.mode and self.env.mode in TestMode.USE_TESTS_OF_OTHER and \
             m in TestMode.USE_TESTS_OF_OTHER[self.env.mode]:
             m = self.env.mode
-        for file in sorted(os.listdir(dir)):
+        file_names = []
+        for root, dirs, files in os.walk(dir):
+            base = os.path.relpath(root, dir)
+            if dir == root:
+                file_names.extend(files)
+            else:
+                file_names.extend(join(base, file) for file in files)
+        for file in sorted(file_names):
             if not TestCase.has_valid_file_ending(self.env.mode, file):
                 _LOG.debug("Skip file " + file)
             elif self.env.only_incorrect_tests and file in correct_test_cases: