Improve `compile-firm` test mode

Move all big test cases should into the folder `big`

mjtest/environment.py

@@ -99,6 +99,8 @@ class Environment:
             except IOError:
                 pass
             self.report_dir = os.path.join(self.report_dir, report_subdir or datetime.now().strftime("%d-%m-%y_%H-%M-%S"))
+            if not os.path.exists(self.report_dir):
+                os.mkdir(self.report_dir)
         else:
             self.report_dir = None
         logging.basicConfig(level=self.LOG_LEVELS[log_level])

mjtest/test/syntax_tests.py

 import shutil
-from mjtest.environment import Environment, TestMode, TEST_MODES
+from mjtest.environment import Environment, TestMode, TEST_MODES, get_test_dirname
 from mjtest.test.tests import TestCase, BasicTestResult
 from os import path
 
@@ -20,7 +20,17 @@ class BasicSyntaxTest(TestCase):
         return self._should_succeed
 
     def short_name(self) -> str:
-        return path.basename(self.file)
+        test_dir = None
+        modes = [self.MODE]
+        if self.MODE in TestMode.USE_TESTS_OF_OTHER:
+            modes += TestMode.USE_TESTS_OF_OTHER[self.MODE]
+        for mode in modes:
+            subdir = path.join(self.env.test_dir, get_test_dirname(mode))
+            if self.file.startswith(subdir):
+                test_dir = subdir
+                break
+        assert test_dir != None
+        return path.relpath(self.file, test_dir)
 
     def run(self) -> BasicTestResult:
         out, err, rtcode = self.env.run_mj_command(self.MODE, self.preprocessed_file)

mjtest/test/tests.py

@@ -41,8 +41,6 @@ class TestSuite:
 
     def _load_test_case_type(self, type: str):
         dir = join(self.env.test_dir, get_test_dirname(type))
-        if get_test_dirname(type) == "exec" and self.env.only_small_exec_tests:
-            dir = join(dir, "small")
         if exists(dir):
             self._load_test_case_dir(type, dir)
         else:
@@ -66,12 +64,19 @@ class TestSuite:
         file_names = []
         for root, dirs, files in os.walk(dir):
             base = os.path.relpath(root, dir)
+            dirname = os.path.dirname(base)
             if dir == root:
                 file_names.extend(files)
-            elif base == ".preprocessed":
+            elif ".preprocessed" in base or (dirname.startswith(".") and dirname != "."):
                 continue
-            else:
-                file_names.extend(join(base, file) for file in files)
+            elif base.startswith("big") and get_test_dirname(mode) == "exec" and self.env.only_small_exec_tests:
+                continue
+            elif base == ".":
+                continue
+            for file in files:
+                p = join(base, file)
+                if not p.startswith("."):
+                    file_names.append(p)
         for file in sorted(file_names):
             if not TestCase.has_valid_file_ending(self.env.mode, file):
                 _LOG.debug("Skip file " + file)
@@ -161,7 +166,6 @@ class TestSuite:
                 self._add_correct_test_case(test_case)
         return result
 
-
     def _run_test_case(self, test_case: 'TestCase') -> Optional['TestResult']:
         try:
             ret = test_case.run()
@@ -281,6 +285,7 @@ class TestResult:
         return self.error_code == 0
 
     def store_at(self, file: str):
+        os.makedirs(os.path.dirname(file), exist_ok=True)
         with open(file, "w") as f:
             print(self.long_message(), file=f)