Really fix unicode code bugs

mjtest/test/ast_tests.py

@@ -58,7 +58,7 @@ class ASTPrettyPrintTest(BasicSyntaxTest):
 
     def _pretty_print(self, input_file: str, output_file: str) -> Tuple[int, str, str]:
         out, err, rtcode = self.env.run_mj_command(TestMode.ast, input_file)
-        with open(output_file, "wb") as f:
+        with open(output_file, "wb", errors="backslashreplace") as f:
             f.write(out)
             f.flush()
         return rtcode, decode(out), decode(err)

mjtest/test/exec_tests.py

@@ -66,10 +66,10 @@ class JavaExecTest(BasicSyntaxTest):
 
         if self._has_input_file:
             if self._has_character_input:
-                with open(self._input_file, "r") as f:
+                with open(self._input_file, "r", errors="backslashreplace") as f:
                     input_str = f.read()
             else:
-                with open(self._input_file, "r") as f:
+                with open(self._input_file, "r", errors="backslashreplace") as f:
                     chars = bytearray('ascii', 'ignore')  # type: bytearray
                     for line in f.readlines():
                         for part in line.split(" "):
@@ -108,15 +108,15 @@ class JavaExecTest(BasicSyntaxTest):
                     return test_result
                 exp_out = decode(exp_out)
 
-                with open(self._prev_out_file, "w") as f:
+                with open(self._prev_out_file, "w", errors="backslashreplace") as f:
                     f.write(exp_out)
                     f.flush()
-                with open(self._prev_out_hash_file, "w") as f:
+                with open(self._prev_out_hash_file, "w", errors="backslashreplace") as f:
                     f.write(self._hash_sum_for_file(base_filename + ".java"))
                     f.flush()
         if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE \
                 and not self._only_compile:
-            with open(self._expected_output_file, "r") as f:
+            with open(self._expected_output_file, "r", errors="backslashreplace") as f:
                 exp_out = f.read()
         try:
             out, err, rtcode = None, None, None
@@ -194,7 +194,7 @@ class JavaExecTest(BasicSyntaxTest):
 
     def _check_hash_sum(self, file: str, hash_sum_file: str) -> bool:
         old_hash = ""
-        with open(hash_sum_file, "r") as f:
+        with open(hash_sum_file, "r", errors="backslashreplace") as f:
             try:
                 old_hash = f.readline().strip()
             except UnicodeDecodeError:
@@ -203,7 +203,7 @@ class JavaExecTest(BasicSyntaxTest):
 
     @classmethod
     def _hash_sum_for_file(self, file: str) -> str:
-        with open(file, "r") as f:
+        with open(file, "r", errors="backslashreplace") as f:
             return hashlib.sha256(f.read().encode()).hexdigest()
 
     @classmethod
@@ -294,7 +294,7 @@ class JavaInfiniteLoopTest(BasicSyntaxTest):
                 if self._has_output_file:
                     out = decode(out).strip()
                     exp_out = ""
-                    with open(self._output_file, "r") as f:
+                    with open(self._output_file, "r", errors="backslashreplace") as f:
                         exp_out = f.read().strip()
                     test_result.add_long_text("Expected output start", exp_out)
                     if not out.startswith(exp_out):

mjtest/test/tests.py

@@ -50,7 +50,7 @@ class TestSuite:
         correct_test_cases = set()
         log_file = self._log_file_for_type(mode)
         if exists(log_file):
-            with open(log_file) as f:
+            with open(log_file, errors="backslashreplace") as f:
                 correct_test_cases = set()
                 for t in f.readlines():
                     t = t.strip()
@@ -223,7 +223,7 @@ class TestSuite:
                     os.mkdir(os.path.dirname(log_file))
                 except IOError:
                     pass
-                with open(log_file, "w") as f:
+                with open(log_file, "w", errors="backslashreplace") as f:
                     f.write("\n".join(self.correct_test_cases[mode]))
             except IOError as e:
                 _LOG.exception("Caught i/o error while storing {}".format(log_file))
@@ -314,7 +314,7 @@ class TestResult:
 
     def store_at(self, file: str):
         os.makedirs(os.path.dirname(file), exist_ok=True)
-        with open(file, "w") as f:
+        with open(file, "w", errors="backslashreplace") as f:
             print(self.long_message(), file=f)
 
     def short_message(self) -> str:
@@ -348,8 +348,8 @@ class ExtensibleTestResult(TestResult):
         self.messages.append(TestResultMessage(title, content, multiline=False, with_line_numbers=False))
 
     def add_file(self, title: str, file_name: str, with_line_numbers: bool = True):
-        with open(file_name, "r") as f:
-            file_content = os.linesep.join([line.rstrip() for line in f])
+        with open(file_name, "r", errors="backslashreplace") as f:
+            file_content = os.linesep.join([line.rstrip() for line in f.read()])
         self.add_long_text(title, file_content, with_line_numbers)
 
     def succeeded(self):
@@ -461,7 +461,7 @@ class BasicTestResult(TestResult):
 
     def long_message(self) -> str:
         file_content = []
-        with open(self.test_case.preprocessed_file, "r") as f:
+        with open(self.test_case.preprocessed_file, "r", errors="backslashreplace") as f:
             file_content = [line.rstrip() for line in f]
         others = []
         for title, content, long_text in self.other_texts:
@@ -563,7 +563,7 @@ class DiffTest(TestCase):
         exp_out = ""
         if rtcode == 0 and self.should_succeed():
             if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE:
-                with open(self._expected_output_file, "r") as f:
+                with open(self._expected_output_file, "r", errors="backslashreplace") as f:
                     exp_out = f.read()
             #else:
             #    _LOG.error("Expected output file for test case {}:{} is missing.".format(self.MODE, self.short_name()))

mjtest/util/utils.py

@@ -47,7 +47,7 @@ def cprint(text: str, *args, **kwargs):
 
 def get_main_class_name(file: str) -> Optional[str]:
     current_class = None
-    with open(file, "r") as f:
+    with open(file, "r", errors="backslashreplace") as f:
         for line in f:
             if line.startswith("class ") or line.startswith("/*public*/ class "):
                 match = re.search("[A-Za-z_0-9]+", line.replace("class ", "").replace("/*public*/", ""))

preproc/preproc/preprocessor.py

@@ -48,7 +48,7 @@ class PreProcessor:
         def add_commented(line: str):
             middle_lines.append("/*{}*/".format(line))
 
-        with open(file, "r") as f:
+        with open(file, "r", errors="backslashreplace") as f:
             for line in f:
                 line = line.rstrip()
                 if self._import_regexp.match(line):
@@ -108,7 +108,7 @@ class PreProcessor:
                 print()
                 print(text)
         else:
-            with open(self.dst_file, "w") as f:
+            with open(self.dst_file, "w", errors="backslashreplace") as f:
                 for text in reversed(self.imported_strs):
                     f.write(text)
                     f.write("\n\n")
@@ -122,7 +122,7 @@ def is_importable_file(file: str) -> bool:
     has_package = False
     has_public_class = False
     has_main_method = False
-    with open(file, "r") as f:
+    with open(file, "r", errors="backslashreplace") as f:
         for line in f:
             if line.startswith("package "):
                 has_package = True