Fix bug related to invalid uncode characters

mjtest/test/ast_tests.py

@@ -7,6 +7,8 @@ from mjtest.test.syntax_tests import BasicSyntaxTest
 from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult
 from os import path
 
+from mjtest.util.utils import decode
+
 _LOG = logging.getLogger("tests")
 
 
@@ -59,6 +61,6 @@ class ASTPrettyPrintTest(BasicSyntaxTest):
         with open(output_file, "wb") as f:
             f.write(out)
             f.flush()
-        return rtcode, out.decode(), err.decode()
+        return rtcode, decode(out), decode(err)
 
 TestCase.TEST_CASE_CLASSES["ast"].append(ASTPrettyPrintTest)

mjtest/test/bench.py

@@ -16,7 +16,7 @@ from mjtest.environment import TestMode, Environment
 from mjtest.test.syntax_tests import BasicSyntaxTest
 from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult
 from mjtest.util.shell import SigKill
-from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict
+from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict, decode
 
 _LOG = logging.getLogger("bench_tests")
 
@@ -85,7 +85,7 @@ class BenchExecTest(BasicSyntaxTest):
                     _LOG.error("File \"{}\" isn't valid Java".format(self.preprocessed_file))
                     test_result.incorrect_msg = "invalid java code, but output file missing"
                     test_result.set_error_code(javac_rtcode)
-                    test_result.add_long_text("Javac error message", err.decode())
+                    test_result.add_long_text("Javac error message", decode(err))
                     test_result.add_file("Source file", self.preprocessed_file)
                     os.chdir(cwd)
                     return test_result
@@ -102,8 +102,8 @@ class BenchExecTest(BasicSyntaxTest):
                     if rtcode != 0:
                         test_result.incorrect_msg = "file can't be compiled"
                         test_result.set_error_code(rtcode)
-                        test_result.add_long_text("Error output", err.decode())
-                        test_result.add_long_text("Output", out.decode())
+                        test_result.add_long_text("Error output", decode(err))
+                        test_result.add_long_text("Output", decode(out))
                         test_result.add_file("Source file", self.preprocessed_file)
                         os.chdir(cwd)
                         return test_result

mjtest/test/exec_tests.py

@@ -14,7 +14,7 @@ from mjtest.environment import TestMode, Environment
 from mjtest.test.syntax_tests import BasicSyntaxTest
 from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult
 from mjtest.util.shell import SigKill
-from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict
+from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict, decode
 
 _LOG = logging.getLogger("exec_tests")
 
@@ -76,7 +76,7 @@ class JavaExecTest(BasicSyntaxTest):
                             part = part.strip()
                             if len(part) > 0:
                                 chars.append(int(part))
-                input_str = chars.decode()
+                input_str = decode(chars)
 
         if not self._has_expected_output_file and not self._only_compile:
             _, err, javac_rtcode = \
@@ -85,7 +85,7 @@ class JavaExecTest(BasicSyntaxTest):
                 _LOG.error("File \"{}\" isn't valid Java".format(self.preprocessed_file))
                 test_result.incorrect_msg = "invalid java code, but output file missing"
                 test_result.set_error_code(javac_rtcode)
-                test_result.add_long_text("Javac error message", err.decode())
+                test_result.add_long_text("Javac error message", decode(err))
                 test_result.add_file("Source file", self.preprocessed_file)
                 os.chdir(cwd)
                 return test_result
@@ -97,16 +97,16 @@ class JavaExecTest(BasicSyntaxTest):
                 exp_out, err, java_rtcode = \
                     self.env.run_command("java", get_main_class_name(base_filename + ".java"), timeout=timeout, input_str=input_str)
                 if javac_rtcode != 0:
-                    test_result.add_long_text("Java output: ", exp_out.decode())
+                    test_result.add_long_text("Java output: ", decode(exp_out))
                     test_result.incorrect_msg = "java runtime error"
                     test_result.set_error_code(java_rtcode)
-                    test_result.add_long_text("Java error message", err.decode())
+                    test_result.add_long_text("Java error message", decode(err))
                     test_result.add_file("Source file", self.preprocessed_file)
                     if self._input_file:
                         test_result.add_file("Input file", self.file)
                     os.chdir(cwd)
                     return test_result
-                exp_out = exp_out.decode()
+                exp_out = decode(exp_out)
 
                 with open(self._prev_out_file, "w") as f:
                     f.write(exp_out)
@@ -118,7 +118,6 @@ class JavaExecTest(BasicSyntaxTest):
                 and not self._only_compile:
             with open(self._expected_output_file, "r") as f:
                 exp_out = f.read()
-
         try:
             out, err, rtcode = None, None, None
             try:
@@ -126,8 +125,8 @@ class JavaExecTest(BasicSyntaxTest):
                 if rtcode != 0:
                     test_result.incorrect_msg = "file can't be compiled"
                     test_result.set_error_code(rtcode)
-                    test_result.add_long_text("Error output", err.decode())
-                    test_result.add_long_text("Output", out.decode())
+                    test_result.add_long_text("Error output", decode(err))
+                    test_result.add_long_text("Output", decode(out))
                     test_result.add_file("Source file", self.preprocessed_file)
                     os.chdir(cwd)
                     return test_result
@@ -151,8 +150,8 @@ class JavaExecTest(BasicSyntaxTest):
                 if rtcode != 0:
                     test_result.incorrect_msg = "binary can't be run, non zero error code"
                     test_result.set_error_code(rtcode)
-                    test_result.add_long_text("Error output", err.decode())
-                    test_result.add_long_text("Output", out.decode())
+                    test_result.add_long_text("Error output", decode(err))
+                    test_result.add_long_text("Output", decode(out))
                     test_result.add_file("Source file", self.preprocessed_file)
                     if self._input_file:
                         test_result.add_file("Input file", self.file)
@@ -169,7 +168,7 @@ class JavaExecTest(BasicSyntaxTest):
             except:
                 os.chdir(cwd)
                 raise
-            out = out.decode()
+            out = decode(out)
             if self.type == self.MODE and self.env.mode == self.MODE:
                 if exp_out.strip() != out.strip():
                     test_result.incorrect_msg = "incorrect output"
@@ -185,7 +184,7 @@ class JavaExecTest(BasicSyntaxTest):
                     test_result.add_long_text("Output", out)
                 os.chdir(cwd)
                 return test_result
-            return BasicTestResult(self, rtcode, out, err.decode())
+            return BasicTestResult(self, rtcode, out, decode(err))
         except SigKill as sig:
             os.chdir(cwd)
             assert False
@@ -274,8 +273,8 @@ class JavaInfiniteLoopTest(BasicSyntaxTest):
             if rtcode != 0:
                 test_result.incorrect_msg = "file can't be compiled"
                 test_result.set_error_code(rtcode)
-                test_result.add_long_text("Error output", err.decode())
-                test_result.add_long_text("Output", out.decode())
+                test_result.add_long_text("Error output", decode(err))
+                test_result.add_long_text("Output", decode(out))
                 test_result.add_file("Source file", self.preprocessed_file)
                 os.chdir(cwd)
                 return test_result
@@ -283,17 +282,17 @@ class JavaInfiniteLoopTest(BasicSyntaxTest):
             if rtcode != 0:
                 test_result.incorrect_msg = "binary can't be run, non zero error code"
                 test_result.set_error_code(rtcode)
-                test_result.add_long_text("Error output", err.decode())
-                test_result.add_long_text("Output", out.decode())
+                test_result.add_long_text("Error output", decode(err))
+                test_result.add_long_text("Output", decode(out))
                 test_result.add_file("Source file", self.preprocessed_file)
                 os.chdir(cwd)
                 return test_result
         except SigKill as sig:
             if sig.retcode == signal.SIGXCPU:
                 out, _, _, _ = self.env.run_command_till_timeout("./a.out", timeout=1)
-                test_result.add_long_text("Output", out.decode())
+                test_result.add_long_text("Output", decode(out))
                 if self._has_output_file:
-                    out = out.decode().strip()
+                    out = decode(out).strip()
                     exp_out = ""
                     with open(self._output_file, "r") as f:
                         exp_out = f.read().strip()
@@ -361,8 +360,8 @@ class InvalidJavaExecTest(BasicSyntaxTest):
                 if rtcode != 0:
                     test_result.incorrect_msg = "file can't be compiled"
                     test_result.set_error_code(-rtcode)
-                    test_result.add_long_text("Error output", err.decode())
-                    test_result.add_long_text("Output", out.decode())
+                    test_result.add_long_text("Error output", decode(err))
+                    test_result.add_long_text("Output", decode(out))
                     test_result.add_file("Source file", self.preprocessed_file)
                     os.chdir(cwd)
                     return test_result
@@ -380,8 +379,8 @@ class InvalidJavaExecTest(BasicSyntaxTest):
                 if rtcode != 0:
                     test_result.incorrect_msg = "binary can't be run, non zero error code"
                     test_result.set_error_code(rtcode)
-                    test_result.add_long_text("Error output", err.decode())
-                    test_result.add_long_text("Output", out.decode())
+                    test_result.add_long_text("Error output", decode(err))
+                    test_result.add_long_text("Output", decode(out))
                     test_result.add_file("Source file", self.preprocessed_file)
                     os.chdir(cwd)
                     return test_result
@@ -394,7 +393,7 @@ class InvalidJavaExecTest(BasicSyntaxTest):
             except:
                 os.chdir(cwd)
                 raise
-            test_result = BasicTestResult(self, rtcode, out.decode(), err.decode())
+            test_result = BasicTestResult(self, rtcode, decode(out), decode(err))
             test_result.require_error_string_in_error_case = False
             return test_result
         except SigKill as sig:

mjtest/test/syntax_tests.py

@@ -3,6 +3,8 @@ from mjtest.environment import Environment, TestMode, TEST_MODES, get_test_dirna
 from mjtest.test.tests import TestCase, BasicTestResult
 from os import path
 
+from mjtest.util.utils import decode
+
 
 class BasicSyntaxTest(TestCase):
 
@@ -35,7 +37,7 @@ class BasicSyntaxTest(TestCase):
 
     def run(self) -> BasicTestResult:
         out, err, rtcode = self.env.run_mj_command(self.MODE, self.preprocessed_file)
-        return BasicTestResult(self, rtcode, out.decode(), err.decode())
+        return BasicTestResult(self, rtcode, decode(out), decode(err))
 
 TestCase.TEST_CASE_CLASSES[TestMode.syntax].append(BasicSyntaxTest)
 
@@ -53,7 +55,7 @@ class JavaCompileTest(BasicSyntaxTest):
         tmp_dir = self.env.create_pid_local_tmpdir()
         _, self.javac_err, self.javac_rtcode = \
             self.env.run_command("javac", path.relpath(preprocessed_file), "-d", tmp_dir, "-verbose")
-        self.javac_err = self.javac_err.decode()  # type: str
+        self.javac_err = decode(self.javac_err)  # type: str
         self._should_succeed = self._is_file_syntactically_correct() if self.SYNTAX_TEST else self.javac_rtcode == 0
 
     def _is_file_syntactically_correct(self):

mjtest/test/tests.py

 from collections import namedtuple
 import shutil
-from typing import Optional, List, Tuple, T, Union, Dict
+from typing import Optional, List, Tuple, Union, Dict
 import collections
 from mjtest.environment import Environment, TestMode, TEST_MODES, get_test_dirname
 from os.path import join, exists, basename
@@ -8,7 +8,7 @@ import logging
 import os
 import multiprocessing
 from mjtest.util.parallelism import available_cpu_count
-from mjtest.util.utils import cprint, colored
+from mjtest.util.utils import cprint, colored, decode
 from pprint import pprint
 import shutil
 import difflib
@@ -568,8 +568,8 @@ class DiffTest(TestCase):
             #else:
             #    _LOG.error("Expected output file for test case {}:{} is missing.".format(self.MODE, self.short_name()))
         if self.type == self.MODE and self.env.mode == self.MODE:
-            return BasicDiffTestResult(self, rtcode, out.decode(), err.decode(), exp_out)
-        return BasicTestResult(self, rtcode, out.decode(), err.decode())
+            return BasicDiffTestResult(self, rtcode, decode(out), decode(err), exp_out)
+        return BasicTestResult(self, rtcode, decode(out), decode(err))
 
 
 class LexerDiffTest(DiffTest):

mjtest/util/utils.py

@@ -113,4 +113,11 @@ class InsertionTimeOrderedDict:
         ret = InsertionTimeOrderedDict()
         for item in items:
             ret[key_func(item)] = item
-        return ret
+        return ret
\ No newline at end of file
+
+
+def decode(arr: bytes) -> str:
+    """
+    Decodes the passed byte array as UTF8 and handles invalid characters
+    """
+    return arr.decode("utf-8", "backslashreplace")
\ No newline at end of file