diff --git a/mjtest/test/ast_tests.py b/mjtest/test/ast_tests.py index 201c20bc863f698c6d3001405e6ab1306ff55c3a..1252527571e5ad2d9f910f5d1143e9b94cf3978b 100644 --- a/mjtest/test/ast_tests.py +++ b/mjtest/test/ast_tests.py @@ -7,6 +7,8 @@ from mjtest.test.syntax_tests import BasicSyntaxTest from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult from os import path +from mjtest.util.utils import decode + _LOG = logging.getLogger("tests") @@ -59,6 +61,6 @@ class ASTPrettyPrintTest(BasicSyntaxTest): with open(output_file, "wb") as f: f.write(out) f.flush() - return rtcode, out.decode(), err.decode() + return rtcode, decode(out), decode(err) TestCase.TEST_CASE_CLASSES["ast"].append(ASTPrettyPrintTest) diff --git a/mjtest/test/bench.py b/mjtest/test/bench.py index 55b8f463aa9cd656c285f387f45217d5de6f52b0..d1fb24102891609d5f51f2d30641e4dfb98b1dc6 100644 --- a/mjtest/test/bench.py +++ b/mjtest/test/bench.py @@ -16,7 +16,7 @@ from mjtest.environment import TestMode, Environment from mjtest.test.syntax_tests import BasicSyntaxTest from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult from mjtest.util.shell import SigKill -from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict +from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict, decode _LOG = logging.getLogger("bench_tests") @@ -85,7 +85,7 @@ class BenchExecTest(BasicSyntaxTest): _LOG.error("File \"{}\" isn't valid Java".format(self.preprocessed_file)) test_result.incorrect_msg = "invalid java code, but output file missing" test_result.set_error_code(javac_rtcode) - test_result.add_long_text("Javac error message", err.decode()) + test_result.add_long_text("Javac error message", decode(err)) test_result.add_file("Source file", self.preprocessed_file) os.chdir(cwd) return test_result @@ -102,8 +102,8 @@ class BenchExecTest(BasicSyntaxTest): if rtcode != 0: test_result.incorrect_msg = "file can't be compiled" test_result.set_error_code(rtcode) - test_result.add_long_text("Error output", err.decode()) - test_result.add_long_text("Output", out.decode()) + test_result.add_long_text("Error output", decode(err)) + test_result.add_long_text("Output", decode(out)) test_result.add_file("Source file", self.preprocessed_file) os.chdir(cwd) return test_result diff --git a/mjtest/test/exec_tests.py b/mjtest/test/exec_tests.py index 2044a64bb364320abcc2f23d92eacac88f3d1291..98129d592d8859c69cd8bb5ba589b0c2fc6bf326 100644 --- a/mjtest/test/exec_tests.py +++ b/mjtest/test/exec_tests.py @@ -14,7 +14,7 @@ from mjtest.environment import TestMode, Environment from mjtest.test.syntax_tests import BasicSyntaxTest from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult from mjtest.util.shell import SigKill -from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict +from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict, decode _LOG = logging.getLogger("exec_tests") @@ -76,7 +76,7 @@ class JavaExecTest(BasicSyntaxTest): part = part.strip() if len(part) > 0: chars.append(int(part)) - input_str = chars.decode() + input_str = decode(chars) if not self._has_expected_output_file and not self._only_compile: _, err, javac_rtcode = \ @@ -85,7 +85,7 @@ class JavaExecTest(BasicSyntaxTest): _LOG.error("File \"{}\" isn't valid Java".format(self.preprocessed_file)) test_result.incorrect_msg = "invalid java code, but output file missing" test_result.set_error_code(javac_rtcode) - test_result.add_long_text("Javac error message", err.decode()) + test_result.add_long_text("Javac error message", decode(err)) test_result.add_file("Source file", self.preprocessed_file) os.chdir(cwd) return test_result @@ -97,16 +97,16 @@ class JavaExecTest(BasicSyntaxTest): exp_out, err, java_rtcode = \ self.env.run_command("java", get_main_class_name(base_filename + ".java"), timeout=timeout, input_str=input_str) if javac_rtcode != 0: - test_result.add_long_text("Java output: ", exp_out.decode()) + test_result.add_long_text("Java output: ", decode(exp_out)) test_result.incorrect_msg = "java runtime error" test_result.set_error_code(java_rtcode) - test_result.add_long_text("Java error message", err.decode()) + test_result.add_long_text("Java error message", decode(err)) test_result.add_file("Source file", self.preprocessed_file) if self._input_file: test_result.add_file("Input file", self.file) os.chdir(cwd) return test_result - exp_out = exp_out.decode() + exp_out = decode(exp_out) with open(self._prev_out_file, "w") as f: f.write(exp_out) @@ -118,7 +118,6 @@ class JavaExecTest(BasicSyntaxTest): and not self._only_compile: with open(self._expected_output_file, "r") as f: exp_out = f.read() - try: out, err, rtcode = None, None, None try: @@ -126,8 +125,8 @@ class JavaExecTest(BasicSyntaxTest): if rtcode != 0: test_result.incorrect_msg = "file can't be compiled" test_result.set_error_code(rtcode) - test_result.add_long_text("Error output", err.decode()) - test_result.add_long_text("Output", out.decode()) + test_result.add_long_text("Error output", decode(err)) + test_result.add_long_text("Output", decode(out)) test_result.add_file("Source file", self.preprocessed_file) os.chdir(cwd) return test_result @@ -151,8 +150,8 @@ class JavaExecTest(BasicSyntaxTest): if rtcode != 0: test_result.incorrect_msg = "binary can't be run, non zero error code" test_result.set_error_code(rtcode) - test_result.add_long_text("Error output", err.decode()) - test_result.add_long_text("Output", out.decode()) + test_result.add_long_text("Error output", decode(err)) + test_result.add_long_text("Output", decode(out)) test_result.add_file("Source file", self.preprocessed_file) if self._input_file: test_result.add_file("Input file", self.file) @@ -169,7 +168,7 @@ class JavaExecTest(BasicSyntaxTest): except: os.chdir(cwd) raise - out = out.decode() + out = decode(out) if self.type == self.MODE and self.env.mode == self.MODE: if exp_out.strip() != out.strip(): test_result.incorrect_msg = "incorrect output" @@ -185,7 +184,7 @@ class JavaExecTest(BasicSyntaxTest): test_result.add_long_text("Output", out) os.chdir(cwd) return test_result - return BasicTestResult(self, rtcode, out, err.decode()) + return BasicTestResult(self, rtcode, out, decode(err)) except SigKill as sig: os.chdir(cwd) assert False @@ -274,8 +273,8 @@ class JavaInfiniteLoopTest(BasicSyntaxTest): if rtcode != 0: test_result.incorrect_msg = "file can't be compiled" test_result.set_error_code(rtcode) - test_result.add_long_text("Error output", err.decode()) - test_result.add_long_text("Output", out.decode()) + test_result.add_long_text("Error output", decode(err)) + test_result.add_long_text("Output", decode(out)) test_result.add_file("Source file", self.preprocessed_file) os.chdir(cwd) return test_result @@ -283,17 +282,17 @@ class JavaInfiniteLoopTest(BasicSyntaxTest): if rtcode != 0: test_result.incorrect_msg = "binary can't be run, non zero error code" test_result.set_error_code(rtcode) - test_result.add_long_text("Error output", err.decode()) - test_result.add_long_text("Output", out.decode()) + test_result.add_long_text("Error output", decode(err)) + test_result.add_long_text("Output", decode(out)) test_result.add_file("Source file", self.preprocessed_file) os.chdir(cwd) return test_result except SigKill as sig: if sig.retcode == signal.SIGXCPU: out, _, _, _ = self.env.run_command_till_timeout("./a.out", timeout=1) - test_result.add_long_text("Output", out.decode()) + test_result.add_long_text("Output", decode(out)) if self._has_output_file: - out = out.decode().strip() + out = decode(out).strip() exp_out = "" with open(self._output_file, "r") as f: exp_out = f.read().strip() @@ -361,8 +360,8 @@ class InvalidJavaExecTest(BasicSyntaxTest): if rtcode != 0: test_result.incorrect_msg = "file can't be compiled" test_result.set_error_code(-rtcode) - test_result.add_long_text("Error output", err.decode()) - test_result.add_long_text("Output", out.decode()) + test_result.add_long_text("Error output", decode(err)) + test_result.add_long_text("Output", decode(out)) test_result.add_file("Source file", self.preprocessed_file) os.chdir(cwd) return test_result @@ -380,8 +379,8 @@ class InvalidJavaExecTest(BasicSyntaxTest): if rtcode != 0: test_result.incorrect_msg = "binary can't be run, non zero error code" test_result.set_error_code(rtcode) - test_result.add_long_text("Error output", err.decode()) - test_result.add_long_text("Output", out.decode()) + test_result.add_long_text("Error output", decode(err)) + test_result.add_long_text("Output", decode(out)) test_result.add_file("Source file", self.preprocessed_file) os.chdir(cwd) return test_result @@ -394,7 +393,7 @@ class InvalidJavaExecTest(BasicSyntaxTest): except: os.chdir(cwd) raise - test_result = BasicTestResult(self, rtcode, out.decode(), err.decode()) + test_result = BasicTestResult(self, rtcode, decode(out), decode(err)) test_result.require_error_string_in_error_case = False return test_result except SigKill as sig: diff --git a/mjtest/test/syntax_tests.py b/mjtest/test/syntax_tests.py index 8d16904bd1b4162c82b65edaec164f1a5bde9332..45178363337b117aac92ebc609b785f802606859 100644 --- a/mjtest/test/syntax_tests.py +++ b/mjtest/test/syntax_tests.py @@ -3,6 +3,8 @@ from mjtest.environment import Environment, TestMode, TEST_MODES, get_test_dirna from mjtest.test.tests import TestCase, BasicTestResult from os import path +from mjtest.util.utils import decode + class BasicSyntaxTest(TestCase): @@ -35,7 +37,7 @@ class BasicSyntaxTest(TestCase): def run(self) -> BasicTestResult: out, err, rtcode = self.env.run_mj_command(self.MODE, self.preprocessed_file) - return BasicTestResult(self, rtcode, out.decode(), err.decode()) + return BasicTestResult(self, rtcode, decode(out), decode(err)) TestCase.TEST_CASE_CLASSES[TestMode.syntax].append(BasicSyntaxTest) @@ -53,7 +55,7 @@ class JavaCompileTest(BasicSyntaxTest): tmp_dir = self.env.create_pid_local_tmpdir() _, self.javac_err, self.javac_rtcode = \ self.env.run_command("javac", path.relpath(preprocessed_file), "-d", tmp_dir, "-verbose") - self.javac_err = self.javac_err.decode() # type: str + self.javac_err = decode(self.javac_err) # type: str self._should_succeed = self._is_file_syntactically_correct() if self.SYNTAX_TEST else self.javac_rtcode == 0 def _is_file_syntactically_correct(self): diff --git a/mjtest/test/tests.py b/mjtest/test/tests.py index 1710eeff2ef8cbc62be0a1f43494b6b09bc819d7..09b215ff3fb5d4bdf94dabf6cb0af25141dd5310 100644 --- a/mjtest/test/tests.py +++ b/mjtest/test/tests.py @@ -1,6 +1,6 @@ from collections import namedtuple import shutil -from typing import Optional, List, Tuple, T, Union, Dict +from typing import Optional, List, Tuple, Union, Dict import collections from mjtest.environment import Environment, TestMode, TEST_MODES, get_test_dirname from os.path import join, exists, basename @@ -8,7 +8,7 @@ import logging import os import multiprocessing from mjtest.util.parallelism import available_cpu_count -from mjtest.util.utils import cprint, colored +from mjtest.util.utils import cprint, colored, decode from pprint import pprint import shutil import difflib @@ -568,8 +568,8 @@ class DiffTest(TestCase): #else: # _LOG.error("Expected output file for test case {}:{} is missing.".format(self.MODE, self.short_name())) if self.type == self.MODE and self.env.mode == self.MODE: - return BasicDiffTestResult(self, rtcode, out.decode(), err.decode(), exp_out) - return BasicTestResult(self, rtcode, out.decode(), err.decode()) + return BasicDiffTestResult(self, rtcode, decode(out), decode(err), exp_out) + return BasicTestResult(self, rtcode, decode(out), decode(err)) class LexerDiffTest(DiffTest): diff --git a/mjtest/util/utils.py b/mjtest/util/utils.py index ba9a430441a0f8750822df4b22a55fd1e172ac34..9665c53f6dfb798d4ea49e104409bd9c87fd5a41 100644 --- a/mjtest/util/utils.py +++ b/mjtest/util/utils.py @@ -113,4 +113,11 @@ class InsertionTimeOrderedDict: ret = InsertionTimeOrderedDict() for item in items: ret[key_func(item)] = item - return ret \ No newline at end of file + return ret + + +def decode(arr: bytes) -> str: + """ + Decodes the passed byte array as UTF8 and handles invalid characters + """ + return arr.decode("utf-8", "backslashreplace") \ No newline at end of file