From ef323b5c9290fa90fdd76ca67252c18d8a8e43cb Mon Sep 17 00:00:00 2001 From: Johannes Bechberger Date: Tue, 11 Dec 2018 16:06:13 +0100 Subject: [PATCH] Really fix unicode code bugs --- mjtest/test/ast_tests.py | 2 +- mjtest/test/exec_tests.py | 16 ++++++++-------- mjtest/test/tests.py | 14 +++++++------- mjtest/util/utils.py | 2 +- preproc/preproc/preprocessor.py | 6 +++--- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/mjtest/test/ast_tests.py b/mjtest/test/ast_tests.py index 1252527..e355a95 100644 --- a/mjtest/test/ast_tests.py +++ b/mjtest/test/ast_tests.py @@ -58,7 +58,7 @@ class ASTPrettyPrintTest(BasicSyntaxTest): def _pretty_print(self, input_file: str, output_file: str) -> Tuple[int, str, str]: out, err, rtcode = self.env.run_mj_command(TestMode.ast, input_file) - with open(output_file, "wb") as f: + with open(output_file, "wb", errors="backslashreplace") as f: f.write(out) f.flush() return rtcode, decode(out), decode(err) diff --git a/mjtest/test/exec_tests.py b/mjtest/test/exec_tests.py index 98129d5..7c49d39 100644 --- a/mjtest/test/exec_tests.py +++ b/mjtest/test/exec_tests.py @@ -66,10 +66,10 @@ class JavaExecTest(BasicSyntaxTest): if self._has_input_file: if self._has_character_input: - with open(self._input_file, "r") as f: + with open(self._input_file, "r", errors="backslashreplace") as f: input_str = f.read() else: - with open(self._input_file, "r") as f: + with open(self._input_file, "r", errors="backslashreplace") as f: chars = bytearray('ascii', 'ignore') # type: bytearray for line in f.readlines(): for part in line.split(" "): @@ -108,15 +108,15 @@ class JavaExecTest(BasicSyntaxTest): return test_result exp_out = decode(exp_out) - with open(self._prev_out_file, "w") as f: + with open(self._prev_out_file, "w", errors="backslashreplace") as f: f.write(exp_out) f.flush() - with open(self._prev_out_hash_file, "w") as f: + with open(self._prev_out_hash_file, "w", errors="backslashreplace") as f: f.write(self._hash_sum_for_file(base_filename + ".java")) f.flush() if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE \ and not self._only_compile: - with open(self._expected_output_file, "r") as f: + with open(self._expected_output_file, "r", errors="backslashreplace") as f: exp_out = f.read() try: out, err, rtcode = None, None, None @@ -194,7 +194,7 @@ class JavaExecTest(BasicSyntaxTest): def _check_hash_sum(self, file: str, hash_sum_file: str) -> bool: old_hash = "" - with open(hash_sum_file, "r") as f: + with open(hash_sum_file, "r", errors="backslashreplace") as f: try: old_hash = f.readline().strip() except UnicodeDecodeError: @@ -203,7 +203,7 @@ class JavaExecTest(BasicSyntaxTest): @classmethod def _hash_sum_for_file(self, file: str) -> str: - with open(file, "r") as f: + with open(file, "r", errors="backslashreplace") as f: return hashlib.sha256(f.read().encode()).hexdigest() @classmethod @@ -294,7 +294,7 @@ class JavaInfiniteLoopTest(BasicSyntaxTest): if self._has_output_file: out = decode(out).strip() exp_out = "" - with open(self._output_file, "r") as f: + with open(self._output_file, "r", errors="backslashreplace") as f: exp_out = f.read().strip() test_result.add_long_text("Expected output start", exp_out) if not out.startswith(exp_out): diff --git a/mjtest/test/tests.py b/mjtest/test/tests.py index 09b215f..8190c5c 100644 --- a/mjtest/test/tests.py +++ b/mjtest/test/tests.py @@ -50,7 +50,7 @@ class TestSuite: correct_test_cases = set() log_file = self._log_file_for_type(mode) if exists(log_file): - with open(log_file) as f: + with open(log_file, errors="backslashreplace") as f: correct_test_cases = set() for t in f.readlines(): t = t.strip() @@ -223,7 +223,7 @@ class TestSuite: os.mkdir(os.path.dirname(log_file)) except IOError: pass - with open(log_file, "w") as f: + with open(log_file, "w", errors="backslashreplace") as f: f.write("\n".join(self.correct_test_cases[mode])) except IOError as e: _LOG.exception("Caught i/o error while storing {}".format(log_file)) @@ -314,7 +314,7 @@ class TestResult: def store_at(self, file: str): os.makedirs(os.path.dirname(file), exist_ok=True) - with open(file, "w") as f: + with open(file, "w", errors="backslashreplace") as f: print(self.long_message(), file=f) def short_message(self) -> str: @@ -348,8 +348,8 @@ class ExtensibleTestResult(TestResult): self.messages.append(TestResultMessage(title, content, multiline=False, with_line_numbers=False)) def add_file(self, title: str, file_name: str, with_line_numbers: bool = True): - with open(file_name, "r") as f: - file_content = os.linesep.join([line.rstrip() for line in f]) + with open(file_name, "r", errors="backslashreplace") as f: + file_content = os.linesep.join([line.rstrip() for line in f.read()]) self.add_long_text(title, file_content, with_line_numbers) def succeeded(self): @@ -461,7 +461,7 @@ class BasicTestResult(TestResult): def long_message(self) -> str: file_content = [] - with open(self.test_case.preprocessed_file, "r") as f: + with open(self.test_case.preprocessed_file, "r", errors="backslashreplace") as f: file_content = [line.rstrip() for line in f] others = [] for title, content, long_text in self.other_texts: @@ -563,7 +563,7 @@ class DiffTest(TestCase): exp_out = "" if rtcode == 0 and self.should_succeed(): if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE: - with open(self._expected_output_file, "r") as f: + with open(self._expected_output_file, "r", errors="backslashreplace") as f: exp_out = f.read() #else: # _LOG.error("Expected output file for test case {}:{} is missing.".format(self.MODE, self.short_name())) diff --git a/mjtest/util/utils.py b/mjtest/util/utils.py index 9665c53..9d4d77f 100644 --- a/mjtest/util/utils.py +++ b/mjtest/util/utils.py @@ -47,7 +47,7 @@ def cprint(text: str, *args, **kwargs): def get_main_class_name(file: str) -> Optional[str]: current_class = None - with open(file, "r") as f: + with open(file, "r", errors="backslashreplace") as f: for line in f: if line.startswith("class ") or line.startswith("/*public*/ class "): match = re.search("[A-Za-z_0-9]+", line.replace("class ", "").replace("/*public*/", "")) diff --git a/preproc/preproc/preprocessor.py b/preproc/preproc/preprocessor.py index b79088c..e02c0da 100644 --- a/preproc/preproc/preprocessor.py +++ b/preproc/preproc/preprocessor.py @@ -48,7 +48,7 @@ class PreProcessor: def add_commented(line: str): middle_lines.append("/*{}*/".format(line)) - with open(file, "r") as f: + with open(file, "r", errors="backslashreplace") as f: for line in f: line = line.rstrip() if self._import_regexp.match(line): @@ -108,7 +108,7 @@ class PreProcessor: print() print(text) else: - with open(self.dst_file, "w") as f: + with open(self.dst_file, "w", errors="backslashreplace") as f: for text in reversed(self.imported_strs): f.write(text) f.write("\n\n") @@ -122,7 +122,7 @@ def is_importable_file(file: str) -> bool: has_package = False has_public_class = False has_main_method = False - with open(file, "r") as f: + with open(file, "r", errors="backslashreplace") as f: for line in f: if line.startswith("package "): has_package = True -- GitLab