Commit ef323b5c authored by Johannes Bechberger's avatar Johannes Bechberger

Really fix unicode code bugs

parent cf00a6a5
......@@ -58,7 +58,7 @@ class ASTPrettyPrintTest(BasicSyntaxTest):
def _pretty_print(self, input_file: str, output_file: str) -> Tuple[int, str, str]:
out, err, rtcode = self.env.run_mj_command(TestMode.ast, input_file)
with open(output_file, "wb") as f:
with open(output_file, "wb", errors="backslashreplace") as f:
f.write(out)
f.flush()
return rtcode, decode(out), decode(err)
......
......@@ -66,10 +66,10 @@ class JavaExecTest(BasicSyntaxTest):
if self._has_input_file:
if self._has_character_input:
with open(self._input_file, "r") as f:
with open(self._input_file, "r", errors="backslashreplace") as f:
input_str = f.read()
else:
with open(self._input_file, "r") as f:
with open(self._input_file, "r", errors="backslashreplace") as f:
chars = bytearray('ascii', 'ignore') # type: bytearray
for line in f.readlines():
for part in line.split(" "):
......@@ -108,15 +108,15 @@ class JavaExecTest(BasicSyntaxTest):
return test_result
exp_out = decode(exp_out)
with open(self._prev_out_file, "w") as f:
with open(self._prev_out_file, "w", errors="backslashreplace") as f:
f.write(exp_out)
f.flush()
with open(self._prev_out_hash_file, "w") as f:
with open(self._prev_out_hash_file, "w", errors="backslashreplace") as f:
f.write(self._hash_sum_for_file(base_filename + ".java"))
f.flush()
if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE \
and not self._only_compile:
with open(self._expected_output_file, "r") as f:
with open(self._expected_output_file, "r", errors="backslashreplace") as f:
exp_out = f.read()
try:
out, err, rtcode = None, None, None
......@@ -194,7 +194,7 @@ class JavaExecTest(BasicSyntaxTest):
def _check_hash_sum(self, file: str, hash_sum_file: str) -> bool:
old_hash = ""
with open(hash_sum_file, "r") as f:
with open(hash_sum_file, "r", errors="backslashreplace") as f:
try:
old_hash = f.readline().strip()
except UnicodeDecodeError:
......@@ -203,7 +203,7 @@ class JavaExecTest(BasicSyntaxTest):
@classmethod
def _hash_sum_for_file(self, file: str) -> str:
with open(file, "r") as f:
with open(file, "r", errors="backslashreplace") as f:
return hashlib.sha256(f.read().encode()).hexdigest()
@classmethod
......@@ -294,7 +294,7 @@ class JavaInfiniteLoopTest(BasicSyntaxTest):
if self._has_output_file:
out = decode(out).strip()
exp_out = ""
with open(self._output_file, "r") as f:
with open(self._output_file, "r", errors="backslashreplace") as f:
exp_out = f.read().strip()
test_result.add_long_text("Expected output start", exp_out)
if not out.startswith(exp_out):
......
......@@ -50,7 +50,7 @@ class TestSuite:
correct_test_cases = set()
log_file = self._log_file_for_type(mode)
if exists(log_file):
with open(log_file) as f:
with open(log_file, errors="backslashreplace") as f:
correct_test_cases = set()
for t in f.readlines():
t = t.strip()
......@@ -223,7 +223,7 @@ class TestSuite:
os.mkdir(os.path.dirname(log_file))
except IOError:
pass
with open(log_file, "w") as f:
with open(log_file, "w", errors="backslashreplace") as f:
f.write("\n".join(self.correct_test_cases[mode]))
except IOError as e:
_LOG.exception("Caught i/o error while storing {}".format(log_file))
......@@ -314,7 +314,7 @@ class TestResult:
def store_at(self, file: str):
os.makedirs(os.path.dirname(file), exist_ok=True)
with open(file, "w") as f:
with open(file, "w", errors="backslashreplace") as f:
print(self.long_message(), file=f)
def short_message(self) -> str:
......@@ -348,8 +348,8 @@ class ExtensibleTestResult(TestResult):
self.messages.append(TestResultMessage(title, content, multiline=False, with_line_numbers=False))
def add_file(self, title: str, file_name: str, with_line_numbers: bool = True):
with open(file_name, "r") as f:
file_content = os.linesep.join([line.rstrip() for line in f])
with open(file_name, "r", errors="backslashreplace") as f:
file_content = os.linesep.join([line.rstrip() for line in f.read()])
self.add_long_text(title, file_content, with_line_numbers)
def succeeded(self):
......@@ -461,7 +461,7 @@ class BasicTestResult(TestResult):
def long_message(self) -> str:
file_content = []
with open(self.test_case.preprocessed_file, "r") as f:
with open(self.test_case.preprocessed_file, "r", errors="backslashreplace") as f:
file_content = [line.rstrip() for line in f]
others = []
for title, content, long_text in self.other_texts:
......@@ -563,7 +563,7 @@ class DiffTest(TestCase):
exp_out = ""
if rtcode == 0 and self.should_succeed():
if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE:
with open(self._expected_output_file, "r") as f:
with open(self._expected_output_file, "r", errors="backslashreplace") as f:
exp_out = f.read()
#else:
# _LOG.error("Expected output file for test case {}:{} is missing.".format(self.MODE, self.short_name()))
......
......@@ -47,7 +47,7 @@ def cprint(text: str, *args, **kwargs):
def get_main_class_name(file: str) -> Optional[str]:
current_class = None
with open(file, "r") as f:
with open(file, "r", errors="backslashreplace") as f:
for line in f:
if line.startswith("class ") or line.startswith("/*public*/ class "):
match = re.search("[A-Za-z_0-9]+", line.replace("class ", "").replace("/*public*/", ""))
......
......@@ -48,7 +48,7 @@ class PreProcessor:
def add_commented(line: str):
middle_lines.append("/*{}*/".format(line))
with open(file, "r") as f:
with open(file, "r", errors="backslashreplace") as f:
for line in f:
line = line.rstrip()
if self._import_regexp.match(line):
......@@ -108,7 +108,7 @@ class PreProcessor:
print()
print(text)
else:
with open(self.dst_file, "w") as f:
with open(self.dst_file, "w", errors="backslashreplace") as f:
for text in reversed(self.imported_strs):
f.write(text)
f.write("\n\n")
......@@ -122,7 +122,7 @@ def is_importable_file(file: str) -> bool:
has_package = False
has_public_class = False
has_main_method = False
with open(file, "r") as f:
with open(file, "r", errors="backslashreplace") as f:
for line in f:
if line.startswith("package "):
has_package = True
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment