Skip to content

GitLab

Commit b2df624c authored by Johannes Bechberger's avatar Johannes Bechberger
Browse files

Fix bug related to invalid uncode characters

parent 570059aa
Hide whitespace changes
Inline Side-by-side
Showing with 46 additions and 36 deletions
mjtest/test/ast_tests.py
View file @ b2df624c
......@@ -7,6 +7,8 @@ from mjtest.test.syntax_tests import BasicSyntaxTest
from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult
from os import path
from mjtest.util.utils import decode
_LOG = logging.getLogger("tests")
......@@ -59,6 +61,6 @@ class ASTPrettyPrintTest(BasicSyntaxTest):
with open(output_file, "wb") as f:
f.write(out)
f.flush()
return rtcode, out.decode(), err.decode()
return rtcode, decode(out), decode(err)
TestCase.TEST_CASE_CLASSES["ast"].append(ASTPrettyPrintTest)
mjtest/test/bench.py
View file @ b2df624c
......@@ -16,7 +16,7 @@ from mjtest.environment import TestMode, Environment
from mjtest.test.syntax_tests import BasicSyntaxTest
from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult
from mjtest.util.shell import SigKill
from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict
from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict, decode
_LOG = logging.getLogger("bench_tests")
......@@ -85,7 +85,7 @@ class BenchExecTest(BasicSyntaxTest):
_LOG.error("File \"{}\" isn't valid Java".format(self.preprocessed_file))
test_result.incorrect_msg = "invalid java code, but output file missing"
test_result.set_error_code(javac_rtcode)
test_result.add_long_text("Javac error message", err.decode())
test_result.add_long_text("Javac error message", decode(err))
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
......@@ -102,8 +102,8 @@ class BenchExecTest(BasicSyntaxTest):
if rtcode != 0:
test_result.incorrect_msg = "file can't be compiled"
test_result.set_error_code(rtcode)
test_result.add_long_text("Error output", err.decode())
test_result.add_long_text("Output", out.decode())
test_result.add_long_text("Error output", decode(err))
test_result.add_long_text("Output", decode(out))
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
......
mjtest/test/exec_tests.py
View file @ b2df624c
......@@ -14,7 +14,7 @@ from mjtest.environment import TestMode, Environment
from mjtest.test.syntax_tests import BasicSyntaxTest
from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult
from mjtest.util.shell import SigKill
from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict
from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict, decode
_LOG = logging.getLogger("exec_tests")
......@@ -76,7 +76,7 @@ class JavaExecTest(BasicSyntaxTest):
part = part.strip()
if len(part) > 0:
chars.append(int(part))
input_str = chars.decode()
input_str = decode(chars)
if not self._has_expected_output_file and not self._only_compile:
_, err, javac_rtcode = \
......@@ -85,7 +85,7 @@ class JavaExecTest(BasicSyntaxTest):
_LOG.error("File \"{}\" isn't valid Java".format(self.preprocessed_file))
test_result.incorrect_msg = "invalid java code, but output file missing"
test_result.set_error_code(javac_rtcode)
test_result.add_long_text("Javac error message", err.decode())
test_result.add_long_text("Javac error message", decode(err))
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
......@@ -97,16 +97,16 @@ class JavaExecTest(BasicSyntaxTest):
exp_out, err, java_rtcode = \
self.env.run_command("java", get_main_class_name(base_filename + ".java"), timeout=timeout, input_str=input_str)
if javac_rtcode != 0:
test_result.add_long_text("Java output: ", exp_out.decode())
test_result.add_long_text("Java output: ", decode(exp_out))
test_result.incorrect_msg = "java runtime error"
test_result.set_error_code(java_rtcode)
test_result.add_long_text("Java error message", err.decode())
test_result.add_long_text("Java error message", decode(err))
test_result.add_file("Source file", self.preprocessed_file)
if self._input_file:
test_result.add_file("Input file", self.file)
os.chdir(cwd)
return test_result
exp_out = exp_out.decode()
exp_out = decode(exp_out)
with open(self._prev_out_file, "w") as f:
f.write(exp_out)
......@@ -118,7 +118,6 @@ class JavaExecTest(BasicSyntaxTest):
and not self._only_compile:
with open(self._expected_output_file, "r") as f:
exp_out = f.read()
try:
out, err, rtcode = None, None, None
try:
......@@ -126,8 +125,8 @@ class JavaExecTest(BasicSyntaxTest):
if rtcode != 0:
test_result.incorrect_msg = "file can't be compiled"
test_result.set_error_code(rtcode)
test_result.add_long_text("Error output", err.decode())
test_result.add_long_text("Output", out.decode())
test_result.add_long_text("Error output", decode(err))
test_result.add_long_text("Output", decode(out))
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
......@@ -151,8 +150,8 @@ class JavaExecTest(BasicSyntaxTest):
if rtcode != 0:
test_result.incorrect_msg = "binary can't be run, non zero error code"
test_result.set_error_code(rtcode)
test_result.add_long_text("Error output", err.decode())
test_result.add_long_text("Output", out.decode())
test_result.add_long_text("Error output", decode(err))
test_result.add_long_text("Output", decode(out))
test_result.add_file("Source file", self.preprocessed_file)
if self._input_file:
test_result.add_file("Input file", self.file)
......@@ -169,7 +168,7 @@ class JavaExecTest(BasicSyntaxTest):
except:
os.chdir(cwd)
raise
out = out.decode()
out = decode(out)
if self.type == self.MODE and self.env.mode == self.MODE:
if exp_out.strip() != out.strip():
test_result.incorrect_msg = "incorrect output"
......@@ -185,7 +184,7 @@ class JavaExecTest(BasicSyntaxTest):
test_result.add_long_text("Output", out)
os.chdir(cwd)
return test_result
return BasicTestResult(self, rtcode, out, err.decode())
return BasicTestResult(self, rtcode, out, decode(err))
except SigKill as sig:
os.chdir(cwd)
assert False
......@@ -274,8 +273,8 @@ class JavaInfiniteLoopTest(BasicSyntaxTest):
if rtcode != 0:
test_result.incorrect_msg = "file can't be compiled"
test_result.set_error_code(rtcode)
test_result.add_long_text("Error output", err.decode())
test_result.add_long_text("Output", out.decode())
test_result.add_long_text("Error output", decode(err))
test_result.add_long_text("Output", decode(out))
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
......@@ -283,17 +282,17 @@ class JavaInfiniteLoopTest(BasicSyntaxTest):
if rtcode != 0:
test_result.incorrect_msg = "binary can't be run, non zero error code"
test_result.set_error_code(rtcode)
test_result.add_long_text("Error output", err.decode())
test_result.add_long_text("Output", out.decode())
test_result.add_long_text("Error output", decode(err))
test_result.add_long_text("Output", decode(out))
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
except SigKill as sig:
if sig.retcode == signal.SIGXCPU:
out, _, _, _ = self.env.run_command_till_timeout("./a.out", timeout=1)
test_result.add_long_text("Output", out.decode())
test_result.add_long_text("Output", decode(out))
if self._has_output_file:
out = out.decode().strip()
out = decode(out).strip()
exp_out = ""
with open(self._output_file, "r") as f:
exp_out = f.read().strip()
......@@ -361,8 +360,8 @@ class InvalidJavaExecTest(BasicSyntaxTest):
if rtcode != 0:
test_result.incorrect_msg = "file can't be compiled"
test_result.set_error_code(-rtcode)
test_result.add_long_text("Error output", err.decode())
test_result.add_long_text("Output", out.decode())
test_result.add_long_text("Error output", decode(err))
test_result.add_long_text("Output", decode(out))
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
......@@ -380,8 +379,8 @@ class InvalidJavaExecTest(BasicSyntaxTest):
if rtcode != 0:
test_result.incorrect_msg = "binary can't be run, non zero error code"
test_result.set_error_code(rtcode)
test_result.add_long_text("Error output", err.decode())
test_result.add_long_text("Output", out.decode())
test_result.add_long_text("Error output", decode(err))
test_result.add_long_text("Output", decode(out))
test_result.add_file("Source file", self.preprocessed_file)
os.chdir(cwd)
return test_result
......@@ -394,7 +393,7 @@ class InvalidJavaExecTest(BasicSyntaxTest):
except:
os.chdir(cwd)
raise
test_result = BasicTestResult(self, rtcode, out.decode(), err.decode())
test_result = BasicTestResult(self, rtcode, decode(out), decode(err))
test_result.require_error_string_in_error_case = False
return test_result
except SigKill as sig:
......
mjtest/test/syntax_tests.py
View file @ b2df624c
......@@ -3,6 +3,8 @@ from mjtest.environment import Environment, TestMode, TEST_MODES, get_test_dirna
from mjtest.test.tests import TestCase, BasicTestResult
from os import path
from mjtest.util.utils import decode
class BasicSyntaxTest(TestCase):
......@@ -35,7 +37,7 @@ class BasicSyntaxTest(TestCase):
def run(self) -> BasicTestResult:
out, err, rtcode = self.env.run_mj_command(self.MODE, self.preprocessed_file)
return BasicTestResult(self, rtcode, out.decode(), err.decode())
return BasicTestResult(self, rtcode, decode(out), decode(err))
TestCase.TEST_CASE_CLASSES[TestMode.syntax].append(BasicSyntaxTest)
......@@ -53,7 +55,7 @@ class JavaCompileTest(BasicSyntaxTest):
tmp_dir = self.env.create_pid_local_tmpdir()
_, self.javac_err, self.javac_rtcode = \
self.env.run_command("javac", path.relpath(preprocessed_file), "-d", tmp_dir, "-verbose")
self.javac_err = self.javac_err.decode() # type: str
self.javac_err = decode(self.javac_err) # type: str
self._should_succeed = self._is_file_syntactically_correct() if self.SYNTAX_TEST else self.javac_rtcode == 0
def _is_file_syntactically_correct(self):
......
mjtest/test/tests.py
View file @ b2df624c
from collections import namedtuple
import shutil
from typing import Optional, List, Tuple, T, Union, Dict
from typing import Optional, List, Tuple, Union, Dict
import collections
from mjtest.environment import Environment, TestMode, TEST_MODES, get_test_dirname
from os.path import join, exists, basename
......@@ -8,7 +8,7 @@ import logging
import os
import multiprocessing
from mjtest.util.parallelism import available_cpu_count
from mjtest.util.utils import cprint, colored
from mjtest.util.utils import cprint, colored, decode
from pprint import pprint
import shutil
import difflib
......@@ -568,8 +568,8 @@ class DiffTest(TestCase):
#else:
# _LOG.error("Expected output file for test case {}:{} is missing.".format(self.MODE, self.short_name()))
if self.type == self.MODE and self.env.mode == self.MODE:
return BasicDiffTestResult(self, rtcode, out.decode(), err.decode(), exp_out)
return BasicTestResult(self, rtcode, out.decode(), err.decode())
return BasicDiffTestResult(self, rtcode, decode(out), decode(err), exp_out)
return BasicTestResult(self, rtcode, decode(out), decode(err))
class LexerDiffTest(DiffTest):
......
mjtest/util/utils.py
View file @ b2df624c
......@@ -113,4 +113,11 @@ class InsertionTimeOrderedDict:
ret = InsertionTimeOrderedDict()
for item in items:
ret[key_func(item)] = item
return ret
\ No newline at end of file
return ret
def decode(arr: bytes) -> str:
"""
Decodes the passed byte array as UTF8 and handles invalid characters
"""
return arr.decode("utf-8", "backslashreplace")
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment