From caa946ba95155525392cb6df3c0a91fa4a6f8e4d Mon Sep 17 00:00:00 2001 From: Johannes Bechberger Date: Sun, 22 Jan 2017 12:31:21 +0100 Subject: [PATCH] Improve README and add invalid exec test case handler --- README.mdwn | 16 ++++++- mjtest/test/exec_tests.py | 87 ++++++++++++++++++++++++++++++++++++++- mjtest/test/tests.py | 6 ++- 3 files changed, 104 insertions(+), 5 deletions(-) diff --git a/README.mdwn b/README.mdwn index c011469..8255f29 100644 --- a/README.mdwn +++ b/README.mdwn @@ -113,9 +113,23 @@ Test types for the compile-firm and compile mode if a [test file].out exists: the content of this file
if no such file exists: the same as the execution of the file with java + + .[number].inputc + This a test case with an input. The input is interpreted as ASCII chars. It requires a corresponding + .input.java or .input.mj file that is executed with this input. + The rest is equivalent to normal test cases from the row above + + + .[number].input + This is an input test case whichs input file contains space seperated decimal numbers that are used as input data. + + + .invalid.mj .invalid.java + The test file should compile. The execution of the resulting binary should abort width a return code > 0 + -__All compile-firm and compile mode tests have to be semantically correct___ +__All compile-firm and compile mode tests have to be semantically correct__ __All bigger test cases (e.g. for benchmarking) should be placed into the `big` folder__ diff --git a/mjtest/test/exec_tests.py b/mjtest/test/exec_tests.py index 024c016..d2b22ff 100644 --- a/mjtest/test/exec_tests.py +++ b/mjtest/test/exec_tests.py @@ -24,7 +24,7 @@ class JavaExecTest(BasicSyntaxTest): """ FILE_ENDINGS = [".java", ".mj"] - INVALID_FILE_ENDINGS = [".inf.java", ".inf.mj", ".input.mj", ".input.java"] + INVALID_FILE_ENDINGS = [".inf.java", ".inf.mj", ".input.mj", ".input.java", ".invalid.java", ".invalid.mj"] OUTPUT_FILE_ENDING = ".out" MODE = TestMode.compile_firm INPUT_FILE_REGEXP = r'(\.[0-9]+)\.input(c?)$' @@ -289,7 +289,7 @@ class JavaInfiniteLoopTest(BasicSyntaxTest): os.chdir(cwd) return test_result else: - test_result.incorrect_msg = "sdfbinary can't be run: " + sig.name + test_result.incorrect_msg = "binary can't be run: " + sig.name test_result.set_error_code(sig.retcode) test_result.add_file("Source file", self.preprocessed_file) os.chdir(cwd) @@ -308,3 +308,86 @@ class JavaInfiniteLoopTest(BasicSyntaxTest): TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(JavaInfiniteLoopTest) + + +class InvalidJavaExecTest(BasicSyntaxTest): + """ + The compiled binary should fail to execute properly + """ + + FILE_ENDINGS = [".invalid.java", ".invalid.mj"] + MODE = TestMode.compile_firm + + def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str): + super().__init__(env, type, file, preprocessed_file, log_file_mode) + self._should_succeed = False + + def run(self) -> BasicDiffTestResult: + is_big_testcase = "big" in self.file + timeout = self.env.big_timeout if is_big_testcase else self.env.timeout + base_filename = path.basename(self.file).split(".")[0] + tmp_dir = self.env.create_pid_local_tmpdir() + shutil.copy(self.preprocessed_file, path.join(tmp_dir, base_filename + ".java")) + cwd = os.getcwd() + os.chdir(tmp_dir) + #print(base_filename, get_main_class_name(base_filename + ".java")) + + test_result = ExtensibleTestResult(self) + test_result.require_error_string_in_error_case = False + try: + out, err, rtcode = None, None, None + try: + out, err, rtcode = self.env.run_mj_command(self.MODE, base_filename + ".java", timeout=-1) + if rtcode != 0: + test_result.incorrect_msg = "file can't be compiled" + test_result.set_error_code(-rtcode) + test_result.add_long_text("Error output", err.decode()) + test_result.add_long_text("Output", out.decode()) + test_result.add_file("Source file", self.preprocessed_file) + os.chdir(cwd) + return test_result + except SigKill as sig: + test_result.incorrect_msg = "file can't be compiled: " + sig.name + test_result.set_error_code(sig.retcode) + test_result.add_file("Source file", self.preprocessed_file) + os.chdir(cwd) + return test_result + except: + os.chdir(cwd) + raise + try: + out, err, rtcode = self.env.run_command("./a.out", timeout=timeout) + if rtcode != 0: + test_result.incorrect_msg = "binary can't be run, non zero error code" + test_result.set_error_code(rtcode) + test_result.add_long_text("Error output", err.decode()) + test_result.add_long_text("Output", out.decode()) + test_result.add_file("Source file", self.preprocessed_file) + os.chdir(cwd) + return test_result + except SigKill as sig: + test_result.incorrect_msg = "binary can't be run: " + sig.name.strip() + test_result.set_error_code(sig.retcode) + test_result.add_file("Source file", self.preprocessed_file) + os.chdir(cwd) + return test_result + except: + os.chdir(cwd) + raise + test_result = BasicTestResult(self, rtcode, out.decode(), err.decode()) + test_result.require_error_string_in_error_case = False + return test_result + except SigKill as sig: + os.chdir(cwd) + assert False + except BaseException: + os.chdir(cwd) + raise + + +TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(InvalidJavaExecTest) + +class InvalidJavaCompileExecTest(InvalidJavaExecTest): + MODE = TestMode.compile + +TestCase.TEST_CASE_CLASSES[TestMode.compile].append(InvalidJavaExecTest) diff --git a/mjtest/test/tests.py b/mjtest/test/tests.py index f92a99d..1710eef 100644 --- a/mjtest/test/tests.py +++ b/mjtest/test/tests.py @@ -439,6 +439,7 @@ class BasicTestResult(TestResult): if error_output: self.add_additional_text("Error output", error_output) self.has_succeeded = error_code == 0 + self.require_error_string_in_error_case = True def succeeded(self): return self.has_succeeded @@ -447,13 +448,14 @@ class BasicTestResult(TestResult): if self.succeeded(): return super().is_correct() else: - return super().is_correct() and self._contains_error_str + return super().is_correct() and (self._contains_error_str or not self.require_error_string_in_error_case) def short_message(self) -> str: if self.is_correct(): return "correct" else: - if not self.succeeded() and not self.test_case.should_succeed() and not self._contains_error_str: + if not self.succeeded() and not self.test_case.should_succeed() and not self._contains_error_str and \ + not self.require_error_string_in_error_case: return "the error output doesn't contain the word \"error\"" return self._incorrect_msg -- 2.26.2