Update mjtest and mjtest-tests

Merge remote-tracking branch 'gitlab-original/master'

README.md

@@ -23,8 +23,10 @@ The test cases are divided in 6 'modes':
    them.
 - __ast__: Test cases that check the generated ast by using the pretty printing functionality.
 - __semantic__: Test cases that check semantic checking of MiniJava programs
-- __compile-firm__: Test cases that check the correct compilation and execution of MiniJava programs with the libfirm backend.
-- __compile__: Test cases that check the correct compilation and execution of MiniJava programs with the self implemented backend.
+- __compile-firm-only__: Test cases that check the compilation of MiniJava programs with the libfirm backend.
+- __compile-firm__ (alias `exec-firm`): Test cases that check the correct compilation and execution of MiniJava programs with the libfirm backend.
+- __compile-only__: Test cases that check the compilation MiniJava programs with the self implemented backend.
+- __compile__ (alias `exec`): Test cases that check the correct compilation and execution of MiniJava programs with the self implemented backend.
 
 The test different test cases for each mode are located in a folder with the same name. 
 Except the compile-firm test cases which are located in the `exec` folder.
@@ -103,6 +105,19 @@ Test types for the semantic mode
 
 __All semantic mode tests have to be syntactically correct__
 
+Test types for the compile-firm-only and compile-only mode
+----------------------------------------------------------
+
+<table>
+<tr><th>File ending(s) of test cases</th><th>Expected behaviour to complete a test of this type</th></tr>
+<tr>
+    <td><code>.valid.mj</code> <code>.mj</code> <code>.valid.java</code> <code>.java</code></td>
+    <td>Return code is <code>0</code> and the compiler produces an `a.out` file, i.e. the MiniJava code can be compiled</td>
+</tr>
+</table>
+
+__All valid semantic test-cases are also used, the non-semantic compile-(firm)-only
+should be placed into the `compile-only` folder__
 
 Test types for the compile-firm and compile mode
 ------------------------------------------------
@@ -112,7 +127,7 @@ Test types for the compile-firm and compile mode
 <tr>
     <td><code>.java</code> <code>.mj</code></td>
     <td>The test file should compile. The resulting binary should be runnable and its output should be:<br/>
-    if a <code>[test file].out</code> exists: the content of this file<br/>
+    if a <code>[input file].out</code> (for test cases with input) or <code>[test file].out</code> (else) exists: the content of this file<br/>
     if no such file exists: the same as the execution of the file with <code>java</code></td>
 </tr>
 <tr>
@@ -171,12 +186,13 @@ usage: mjt.py [-h] [--only_incorrect_tests] [--all_exec_tests]
               [--produce_no_reports] [--produce_all_reports] [--parallel]
               [--output_no_incorrect_reports] [--color] [--ci_testing]
               [--log_level LOG_LEVEL]
-              {all,lexer,syntax,ast,semantic,compile-firm,compile} MJ_RUN
+              {all,lexer,syntax,ast,semantic,compile-firm-only,compile-only,
+               compile-firm,compile,exec,exec-firm} MJ_RUN
 
 MiniJava test runner
 
 positional arguments:
-  {all,lexer,syntax,ast,semantic,compile-firm,exec}
+  {all,lexer,syntax,ast,semantic,compile-firm-only,compile-only,compile-firm,exec,exec-firm}
                         What do you want to test?
   MJ_RUN                Command to run your MiniJava implementation, e.g.
                         `mj/run`, can be omitted by assigning the environment

mjtest/cli.py

@@ -27,7 +27,7 @@ class LogLevelChoices(argparse.Action):
 if True:#__name__ == '__main__':
     parser = argparse.ArgumentParser(description="MiniJava test runner", add_help=True)
     parser.add_argument("mode",
-                        choices=["all"] + TEST_MODES,
+                        choices=["all"] + TEST_MODES + ["exec-firm"],
                         help="What do you want to test?")
     if os.getenv("MJ_RUN", None) is None:
         parser.add_argument("mj_run",
@@ -84,9 +84,16 @@ if True:#__name__ == '__main__':
         if ret is not None:
             failed += ret.failed
             count += ret.count
+    if args["mode"] in ["exec", "exec-firm"]:
+        new_mode = args["mode"].replace("exec", "compile")
+        print(colored("\"{}\" is just an alias for \"{}\", using the latter instead"
+                        .format(args["mode"], new_mode),
+                      "yellow"))
+        args["mode"] = new_mode
     if args["mode"] == "all":
         report_subdir = datetime.now().strftime("%d-%m-%y_%H-%M-%S")
         for mode in [TestMode.lexer, TestMode.syntax, TestMode.ast, TestMode.semantic,
+                     TestMode.compile_firm_only, TestMode.compile_only,
                      TestMode.compile_firm, TestMode.compile]:
             args["all_exec_tests"] = True
             args["report_subdir"] = report_subdir + "_" + mode

mjtest/environment.py

@@ -34,6 +34,10 @@ class TestMode:
 
     compile = "compile"
 
+    compile_firm_only = "compile-firm-only"
+
+    compile_only = "compile-only"
+
     exec = "exec"
 
     bench = "bench"
@@ -42,12 +46,14 @@ class TestMode:
         ast: [syntax],
         compile_firm: [exec],
         compile: [exec],
+        compile_firm_only: [semantic, compile_only],
+        compile_only: [semantic],
         bench: [exec]
     }
 
 """ All 'success' tests of the n.th mode can used as 'success' tests for the n-1.th mode"""
-TEST_MODES = [TestMode.lexer, TestMode.syntax, TestMode.ast, TestMode.semantic, TestMode.compile_firm,
-              TestMode.exec, TestMode.bench, TestMode.compile]
+TEST_MODES = [TestMode.lexer, TestMode.syntax, TestMode.ast, TestMode.semantic, TestMode.compile_firm_only,
+              TestMode.compile_only, TestMode.compile_firm, TestMode.exec, TestMode.bench, TestMode.compile]
 
 
 def get_test_dirname(mode: str) -> str:
@@ -163,8 +169,10 @@ class Environment:
             TestMode.syntax: "--parsetest",
             TestMode.ast: "--print-ast",
             TestMode.semantic: "--check",
+            TestMode.compile_only: None,
+            TestMode.compile_firm_only: "--compile-firm",
             TestMode.compile_firm: "--compile-firm",
-            TestMode.compile: None
+            TestMode.compile: None,
         }[mode]
         cmd = [self.mj_run_cmd]
         if mode_flag:
@@ -172,13 +180,13 @@ class Environment:
         cmd.extend(list(args))
         return execute(cmd, timeout=timeout or self.timeout)
 
-    def run_command(self, cmd: str, *args: Tuple[str], timeout: float = None, input_str: Optional[str] = None) -> Tuple[bytes, bytes, int]:
+    def run_command(self, cmd: str, *args: Tuple[str], timeout: float = None, input_bytes: Optional[bytes] = None) -> Tuple[bytes, bytes, int]:
         """
         Execute the passend command with its arguments
 
         :return: (out, err, return code)
         """
-        return execute([cmd] + list(args), timeout=timeout or self.timeout, input_str=input_str)
+        return execute([cmd] + list(args), timeout=timeout or self.timeout, input_bytes=input_bytes)
 
     def has_to_preprocess(self, file: str) -> bool:
         return os.path.relpath(file, self.test_dir).startswith("exec")
@@ -241,4 +249,4 @@ class Environment:
             timer.cancel()
         else:
             stdout, stderr = proc.communicate()
-        return stdout, stderr, proc.returncode, timeouted
\ No newline at end of file
+        return stdout, stderr, proc.returncode, timeouted

mjtest/test/ast_tests.py

@@ -7,6 +7,8 @@ from mjtest.test.syntax_tests import BasicSyntaxTest
 from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult
 from os import path
 
+from mjtest.util.utils import decode
+
 _LOG = logging.getLogger("tests")
 
 
@@ -59,6 +61,6 @@ class ASTPrettyPrintTest(BasicSyntaxTest):
         with open(output_file, "wb") as f:
             f.write(out)
             f.flush()
-        return rtcode, out.decode(), err.decode()
+        return rtcode, decode(out), decode(err)
 
 TestCase.TEST_CASE_CLASSES["ast"].append(ASTPrettyPrintTest)

mjtest/test/bench.py

@@ -16,7 +16,7 @@ from mjtest.environment import TestMode, Environment
 from mjtest.test.syntax_tests import BasicSyntaxTest
 from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult
 from mjtest.util.shell import SigKill
-from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict
+from mjtest.util.utils import get_main_class_name, InsertionTimeOrderedDict, decode
 
 _LOG = logging.getLogger("bench_tests")
 
@@ -49,8 +49,8 @@ class BenchExecTest(BasicSyntaxTest):
     INVALID_FILE_ENDINGS = [".inf.java", ".inf.mj"]
     MODE = TestMode.compile_firm
 
-    def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str):
-        super().__init__(env, type, file, preprocessed_file)
+    def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str, log_file_mode: str = ""):
+        super().__init__(env, type, file, preprocessed_file, log_file_mode)
         self._should_succeed = True
 
     def _bench_command(self, cmd: str, *args: Tuple[str]) -> _RunResult:
@@ -85,7 +85,7 @@ class BenchExecTest(BasicSyntaxTest):
                     _LOG.error("File \"{}\" isn't valid Java".format(self.preprocessed_file))
                     test_result.incorrect_msg = "invalid java code, but output file missing"
                     test_result.set_error_code(javac_rtcode)
-                    test_result.add_long_text("Javac error message", err.decode())
+                    test_result.add_long_text("Javac error message", decode(err))
                     test_result.add_file("Source file", self.preprocessed_file)
                     os.chdir(cwd)
                     return test_result
@@ -102,8 +102,8 @@ class BenchExecTest(BasicSyntaxTest):
                     if rtcode != 0:
                         test_result.incorrect_msg = "file can't be compiled"
                         test_result.set_error_code(rtcode)
-                        test_result.add_long_text("Error output", err.decode())
-                        test_result.add_long_text("Output", out.decode())
+                        test_result.add_long_text("Error output", decode(err))
+                        test_result.add_long_text("Output", decode(out))
                         test_result.add_file("Source file", self.preprocessed_file)
                         os.chdir(cwd)
                         return test_result

mjtest/test/syntax_tests.py

@@ -3,6 +3,8 @@ from mjtest.environment import Environment, TestMode, TEST_MODES, get_test_dirna
 from mjtest.test.tests import TestCase, BasicTestResult
 from os import path
 
+from mjtest.util.utils import decode
+
 
 class BasicSyntaxTest(TestCase):
 
@@ -35,7 +37,7 @@ class BasicSyntaxTest(TestCase):
 
     def run(self) -> BasicTestResult:
         out, err, rtcode = self.env.run_mj_command(self.MODE, self.preprocessed_file)
-        return BasicTestResult(self, rtcode, out.decode(), err.decode())
+        return BasicTestResult(self, rtcode, decode(out), decode(err))
 
 TestCase.TEST_CASE_CLASSES[TestMode.syntax].append(BasicSyntaxTest)
 
@@ -53,7 +55,7 @@ class JavaCompileTest(BasicSyntaxTest):
         tmp_dir = self.env.create_pid_local_tmpdir()
         _, self.javac_err, self.javac_rtcode = \
             self.env.run_command("javac", path.relpath(preprocessed_file), "-d", tmp_dir, "-verbose")
-        self.javac_err = self.javac_err.decode()  # type: str
+        self.javac_err = decode(self.javac_err)  # type: str
         self._should_succeed = self._is_file_syntactically_correct() if self.SYNTAX_TEST else self.javac_rtcode == 0
 
     def _is_file_syntactically_correct(self):

mjtest/test/tests.py

 from collections import namedtuple
 import shutil
-from typing import Optional, List, Tuple, T, Union, Dict
+from typing import Optional, List, Tuple, Union, Dict
 import collections
 from mjtest.environment import Environment, TestMode, TEST_MODES, get_test_dirname
 from os.path import join, exists, basename
@@ -8,7 +8,7 @@ import logging
 import os
 import multiprocessing
 from mjtest.util.parallelism import available_cpu_count
-from mjtest.util.utils import cprint, colored
+from mjtest.util.utils import cprint, colored, decode
 from pprint import pprint
 import shutil
 import difflib
@@ -50,7 +50,7 @@ class TestSuite:
         correct_test_cases = set()
         log_file = self._log_file_for_type(mode)
         if exists(log_file):
-            with open(log_file) as f:
+            with open(log_file, errors="backslashreplace") as f:
                 correct_test_cases = set()
                 for t in f.readlines():
                     t = t.strip()
@@ -223,7 +223,7 @@ class TestSuite:
                     os.mkdir(os.path.dirname(log_file))
                 except IOError:
                     pass
-                with open(log_file, "w") as f:
+                with open(log_file, "w", errors="backslashreplace") as f:
                     f.write("\n".join(self.correct_test_cases[mode]))
             except IOError as e:
                 _LOG.exception("Caught i/o error while storing {}".format(log_file))
@@ -314,7 +314,7 @@ class TestResult:
 
     def store_at(self, file: str):
         os.makedirs(os.path.dirname(file), exist_ok=True)
-        with open(file, "w") as f:
+        with open(file, "w", errors="backslashreplace") as f:
             print(self.long_message(), file=f)
 
     def short_message(self) -> str:
@@ -348,8 +348,8 @@ class ExtensibleTestResult(TestResult):
         self.messages.append(TestResultMessage(title, content, multiline=False, with_line_numbers=False))
 
     def add_file(self, title: str, file_name: str, with_line_numbers: bool = True):
-        with open(file_name, "r") as f:
-            file_content = os.linesep.join([line.rstrip() for line in f])
+        with open(file_name, "r", errors="backslashreplace") as f:
+            file_content = os.linesep.join([line.rstrip() for line in f.readlines()])
         self.add_long_text(title, file_content, with_line_numbers)
 
     def succeeded(self):
@@ -461,7 +461,7 @@ class BasicTestResult(TestResult):
 
     def long_message(self) -> str:
         file_content = []
-        with open(self.test_case.preprocessed_file, "r") as f:
+        with open(self.test_case.preprocessed_file, "r", errors="backslashreplace") as f:
             file_content = [line.rstrip() for line in f]
         others = []
         for title, content, long_text in self.other_texts:
@@ -563,13 +563,13 @@ class DiffTest(TestCase):
         exp_out = ""
         if rtcode == 0 and self.should_succeed():
             if self._has_expected_output_file and self.type == self.MODE and self.env.mode == self.MODE:
-                with open(self._expected_output_file, "r") as f:
+                with open(self._expected_output_file, "r", errors="backslashreplace") as f:
                     exp_out = f.read()
             #else:
             #    _LOG.error("Expected output file for test case {}:{} is missing.".format(self.MODE, self.short_name()))
         if self.type == self.MODE and self.env.mode == self.MODE:
-            return BasicDiffTestResult(self, rtcode, out.decode(), err.decode(), exp_out)
-        return BasicTestResult(self, rtcode, out.decode(), err.decode())
+            return BasicDiffTestResult(self, rtcode, decode(out), decode(err), exp_out)
+        return BasicTestResult(self, rtcode, decode(out), decode(err))
 
 
 class LexerDiffTest(DiffTest):
@@ -582,4 +582,4 @@ import mjtest.test.syntax_tests
 import mjtest.test.ast_tests
 import mjtest.test.semantic_tests
 import mjtest.test.exec_tests
-import mjtest.test.bench
\ No newline at end of file
+import mjtest.test.bench

mjtest/util/shell.py

@@ -16,7 +16,7 @@ try:
 except ImportError:
     has_resource_module = False
     pass
-	
+
 import sys
 import signal
 import threading
@@ -46,14 +46,14 @@ def _lower_rlimit(res, limit):
 
 
 class _Execute(object):
-    def __init__(self, cmd, timeout, env, rlimit, input_str):
+    def __init__(self, cmd, timeout, env, rlimit, input_bytes):
         self.cmd       = cmd
         self.timeout   = timeout
         self.env       = env
         self.proc      = None
         self.exception = None
         self.rlimit    = rlimit
-        self.input_str = input_str
+        self.input_bytes = input_bytes
         MB = 1024 * 1024
         if not 'RLIMIT_CORE' in rlimit:
             rlimit['RLIMIT_CORE'] = 0
@@ -80,14 +80,13 @@ class _Execute(object):
             self.proc = subprocess.Popen(self.cmd,
                                          stdout=subprocess.PIPE,
                                          stderr=subprocess.PIPE,
-                                         stdin=subprocess.PIPE if self.input_str else None,
+                                         stdin=subprocess.PIPE if self.input_bytes else None,
                                          env=self.env,
                                          shell=True,
                                          **prexec_args)
-            #if self.input_str:
-            #    self.proc.stdin.write(self.input_str.decode)
-            input_bytes = self.input_str.encode() if self.input_str else None
-            x = self.proc.communicate(input=input_bytes, timeout=self.timeout if self.timeout > 0.0 else None)
+            #if self.input_bytes:
+            #    self.proc.stdin.write(self.input_bytes.decode)
+            x = self.proc.communicate(input=self.input_bytes, timeout=self.timeout if self.timeout > 0.0 else None)
             self.out, self.err = x
             self.returncode = self.proc.returncode
         except subprocess.TimeoutExpired as t:
@@ -110,7 +109,7 @@ class _Execute(object):
             raise SigKill(-self.returncode, _EXIT_CODES[self.returncode] + ": " + os.strerror(-self.returncode))
         return (self.out, self.err, self.returncode)
 
-def execute(cmd, env=None, timeout=0, rlimit=None, propagate_sigint=True, input_str = None):
+def execute(cmd, env=None, timeout=0, rlimit=None, propagate_sigint=True, input_bytes = None):
     """Execute a command and return stderr and stdout data"""
     if not rlimit:
         rlimit = dict()
@@ -120,7 +119,7 @@ def execute(cmd, env=None, timeout=0, rlimit=None, propagate_sigint=True, input_
     else:
         #cmd = shlex.split(cmd[0]) + cmd[1:]
         cmd = " ".join(shlex.quote(c) for c in cmd)
-    exc = _Execute(cmd, timeout, env, rlimit, input_str)
+    exc = _Execute(cmd, timeout, env, rlimit, input_bytes)
     (out, err, returncode) = exc.run()
     if returncode == -signal.SIGINT:
         raise KeyboardInterrupt

mjtest/util/utils.py

@@ -47,7 +47,7 @@ def cprint(text: str, *args, **kwargs):
 
 def get_main_class_name(file: str) -> Optional[str]:
     current_class = None
-    with open(file, "r") as f:
+    with open(file, "r", errors="backslashreplace") as f:
         for line in f:
             if line.startswith("class ") or line.startswith("/*public*/ class "):
                 match = re.search("[A-Za-z_0-9]+", line.replace("class ", "").replace("/*public*/", ""))
@@ -113,4 +113,11 @@ class InsertionTimeOrderedDict:
         ret = InsertionTimeOrderedDict()
         for item in items:
             ret[key_func(item)] = item
-        return ret
\ No newline at end of file
+        return ret
+
+
+def decode(arr: bytes) -> str:
+    """
+    Decodes the passed byte array as UTF8 and handles invalid characters
+    """
+    return arr.decode("utf-8", "backslashreplace")
\ No newline at end of file

preproc/preproc/preprocessor.py

@@ -48,7 +48,7 @@ class PreProcessor:
         def add_commented(line: str):
             middle_lines.append("/*{}*/".format(line))
 
-        with open(file, "r") as f:
+        with open(file, "r", errors="backslashreplace") as f:
             for line in f:
                 line = line.rstrip()
                 if self._import_regexp.match(line):
@@ -108,7 +108,7 @@ class PreProcessor:
                 print()
                 print(text)
         else:
-            with open(self.dst_file, "w") as f:
+            with open(self.dst_file, "w", errors="backslashreplace") as f:
                 for text in reversed(self.imported_strs):
                     f.write(text)
                     f.write("\n\n")
@@ -122,7 +122,7 @@ def is_importable_file(file: str) -> bool:
     has_package = False
     has_public_class = False
     has_main_method = False
-    with open(file, "r") as f:
+    with open(file, "r", errors="backslashreplace") as f:
         for line in f:
             if line.startswith("package "):
                 has_package = True

tests @ 9e7b9291

-Subproject commit 541910c3730523949b1b309e457f9cfdff7808b3
+Subproject commit 9e7b9291db05fa7049738b435b2f818ed2aa29e1