Add simple import statement preprocessor

.gitignore

@@ -9,5 +9,5 @@ build/
 .idea
 .mjtest*
 mjtest.egg-info
-
+.preprocessed
 .DS_Store

mjtest/environment.py

@@ -8,7 +8,9 @@ import time
 from mjtest.util.shell import execute
 from mjtest.util.utils import get_mjtest_basedir
 from typing import Tuple, List
+from preproc.preproc.preprocessor import PreProcessor, PreProcessorError, is_importable_file
 
+_LOG = logging.getLogger("env")
 
 class TestMode:
 
@@ -86,6 +88,7 @@ class Environment:
         self.produce_all_reports = produce_all_reports
         self.ci_testing = ci_testing
         self._tmp_file_ctr = 0
+        self._already_preprocessed_files = set()
 
     def create_tmpfile(self) -> str:
         self._tmp_file_ctr += 1
@@ -124,3 +127,45 @@ class Environment:
         :return: (out, err, return code)
         """
         return execute([cmd] + list(args), timeout=self.timeout)
+
+    def has_to_preprocess(self, file: str) -> bool:
+        return os.path.relpath(file, self.test_dir).startswith("exec")
+
+    def is_lib_file(self, file: str) -> bool:
+        return is_importable_file(file)
+
+    def preprocess(self, file: str) -> str:
+        """
+        Pre process the passed file if needed and return the resulting file
+        """
+        if not self.has_to_preprocess(file):
+            return file
+        if ".preprocessed" in os.path.relpath(file, self.test_dir):
+            return file
+
+        import_base_dir = os.path.join(self.test_dir, os.path.relpath(file, self.test_dir).lstrip(os.sep).split(os.sep)[0])
+        dst_dir = os.path.join(import_base_dir, ".preprocessed", os.path.split(os.path.relpath(file, import_base_dir))[0])
+        dst_file = os.path.join(dst_dir, os.path.basename(file))
+        if dst_file.endswith(".mj"):
+            dst_file = dst_file.replace(".mj", ".java")
+        if dst_file in self._already_preprocessed_files:
+            return dst_file
+        self._already_preprocessed_files.add(dst_file)
+
+        if os.path.exists(dst_file) and os.path.isfile(dst_file) and os.path.getmtime(file) < os.path.getmtime(dst_file):
+            _LOG.debug("File '{}' already exists in a pre processed form".format(os.path.relpath(file)))
+            return dst_file
+        cur = os.path.split(dst_file)[0]
+        while not os.path.exists(cur):
+            os.mkdir(cur)
+            cur = os.path.split(cur)[0]
+
+        if not os.path.exists(dst_dir):
+            os.mkdir(dst_dir)
+        try:
+            PreProcessor(file, import_base_dir, dst_file).preprocess()
+        except:
+            _LOG.exception("Pre processing file '{}'".format(file))
+            raise
+        return dst_file
+

mjtest/test/ast_tests.py

@@ -15,14 +15,14 @@ class ASTPrettyPrintTest(BasicSyntaxTest):
     FILE_ENDINGS = [".mj", ".valid.mj"]
     INVALID_FILE_ENDINGS = [".invalid.mj"]
 
-    def __init__(self, env: Environment, type: str, file: str):
-        super().__init__(env, type, file)
+    def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str):
+        super().__init__(env, type, file, preprocessed_file)
         if type != TestMode.ast and TEST_MODES.index(TestMode.ast) < TEST_MODES.index(type):
             self._should_succeed = True
 
     def run(self) -> BasicTestResult:
         tmp_file = self.env.create_tmpfile()
-        rtcode, out, err = self._pretty_print(self.file, tmp_file)
+        rtcode, out, err = self._pretty_print(self.preprocessed_file, tmp_file)
         if rtcode > 0:
             os.remove(tmp_file)
             return BasicTestResult(self, rtcode, out, err)

mjtest/test/syntax_tests.py

@@ -9,8 +9,8 @@ class BasicSyntaxTest(TestCase):
     FILE_ENDINGS = [".invalid.mj", ".valid.mj", ".mj", ".invalid.java", ".java"]
     MODE = TestMode.syntax
 
-    def __init__(self, env: Environment, type: str, file: str):
-        super().__init__(env, type, file)
+    def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str):
+        super().__init__(env, type, file, preprocessed_file)
         if type != self.MODE and TEST_MODES.index(type) > TEST_MODES.index(self.MODE):
             self._should_succeed = True
         else:
@@ -23,7 +23,7 @@ class BasicSyntaxTest(TestCase):
         return path.basename(self.file)
 
     def run(self) -> BasicTestResult:
-        out, err, rtcode = self.env.run_mj_command(self.MODE, self.file)
+        out, err, rtcode = self.env.run_mj_command(self.MODE, self.preprocessed_file)
         return BasicTestResult(self, rtcode, out.decode(), err.decode())
 
 TestCase.TEST_CASE_CLASSES[TestMode.syntax].append(BasicSyntaxTest)
@@ -37,11 +37,11 @@ class JavaCompileTest(BasicSyntaxTest):
     FILE_ENDINGS = [".java"]
     SYNTAX_TEST = True
 
-    def __init__(self, env: Environment, type: str, file: str):
-        super().__init__(env, type, file)
+    def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str):
+        super().__init__(env, type, file, preprocessed_file)
         tmp_dir = self.env.create_tmpdir()
         _, self.javac_err, self.javac_rtcode = \
-            self.env.run_command("javac", path.relpath(file), "-d", tmp_dir, "-verbose")
+            self.env.run_command("javac", path.relpath(preprocessed_file), "-d", tmp_dir, "-verbose")
         self.javac_err = self.javac_err.decode()  # type: str
         shutil.rmtree(tmp_dir, ignore_errors=True)
         self._should_succeed = self._is_file_syntactically_correct() if self.SYNTAX_TEST else self.javac_rtcode == 0

mjtest/test/tests.py

@@ -67,6 +67,8 @@ class TestSuite:
             base = os.path.relpath(root, dir)
             if dir == root:
                 file_names.extend(files)
+            elif base == ".preprocessed":
+                continue
             else:
                 file_names.extend(join(base, file) for file in files)
         for file in sorted(file_names):
@@ -75,7 +77,12 @@ class TestSuite:
             elif self.env.only_incorrect_tests and file in correct_test_cases:
                 _LOG.info("Skip file {} as its test case was executed correctly the last run")
             else:
-                test_case = TestCase.create_from_file(self.env, m, join(dir, file))
+                file_path = join(dir, file)
+                if self.env.has_to_preprocess(file_path) and self.env.is_lib_file(file_path):
+                    _LOG.debug("Skip lib file '{}'".format(file))
+                    continue
+                preprocessed = self.env.preprocess(join(dir, file))
+                test_case = TestCase.create_from_file(self.env, m, join(dir, file), preprocessed)
                 if not test_case:
                     pass
                 elif not test_case.can_run():
@@ -211,10 +218,11 @@ class TestCase:
     FILE_ENDINGS = []
     INVALID_FILE_ENDINGS = []
 
-    def __init__(self, env: Environment, type: str, file: str):
+    def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str):
         self.env = env
         self.type = type
         self.file = file
+        self.preprocessed_file = preprocessed_file
 
     def should_succeed(self) -> bool:
         raise NotImplementedError()
@@ -234,9 +242,9 @@ class TestCase:
         raise NotImplementedError()
 
     @classmethod
-    def create_from_file(cls, env: Environment, mode: str, file: str) -> Optional['TestCase']:
+    def create_from_file(cls, env: Environment, mode: str, file: str, preprocessed_file: str) -> Optional['TestCase']:
         if cls.has_valid_file_ending(env.mode, file):
-            return cls._test_case_class_for_file(env.mode, file)(env, mode, file)
+            return cls._test_case_class_for_file(env.mode, file)(env, mode, file, preprocessed_file)
         return None
 
     def name(self):
@@ -313,7 +321,7 @@ class BasicTestResult(TestResult):
 
     def long_message(self) -> str:
         file_content = []
-        with open(self.test_case.file, "r") as f:
+        with open(self.test_case.preprocessed_file, "r") as f:
             file_content = [line.rstrip() for line in f]
         others = []
         for title, content, long_text in self.other_texts:
@@ -394,8 +402,8 @@ class DiffTest(TestCase):
     OUTPUT_FILE_ENDING = ".out"
     MODE = TestMode.ast
 
-    def __init__(self, env: Environment, type: str, file: str):
-        super().__init__(env, type, file)
+    def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str):
+        super().__init__(env, type, file, preprocessed_file)
         self._should_succeed = not file.endswith(".invalid.mj")
         self._expected_output_file = file + self.OUTPUT_FILE_ENDING
         self._has_expected_output_file = exists(self._expected_output_file)

preproc/README.md

+MiniJava preprocessor
+=====================
+
+Why?
+----
+It allows to create a library of utility classes and reduces the overall code completion in the test file.
+As standard java import and package statements are used, the unprocessed files are valid java files (with all the advantages in the development process).
+
+Usage
+-----
+
+```
+usage: process.py [-h] [--log_level LOG_LEVEL] FILE MJ_IMPORT_DIR DEST
+
+MiniJava preprocessor
+
+positional arguments:
+  FILE                  File to pre process
+  MJ_IMPORT_DIR         Base directory for imports. Can be omitted by
+                        assigning the environment variable MJ_IMPORT_DIR.
+  DEST                  Destination file, '-' if it should be printed on std
+                        out.
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --log_level LOG_LEVEL
+                        Logging level (error, warn, info or debug)
+```
+
+Importable file
+---------------
+
+A importable file "X.java" contains at least:
+
+- a `package` statement
+- the string `public class X` somewhere but not in a comment (this isn't checked automatically)
+
+It can import other importable files.
+
+Check for "normal" mj files via `./is_normal_mj_file.py FILE`.
+
+What `import x.y.Math` does:
+----------------------------
+
+1. Find file `MJ_IMPORT_DIR/x/y/Math.java` and check that it's an importable file
+2. Check that the file isn't already imported (ignore it otherwise) and abort if two imported classes are in different packages but have the same name.
+3. Store the file in a list of files that should be imported
+4. Start with 1. for each `import` statement in the file that isn't already satisfied
+5. For each file that should be imported:
+	1. Read the file
+	2. Remove all strings that match `package.*;`
+	3. Replace all `public class` strings with just `class`
+	4. Insert it at the end of the original source file
+6. Output the modified source file
\ No newline at end of file

preproc/is_normal_mj_file.py

+#!/usr/bin/env python3
+import argparse
+
+import sys
+from os.path import dirname, realpath
+p = dirname(realpath(__file__))
+sys.path.append(p)
+from preproc.preprocessor import PreProcessor, PreProcessorError, is_importable_file
+import logging
+
+parser = argparse.ArgumentParser(description="Is normal MiniJava file?", add_help=True)
+parser.add_argument("file", metavar="FILE", help="File to check")
+file = parser.parse_args().file
+
+try:
+    ret = is_importable_file(file)
+    sys.exit(1 if ret else 0)
+except PreProcessorError as ex:
+    logging.exception("")
+    sys.exit(1)
\ No newline at end of file

preproc/preproc/cli.py

+import logging
+import os
+import sys
+import argparse
+from typing import Dict
+from datetime import datetime
+
+from preproc.preprocessor import PreProcessor, PreProcessorError
+
+_LOG = logging.getLogger("preprocessor")
+
+# adapted from http://stackoverflow.com/a/8527629
+class LogLevelChoices(argparse.Action):
+    CHOICES = ["error", "warn", "info", "debug"]
+    def __call__(self, parser, namespace, value, option_string=None):
+        if value:
+            if value not in self.CHOICES:
+                message = ("invalid choice: {0!r} (choose from {1})"
+                           .format(value,
+                                   ', '.join([repr(action)
+                                              for action in self.CHOICES])))
+
+                raise argparse.ArgumentError(self, message)
+            setattr(namespace, self.dest, value)
+
+def run():
+    parser = argparse.ArgumentParser(description="MiniJava preprocessor", add_help=True)
+    parser.add_argument("src_file", metavar="FILE", help="File to pre process")
+    if os.getenv("MJ_IMPORT_DIR", None) is None:
+        parser.add_argument("import_base_dir",
+                            metavar="MJ_IMPORT_DIR",
+                            help="Base directory for imports. "
+                                 "Can be omitted by assigning the environment variable MJ_IMPORT_DIR.")
+    parser.add_argument("dst_file", metavar="DEST", help="Destination file, '-' if it should be printed on std out.")
+
+    parser.add_argument("--log_level", action=LogLevelChoices, default="warn", const="log_level",
+                        help="Logging level (error, warn, info or debug)")
+    args = vars(parser.parse_args())
+
+    if os.getenv("MJ_IMPORT_DIR", None) is not None:
+            args["import_base_dir"] = os.getenv("MJ_IMPORT_DIR")
+
+    LOG_LEVELS = {
+        "info": logging.INFO,
+        "error": logging.ERROR,
+        "warn": logging.WARN,
+        "debug": logging.DEBUG
+    }
+
+    logging.basicConfig(level=LOG_LEVELS[args["log_level"]])
+
+    try:
+        PreProcessor(args["src_file"], args["import_base_dir"], args["dst_file"]).preprocess()
+    except PreProcessorError as err:
+        _LOG.error(str(err))
+        return 1
\ No newline at end of file

preproc/preproc/preprocessor.py

+from collections import defaultdict
+import logging
+import os
+from os.path import relpath
+from pprint import pprint
+import re
+import sys
+
+_LOG = logging.getLogger("preprocessor")
+
+class PreProcessorError(BaseException):
+
+    def __init__(self, msg):
+        super().__init__(msg)
+
+class PreProcessor:
+
+    def __init__(self, src_file: str, import_base_dir: str, dst_file: str):
+        self.src_file = src_file
+        self.import_base_dir = import_base_dir
+        self.dst_file = dst_file
+        self.imported_strs = []
+        self._already_imported_classes = {}  # name -> full_name
+        self._import_regexp = re.compile("import [A-Za-z.]+;")
+        self._imported_class_regexp = re.compile("[A-Za-z.]+;")
+        self._imported_classes = defaultdict(lambda: [])  # name -> embedding files
+        if not os.path.isfile(src_file):
+            raise PreProcessorError("Source file '{}' isn't a file".format(src_file))
+        if not os.path.isdir(import_base_dir):
+            raise PreProcessorError("MJ_IMPORT_DIR '{}' isn't a directory".format(import_base_dir))
+        if not dst_file != "-":
+            if os.path.isdir(dst_file):
+                raise PreProcessorError("Destination file '{}' isn't a file or '-'".format(dst_file))
+            if os.path.realpath(src_file) == os.path.realpath(dst_file):
+                raise PreProcessorError("Destination file '{}' is equal to the source file".format(dst_file))
+
+
+    def preprocess(self):
+        #if is_importable_file(self.src_file):
+        #    raise PreProcessorError("Can't pre process importable file '{}'".format(self.src_file))
+        self._preprocess_file(self.src_file)
+        self._store_in_dst_file()
+
+    def _preprocess_file(self, file: str):
+        lines = []
+        middle_lines = []
+
+        def add_commented(line: str):
+            middle_lines.append("/*{}*/".format(line))
+
+        with open(file, "r") as f:
+            for line in f:
+                line = line.rstrip()
+                if self._import_regexp.match(line):
+                    _LOG.debug("File '{}': parse '{}'".format(file, line))
+                    full_name = self._imported_class_regexp.search(line).group(0)[:-1]
+                    _LOG.debug("File '{}': import class {}".format(file, full_name))
+                    self._import_file(full_name)
+                    self._imported_classes[relpath(self._file_name_for_full_class_name(full_name))]\
+                        .append(_class_name_for_file(file))
+                    add_commented(line)
+                elif line.startswith("public class "):
+                    _LOG.debug("File '{}': modify '{}'".format(file, line))
+                    line = line.replace("public class ", "/*public*/ class ", 1)
+                    middle_lines.append(line)
+                elif line.startswith("package"):
+                    _LOG.debug("File '{}': ignore '{}'".format(file, line))
+                    add_commented(line)
+                else:
+                    middle_lines.append(line)
+
+        if file != self.src_file:
+            lines.append("/* ##################### \n")
+            lines.append("   imported from file: {}".format(relpath(file, self.import_base_dir)))
+            #lines.append("   imported by: {}".format(",".join(sorted(self._imported_classes[relpath(file)]))))
+            lines.append("\n   ##################### \n*/")
+        lines.extend(middle_lines)
+        self.imported_strs.append("\n".join(lines))
+
+
+    def _import_file(self, full_name: str):
+        _LOG.debug("Try to import {}".format(full_name))
+        path = self._file_name_for_full_class_name(full_name)
+        name = _class_name_for_file(path)
+        if name in self._already_imported_classes:
+            if full_name != self._already_imported_classes[name]:
+                raise PreProcessorError("Can't import {}, as another class with the same name was already imported: {}"
+                           .format(full_name, self._already_imported_classes[name]))
+        else:
+            self._already_imported_classes[name] = full_name
+            self._preprocess_file(path)
+
+    def _file_name_for_full_class_name(self, full_name: str) -> str:
+        parts = full_name.split(".")
+        parts[-1] += ".java"
+        path = os.path.join(self.import_base_dir, *parts)
+        if not os.path.exists(path):
+            raise PreProcessorError("File '{}' doesn't exist, can't import class {}".format(path, _class_name_for_file(path)))
+        if not is_importable_file(path):
+            raise PreProcessorError("File '{}' isn't importable, can't import class {}".format(path, _class_name_for_file(path)))
+        return path
+
+    def _store_in_dst_file(self):
+        _LOG.debug("Try to output to '{}'".format(self.dst_file))
+        if self.dst_file == "-":
+            for text in reversed(self.imported_strs):
+                print()
+                print()
+                print(text)
+        else:
+            with open(self.dst_file, "w") as f:
+                for text in reversed(self.imported_strs):
+                    f.write(text)
+                    f.write("\n\n")
+                f.flush()
+
+def _class_name_for_file(file: str):
+    return os.path.basename(file).split(".")[0]
+
+def is_importable_file(file: str) -> bool:
+    name = _class_name_for_file(file)
+    has_package = False
+    has_public_class = False
+    has_main_method = False
+    with open(file, "r") as f:
+        for line in f:
+            if line.startswith("package "):
+                has_package = True
+            elif line.startswith("public class "):
+                match = re.search("[A-Za-z_]+", line.replace("public class ", ""))
+                if match:
+                    has_public_class = True
+                    if match.group(0) != name:
+                        raise PreProcessorError("File '{}' has invalid format: expected a public class {}, got {}"
+                                   .format(file, name, match.group(0)))
+            elif "String[]" in line and "main" in line and "void" in line and "static" in line and "public" in line:
+                has_main_method = True
+    if all([has_package, has_public_class, not has_main_method]):
+        return True
+    if (has_package or has_public_class) == has_main_method:
+        raise PreProcessorError("File '{}' has invalid format: "
+                                "package={}, 'public class'={}, main method={}"
+                                .format(file, has_package, has_public_class, has_main_method))
+    return False
\ No newline at end of file

preproc/process.py

+#!/usr/bin/env python3
+
+import sys
+from os.path import dirname, realpath
+p = dirname(realpath(__file__))
+sys.path.append(p)
+from preproc.cli import run
+run()
\ No newline at end of file