diff --git a/.gitignore b/.gitignore index decb10a55f18c4aad78306ff7ac49313b740d77f..435528da82f61664d04eee12e851c5499b78f478 100644 --- a/.gitignore +++ b/.gitignore @@ -9,5 +9,5 @@ build/ .idea .mjtest* mjtest.egg-info - +.preprocessed .DS_Store diff --git a/mjtest/environment.py b/mjtest/environment.py index 6117c70ab5d475a9099c170a94a3a963120f5c20..23b32b392ab9ecf30e91165948e0b8ad4161fbf8 100644 --- a/mjtest/environment.py +++ b/mjtest/environment.py @@ -8,7 +8,9 @@ import time from mjtest.util.shell import execute from mjtest.util.utils import get_mjtest_basedir from typing import Tuple, List +from preproc.preproc.preprocessor import PreProcessor, PreProcessorError, is_importable_file +_LOG = logging.getLogger("env") class TestMode: @@ -86,6 +88,7 @@ class Environment: self.produce_all_reports = produce_all_reports self.ci_testing = ci_testing self._tmp_file_ctr = 0 + self._already_preprocessed_files = set() def create_tmpfile(self) -> str: self._tmp_file_ctr += 1 @@ -124,3 +127,45 @@ class Environment: :return: (out, err, return code) """ return execute([cmd] + list(args), timeout=self.timeout) + + def has_to_preprocess(self, file: str) -> bool: + return os.path.relpath(file, self.test_dir).startswith("exec") + + def is_lib_file(self, file: str) -> bool: + return is_importable_file(file) + + def preprocess(self, file: str) -> str: + """ + Pre process the passed file if needed and return the resulting file + """ + if not self.has_to_preprocess(file): + return file + if ".preprocessed" in os.path.relpath(file, self.test_dir): + return file + + import_base_dir = os.path.join(self.test_dir, os.path.relpath(file, self.test_dir).lstrip(os.sep).split(os.sep)[0]) + dst_dir = os.path.join(import_base_dir, ".preprocessed", os.path.split(os.path.relpath(file, import_base_dir))[0]) + dst_file = os.path.join(dst_dir, os.path.basename(file)) + if dst_file.endswith(".mj"): + dst_file = dst_file.replace(".mj", ".java") + if dst_file in self._already_preprocessed_files: + return dst_file + self._already_preprocessed_files.add(dst_file) + + if os.path.exists(dst_file) and os.path.isfile(dst_file) and os.path.getmtime(file) < os.path.getmtime(dst_file): + _LOG.debug("File '{}' already exists in a pre processed form".format(os.path.relpath(file))) + return dst_file + cur = os.path.split(dst_file)[0] + while not os.path.exists(cur): + os.mkdir(cur) + cur = os.path.split(cur)[0] + + if not os.path.exists(dst_dir): + os.mkdir(dst_dir) + try: + PreProcessor(file, import_base_dir, dst_file).preprocess() + except: + _LOG.exception("Pre processing file '{}'".format(file)) + raise + return dst_file + diff --git a/mjtest/test/ast_tests.py b/mjtest/test/ast_tests.py index 8c5b7130556f4573e45777b1e5299834ece6ca05..0de109060461048296a0cee3b4131d5cc1134029 100644 --- a/mjtest/test/ast_tests.py +++ b/mjtest/test/ast_tests.py @@ -15,14 +15,14 @@ class ASTPrettyPrintTest(BasicSyntaxTest): FILE_ENDINGS = [".mj", ".valid.mj"] INVALID_FILE_ENDINGS = [".invalid.mj"] - def __init__(self, env: Environment, type: str, file: str): - super().__init__(env, type, file) + def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str): + super().__init__(env, type, file, preprocessed_file) if type != TestMode.ast and TEST_MODES.index(TestMode.ast) < TEST_MODES.index(type): self._should_succeed = True def run(self) -> BasicTestResult: tmp_file = self.env.create_tmpfile() - rtcode, out, err = self._pretty_print(self.file, tmp_file) + rtcode, out, err = self._pretty_print(self.preprocessed_file, tmp_file) if rtcode > 0: os.remove(tmp_file) return BasicTestResult(self, rtcode, out, err) diff --git a/mjtest/test/syntax_tests.py b/mjtest/test/syntax_tests.py index 2af185ddddcf05e79d776e633b47a1f2ab3419b5..46e2ff61c95ccf55cab234c86b4f846ca0eba7d2 100644 --- a/mjtest/test/syntax_tests.py +++ b/mjtest/test/syntax_tests.py @@ -9,8 +9,8 @@ class BasicSyntaxTest(TestCase): FILE_ENDINGS = [".invalid.mj", ".valid.mj", ".mj", ".invalid.java", ".java"] MODE = TestMode.syntax - def __init__(self, env: Environment, type: str, file: str): - super().__init__(env, type, file) + def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str): + super().__init__(env, type, file, preprocessed_file) if type != self.MODE and TEST_MODES.index(type) > TEST_MODES.index(self.MODE): self._should_succeed = True else: @@ -23,7 +23,7 @@ class BasicSyntaxTest(TestCase): return path.basename(self.file) def run(self) -> BasicTestResult: - out, err, rtcode = self.env.run_mj_command(self.MODE, self.file) + out, err, rtcode = self.env.run_mj_command(self.MODE, self.preprocessed_file) return BasicTestResult(self, rtcode, out.decode(), err.decode()) TestCase.TEST_CASE_CLASSES[TestMode.syntax].append(BasicSyntaxTest) @@ -37,11 +37,11 @@ class JavaCompileTest(BasicSyntaxTest): FILE_ENDINGS = [".java"] SYNTAX_TEST = True - def __init__(self, env: Environment, type: str, file: str): - super().__init__(env, type, file) + def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str): + super().__init__(env, type, file, preprocessed_file) tmp_dir = self.env.create_tmpdir() _, self.javac_err, self.javac_rtcode = \ - self.env.run_command("javac", path.relpath(file), "-d", tmp_dir, "-verbose") + self.env.run_command("javac", path.relpath(preprocessed_file), "-d", tmp_dir, "-verbose") self.javac_err = self.javac_err.decode() # type: str shutil.rmtree(tmp_dir, ignore_errors=True) self._should_succeed = self._is_file_syntactically_correct() if self.SYNTAX_TEST else self.javac_rtcode == 0 diff --git a/mjtest/test/tests.py b/mjtest/test/tests.py index 549599c877e48548f51503f0a69d691af4595c56..f1644d940661b18cafdb19dff8bf13c0dfdb73e7 100644 --- a/mjtest/test/tests.py +++ b/mjtest/test/tests.py @@ -67,6 +67,8 @@ class TestSuite: base = os.path.relpath(root, dir) if dir == root: file_names.extend(files) + elif base == ".preprocessed": + continue else: file_names.extend(join(base, file) for file in files) for file in sorted(file_names): @@ -75,7 +77,12 @@ class TestSuite: elif self.env.only_incorrect_tests and file in correct_test_cases: _LOG.info("Skip file {} as its test case was executed correctly the last run") else: - test_case = TestCase.create_from_file(self.env, m, join(dir, file)) + file_path = join(dir, file) + if self.env.has_to_preprocess(file_path) and self.env.is_lib_file(file_path): + _LOG.debug("Skip lib file '{}'".format(file)) + continue + preprocessed = self.env.preprocess(join(dir, file)) + test_case = TestCase.create_from_file(self.env, m, join(dir, file), preprocessed) if not test_case: pass elif not test_case.can_run(): @@ -211,10 +218,11 @@ class TestCase: FILE_ENDINGS = [] INVALID_FILE_ENDINGS = [] - def __init__(self, env: Environment, type: str, file: str): + def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str): self.env = env self.type = type self.file = file + self.preprocessed_file = preprocessed_file def should_succeed(self) -> bool: raise NotImplementedError() @@ -234,9 +242,9 @@ class TestCase: raise NotImplementedError() @classmethod - def create_from_file(cls, env: Environment, mode: str, file: str) -> Optional['TestCase']: + def create_from_file(cls, env: Environment, mode: str, file: str, preprocessed_file: str) -> Optional['TestCase']: if cls.has_valid_file_ending(env.mode, file): - return cls._test_case_class_for_file(env.mode, file)(env, mode, file) + return cls._test_case_class_for_file(env.mode, file)(env, mode, file, preprocessed_file) return None def name(self): @@ -313,7 +321,7 @@ class BasicTestResult(TestResult): def long_message(self) -> str: file_content = [] - with open(self.test_case.file, "r") as f: + with open(self.test_case.preprocessed_file, "r") as f: file_content = [line.rstrip() for line in f] others = [] for title, content, long_text in self.other_texts: @@ -394,8 +402,8 @@ class DiffTest(TestCase): OUTPUT_FILE_ENDING = ".out" MODE = TestMode.ast - def __init__(self, env: Environment, type: str, file: str): - super().__init__(env, type, file) + def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str): + super().__init__(env, type, file, preprocessed_file) self._should_succeed = not file.endswith(".invalid.mj") self._expected_output_file = file + self.OUTPUT_FILE_ENDING self._has_expected_output_file = exists(self._expected_output_file) diff --git a/preproc/README.md b/preproc/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9e509a432f4ae8710cb89b00a8e016f829cd20f8 --- /dev/null +++ b/preproc/README.md @@ -0,0 +1,54 @@ +MiniJava preprocessor +===================== + +Why? +---- +It allows to create a library of utility classes and reduces the overall code completion in the test file. +As standard java import and package statements are used, the unprocessed files are valid java files (with all the advantages in the development process). + +Usage +----- + +``` +usage: process.py [-h] [--log_level LOG_LEVEL] FILE MJ_IMPORT_DIR DEST + +MiniJava preprocessor + +positional arguments: + FILE File to pre process + MJ_IMPORT_DIR Base directory for imports. Can be omitted by + assigning the environment variable MJ_IMPORT_DIR. + DEST Destination file, '-' if it should be printed on std + out. + +optional arguments: + -h, --help show this help message and exit + --log_level LOG_LEVEL + Logging level (error, warn, info or debug) +``` + +Importable file +--------------- + +A importable file "X.java" contains at least: + +- a `package` statement +- the string `public class X` somewhere but not in a comment (this isn't checked automatically) + +It can import other importable files. + +Check for "normal" mj files via `./is_normal_mj_file.py FILE`. + +What `import x.y.Math` does: +---------------------------- + +1. Find file `MJ_IMPORT_DIR/x/y/Math.java` and check that it's an importable file +2. Check that the file isn't already imported (ignore it otherwise) and abort if two imported classes are in different packages but have the same name. +3. Store the file in a list of files that should be imported +4. Start with 1. for each `import` statement in the file that isn't already satisfied +5. For each file that should be imported: + 1. Read the file + 2. Remove all strings that match `package.*;` + 3. Replace all `public class` strings with just `class` + 4. Insert it at the end of the original source file +6. Output the modified source file \ No newline at end of file diff --git a/preproc/__init__.py b/preproc/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/preproc/is_normal_mj_file.py b/preproc/is_normal_mj_file.py new file mode 100755 index 0000000000000000000000000000000000000000..6e0a929029e1ad64cfe679ff7d881eced6ee1d46 --- /dev/null +++ b/preproc/is_normal_mj_file.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +import argparse + +import sys +from os.path import dirname, realpath +p = dirname(realpath(__file__)) +sys.path.append(p) +from preproc.preprocessor import PreProcessor, PreProcessorError, is_importable_file +import logging + +parser = argparse.ArgumentParser(description="Is normal MiniJava file?", add_help=True) +parser.add_argument("file", metavar="FILE", help="File to check") +file = parser.parse_args().file + +try: + ret = is_importable_file(file) + sys.exit(1 if ret else 0) +except PreProcessorError as ex: + logging.exception("") + sys.exit(1) \ No newline at end of file diff --git a/preproc/preproc/__init__.py b/preproc/preproc/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/preproc/preproc/cli.py b/preproc/preproc/cli.py new file mode 100644 index 0000000000000000000000000000000000000000..d7691044ba5c54f4f686456fb8394152b554066c --- /dev/null +++ b/preproc/preproc/cli.py @@ -0,0 +1,56 @@ +import logging +import os +import sys +import argparse +from typing import Dict +from datetime import datetime + +from preproc.preprocessor import PreProcessor, PreProcessorError + +_LOG = logging.getLogger("preprocessor") + +# adapted from http://stackoverflow.com/a/8527629 +class LogLevelChoices(argparse.Action): + CHOICES = ["error", "warn", "info", "debug"] + def __call__(self, parser, namespace, value, option_string=None): + if value: + if value not in self.CHOICES: + message = ("invalid choice: {0!r} (choose from {1})" + .format(value, + ', '.join([repr(action) + for action in self.CHOICES]))) + + raise argparse.ArgumentError(self, message) + setattr(namespace, self.dest, value) + +def run(): + parser = argparse.ArgumentParser(description="MiniJava preprocessor", add_help=True) + parser.add_argument("src_file", metavar="FILE", help="File to pre process") + if os.getenv("MJ_IMPORT_DIR", None) is None: + parser.add_argument("import_base_dir", + metavar="MJ_IMPORT_DIR", + help="Base directory for imports. " + "Can be omitted by assigning the environment variable MJ_IMPORT_DIR.") + parser.add_argument("dst_file", metavar="DEST", help="Destination file, '-' if it should be printed on std out.") + + parser.add_argument("--log_level", action=LogLevelChoices, default="warn", const="log_level", + help="Logging level (error, warn, info or debug)") + args = vars(parser.parse_args()) + + if os.getenv("MJ_IMPORT_DIR", None) is not None: + args["import_base_dir"] = os.getenv("MJ_IMPORT_DIR") + + LOG_LEVELS = { + "info": logging.INFO, + "error": logging.ERROR, + "warn": logging.WARN, + "debug": logging.DEBUG + } + + logging.basicConfig(level=LOG_LEVELS[args["log_level"]]) + + try: + PreProcessor(args["src_file"], args["import_base_dir"], args["dst_file"]).preprocess() + except PreProcessorError as err: + _LOG.error(str(err)) + return 1 \ No newline at end of file diff --git a/preproc/preproc/preprocessor.py b/preproc/preproc/preprocessor.py new file mode 100644 index 0000000000000000000000000000000000000000..e01bcf028a9c1a056e4d619bf771e5137b14284a --- /dev/null +++ b/preproc/preproc/preprocessor.py @@ -0,0 +1,144 @@ +from collections import defaultdict +import logging +import os +from os.path import relpath +from pprint import pprint +import re +import sys + +_LOG = logging.getLogger("preprocessor") + +class PreProcessorError(BaseException): + + def __init__(self, msg): + super().__init__(msg) + +class PreProcessor: + + def __init__(self, src_file: str, import_base_dir: str, dst_file: str): + self.src_file = src_file + self.import_base_dir = import_base_dir + self.dst_file = dst_file + self.imported_strs = [] + self._already_imported_classes = {} # name -> full_name + self._import_regexp = re.compile("import [A-Za-z.]+;") + self._imported_class_regexp = re.compile("[A-Za-z.]+;") + self._imported_classes = defaultdict(lambda: []) # name -> embedding files + if not os.path.isfile(src_file): + raise PreProcessorError("Source file '{}' isn't a file".format(src_file)) + if not os.path.isdir(import_base_dir): + raise PreProcessorError("MJ_IMPORT_DIR '{}' isn't a directory".format(import_base_dir)) + if not dst_file != "-": + if os.path.isdir(dst_file): + raise PreProcessorError("Destination file '{}' isn't a file or '-'".format(dst_file)) + if os.path.realpath(src_file) == os.path.realpath(dst_file): + raise PreProcessorError("Destination file '{}' is equal to the source file".format(dst_file)) + + + def preprocess(self): + #if is_importable_file(self.src_file): + # raise PreProcessorError("Can't pre process importable file '{}'".format(self.src_file)) + self._preprocess_file(self.src_file) + self._store_in_dst_file() + + def _preprocess_file(self, file: str): + lines = [] + middle_lines = [] + + def add_commented(line: str): + middle_lines.append("/*{}*/".format(line)) + + with open(file, "r") as f: + for line in f: + line = line.rstrip() + if self._import_regexp.match(line): + _LOG.debug("File '{}': parse '{}'".format(file, line)) + full_name = self._imported_class_regexp.search(line).group(0)[:-1] + _LOG.debug("File '{}': import class {}".format(file, full_name)) + self._import_file(full_name) + self._imported_classes[relpath(self._file_name_for_full_class_name(full_name))]\ + .append(_class_name_for_file(file)) + add_commented(line) + elif line.startswith("public class "): + _LOG.debug("File '{}': modify '{}'".format(file, line)) + line = line.replace("public class ", "/*public*/ class ", 1) + middle_lines.append(line) + elif line.startswith("package"): + _LOG.debug("File '{}': ignore '{}'".format(file, line)) + add_commented(line) + else: + middle_lines.append(line) + + if file != self.src_file: + lines.append("/* ##################### \n") + lines.append(" imported from file: {}".format(relpath(file, self.import_base_dir))) + #lines.append(" imported by: {}".format(",".join(sorted(self._imported_classes[relpath(file)])))) + lines.append("\n ##################### \n*/") + lines.extend(middle_lines) + self.imported_strs.append("\n".join(lines)) + + + def _import_file(self, full_name: str): + _LOG.debug("Try to import {}".format(full_name)) + path = self._file_name_for_full_class_name(full_name) + name = _class_name_for_file(path) + if name in self._already_imported_classes: + if full_name != self._already_imported_classes[name]: + raise PreProcessorError("Can't import {}, as another class with the same name was already imported: {}" + .format(full_name, self._already_imported_classes[name])) + else: + self._already_imported_classes[name] = full_name + self._preprocess_file(path) + + def _file_name_for_full_class_name(self, full_name: str) -> str: + parts = full_name.split(".") + parts[-1] += ".java" + path = os.path.join(self.import_base_dir, *parts) + if not os.path.exists(path): + raise PreProcessorError("File '{}' doesn't exist, can't import class {}".format(path, _class_name_for_file(path))) + if not is_importable_file(path): + raise PreProcessorError("File '{}' isn't importable, can't import class {}".format(path, _class_name_for_file(path))) + return path + + def _store_in_dst_file(self): + _LOG.debug("Try to output to '{}'".format(self.dst_file)) + if self.dst_file == "-": + for text in reversed(self.imported_strs): + print() + print() + print(text) + else: + with open(self.dst_file, "w") as f: + for text in reversed(self.imported_strs): + f.write(text) + f.write("\n\n") + f.flush() + +def _class_name_for_file(file: str): + return os.path.basename(file).split(".")[0] + +def is_importable_file(file: str) -> bool: + name = _class_name_for_file(file) + has_package = False + has_public_class = False + has_main_method = False + with open(file, "r") as f: + for line in f: + if line.startswith("package "): + has_package = True + elif line.startswith("public class "): + match = re.search("[A-Za-z_]+", line.replace("public class ", "")) + if match: + has_public_class = True + if match.group(0) != name: + raise PreProcessorError("File '{}' has invalid format: expected a public class {}, got {}" + .format(file, name, match.group(0))) + elif "String[]" in line and "main" in line and "void" in line and "static" in line and "public" in line: + has_main_method = True + if all([has_package, has_public_class, not has_main_method]): + return True + if (has_package or has_public_class) == has_main_method: + raise PreProcessorError("File '{}' has invalid format: " + "package={}, 'public class'={}, main method={}" + .format(file, has_package, has_public_class, has_main_method)) + return False \ No newline at end of file diff --git a/preproc/process.py b/preproc/process.py new file mode 100755 index 0000000000000000000000000000000000000000..f66b29ea818c8505e5000806edf7c4581a82ec9a --- /dev/null +++ b/preproc/process.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 + +import sys +from os.path import dirname, realpath +p = dirname(realpath(__file__)) +sys.path.append(p) +from preproc.cli import run +run() \ No newline at end of file