Commit 34d9605a authored by Johannes Bechberger's avatar Johannes Bechberger

Add simple import statement preprocessor

parent ccf16311
......@@ -9,5 +9,5 @@ build/
.idea
.mjtest*
mjtest.egg-info
.preprocessed
.DS_Store
......@@ -8,7 +8,9 @@ import time
from mjtest.util.shell import execute
from mjtest.util.utils import get_mjtest_basedir
from typing import Tuple, List
from preproc.preproc.preprocessor import PreProcessor, PreProcessorError, is_importable_file
_LOG = logging.getLogger("env")
class TestMode:
......@@ -86,6 +88,7 @@ class Environment:
self.produce_all_reports = produce_all_reports
self.ci_testing = ci_testing
self._tmp_file_ctr = 0
self._already_preprocessed_files = set()
def create_tmpfile(self) -> str:
self._tmp_file_ctr += 1
......@@ -124,3 +127,45 @@ class Environment:
:return: (out, err, return code)
"""
return execute([cmd] + list(args), timeout=self.timeout)
def has_to_preprocess(self, file: str) -> bool:
return os.path.relpath(file, self.test_dir).startswith("exec")
def is_lib_file(self, file: str) -> bool:
return is_importable_file(file)
def preprocess(self, file: str) -> str:
"""
Pre process the passed file if needed and return the resulting file
"""
if not self.has_to_preprocess(file):
return file
if ".preprocessed" in os.path.relpath(file, self.test_dir):
return file
import_base_dir = os.path.join(self.test_dir, os.path.relpath(file, self.test_dir).lstrip(os.sep).split(os.sep)[0])
dst_dir = os.path.join(import_base_dir, ".preprocessed", os.path.split(os.path.relpath(file, import_base_dir))[0])
dst_file = os.path.join(dst_dir, os.path.basename(file))
if dst_file.endswith(".mj"):
dst_file = dst_file.replace(".mj", ".java")
if dst_file in self._already_preprocessed_files:
return dst_file
self._already_preprocessed_files.add(dst_file)
if os.path.exists(dst_file) and os.path.isfile(dst_file) and os.path.getmtime(file) < os.path.getmtime(dst_file):
_LOG.debug("File '{}' already exists in a pre processed form".format(os.path.relpath(file)))
return dst_file
cur = os.path.split(dst_file)[0]
while not os.path.exists(cur):
os.mkdir(cur)
cur = os.path.split(cur)[0]
if not os.path.exists(dst_dir):
os.mkdir(dst_dir)
try:
PreProcessor(file, import_base_dir, dst_file).preprocess()
except:
_LOG.exception("Pre processing file '{}'".format(file))
raise
return dst_file
......@@ -15,14 +15,14 @@ class ASTPrettyPrintTest(BasicSyntaxTest):
FILE_ENDINGS = [".mj", ".valid.mj"]
INVALID_FILE_ENDINGS = [".invalid.mj"]
def __init__(self, env: Environment, type: str, file: str):
super().__init__(env, type, file)
def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str):
super().__init__(env, type, file, preprocessed_file)
if type != TestMode.ast and TEST_MODES.index(TestMode.ast) < TEST_MODES.index(type):
self._should_succeed = True
def run(self) -> BasicTestResult:
tmp_file = self.env.create_tmpfile()
rtcode, out, err = self._pretty_print(self.file, tmp_file)
rtcode, out, err = self._pretty_print(self.preprocessed_file, tmp_file)
if rtcode > 0:
os.remove(tmp_file)
return BasicTestResult(self, rtcode, out, err)
......
......@@ -9,8 +9,8 @@ class BasicSyntaxTest(TestCase):
FILE_ENDINGS = [".invalid.mj", ".valid.mj", ".mj", ".invalid.java", ".java"]
MODE = TestMode.syntax
def __init__(self, env: Environment, type: str, file: str):
super().__init__(env, type, file)
def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str):
super().__init__(env, type, file, preprocessed_file)
if type != self.MODE and TEST_MODES.index(type) > TEST_MODES.index(self.MODE):
self._should_succeed = True
else:
......@@ -23,7 +23,7 @@ class BasicSyntaxTest(TestCase):
return path.basename(self.file)
def run(self) -> BasicTestResult:
out, err, rtcode = self.env.run_mj_command(self.MODE, self.file)
out, err, rtcode = self.env.run_mj_command(self.MODE, self.preprocessed_file)
return BasicTestResult(self, rtcode, out.decode(), err.decode())
TestCase.TEST_CASE_CLASSES[TestMode.syntax].append(BasicSyntaxTest)
......@@ -37,11 +37,11 @@ class JavaCompileTest(BasicSyntaxTest):
FILE_ENDINGS = [".java"]
SYNTAX_TEST = True
def __init__(self, env: Environment, type: str, file: str):
super().__init__(env, type, file)
def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str):
super().__init__(env, type, file, preprocessed_file)
tmp_dir = self.env.create_tmpdir()
_, self.javac_err, self.javac_rtcode = \
self.env.run_command("javac", path.relpath(file), "-d", tmp_dir, "-verbose")
self.env.run_command("javac", path.relpath(preprocessed_file), "-d", tmp_dir, "-verbose")
self.javac_err = self.javac_err.decode() # type: str
shutil.rmtree(tmp_dir, ignore_errors=True)
self._should_succeed = self._is_file_syntactically_correct() if self.SYNTAX_TEST else self.javac_rtcode == 0
......
......@@ -67,6 +67,8 @@ class TestSuite:
base = os.path.relpath(root, dir)
if dir == root:
file_names.extend(files)
elif base == ".preprocessed":
continue
else:
file_names.extend(join(base, file) for file in files)
for file in sorted(file_names):
......@@ -75,7 +77,12 @@ class TestSuite:
elif self.env.only_incorrect_tests and file in correct_test_cases:
_LOG.info("Skip file {} as its test case was executed correctly the last run")
else:
test_case = TestCase.create_from_file(self.env, m, join(dir, file))
file_path = join(dir, file)
if self.env.has_to_preprocess(file_path) and self.env.is_lib_file(file_path):
_LOG.debug("Skip lib file '{}'".format(file))
continue
preprocessed = self.env.preprocess(join(dir, file))
test_case = TestCase.create_from_file(self.env, m, join(dir, file), preprocessed)
if not test_case:
pass
elif not test_case.can_run():
......@@ -211,10 +218,11 @@ class TestCase:
FILE_ENDINGS = []
INVALID_FILE_ENDINGS = []
def __init__(self, env: Environment, type: str, file: str):
def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str):
self.env = env
self.type = type
self.file = file
self.preprocessed_file = preprocessed_file
def should_succeed(self) -> bool:
raise NotImplementedError()
......@@ -234,9 +242,9 @@ class TestCase:
raise NotImplementedError()
@classmethod
def create_from_file(cls, env: Environment, mode: str, file: str) -> Optional['TestCase']:
def create_from_file(cls, env: Environment, mode: str, file: str, preprocessed_file: str) -> Optional['TestCase']:
if cls.has_valid_file_ending(env.mode, file):
return cls._test_case_class_for_file(env.mode, file)(env, mode, file)
return cls._test_case_class_for_file(env.mode, file)(env, mode, file, preprocessed_file)
return None
def name(self):
......@@ -313,7 +321,7 @@ class BasicTestResult(TestResult):
def long_message(self) -> str:
file_content = []
with open(self.test_case.file, "r") as f:
with open(self.test_case.preprocessed_file, "r") as f:
file_content = [line.rstrip() for line in f]
others = []
for title, content, long_text in self.other_texts:
......@@ -394,8 +402,8 @@ class DiffTest(TestCase):
OUTPUT_FILE_ENDING = ".out"
MODE = TestMode.ast
def __init__(self, env: Environment, type: str, file: str):
super().__init__(env, type, file)
def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str):
super().__init__(env, type, file, preprocessed_file)
self._should_succeed = not file.endswith(".invalid.mj")
self._expected_output_file = file + self.OUTPUT_FILE_ENDING
self._has_expected_output_file = exists(self._expected_output_file)
......
MiniJava preprocessor
=====================
Why?
----
It allows to create a library of utility classes and reduces the overall code completion in the test file.
As standard java import and package statements are used, the unprocessed files are valid java files (with all the advantages in the development process).
Usage
-----
```
usage: process.py [-h] [--log_level LOG_LEVEL] FILE MJ_IMPORT_DIR DEST
MiniJava preprocessor
positional arguments:
FILE File to pre process
MJ_IMPORT_DIR Base directory for imports. Can be omitted by
assigning the environment variable MJ_IMPORT_DIR.
DEST Destination file, '-' if it should be printed on std
out.
optional arguments:
-h, --help show this help message and exit
--log_level LOG_LEVEL
Logging level (error, warn, info or debug)
```
Importable file
---------------
A importable file "X.java" contains at least:
- a `package` statement
- the string `public class X` somewhere but not in a comment (this isn't checked automatically)
It can import other importable files.
Check for "normal" mj files via `./is_normal_mj_file.py FILE`.
What `import x.y.Math` does:
----------------------------
1. Find file `MJ_IMPORT_DIR/x/y/Math.java` and check that it's an importable file
2. Check that the file isn't already imported (ignore it otherwise) and abort if two imported classes are in different packages but have the same name.
3. Store the file in a list of files that should be imported
4. Start with 1. for each `import` statement in the file that isn't already satisfied
5. For each file that should be imported:
1. Read the file
2. Remove all strings that match `package.*;`
3. Replace all `public class` strings with just `class`
4. Insert it at the end of the original source file
6. Output the modified source file
\ No newline at end of file
#!/usr/bin/env python3
import argparse
import sys
from os.path import dirname, realpath
p = dirname(realpath(__file__))
sys.path.append(p)
from preproc.preprocessor import PreProcessor, PreProcessorError, is_importable_file
import logging
parser = argparse.ArgumentParser(description="Is normal MiniJava file?", add_help=True)
parser.add_argument("file", metavar="FILE", help="File to check")
file = parser.parse_args().file
try:
ret = is_importable_file(file)
sys.exit(1 if ret else 0)
except PreProcessorError as ex:
logging.exception("")
sys.exit(1)
\ No newline at end of file
import logging
import os
import sys
import argparse
from typing import Dict
from datetime import datetime
from preproc.preprocessor import PreProcessor, PreProcessorError
_LOG = logging.getLogger("preprocessor")
# adapted from http://stackoverflow.com/a/8527629
class LogLevelChoices(argparse.Action):
CHOICES = ["error", "warn", "info", "debug"]
def __call__(self, parser, namespace, value, option_string=None):
if value:
if value not in self.CHOICES:
message = ("invalid choice: {0!r} (choose from {1})"
.format(value,
', '.join([repr(action)
for action in self.CHOICES])))
raise argparse.ArgumentError(self, message)
setattr(namespace, self.dest, value)
def run():
parser = argparse.ArgumentParser(description="MiniJava preprocessor", add_help=True)
parser.add_argument("src_file", metavar="FILE", help="File to pre process")
if os.getenv("MJ_IMPORT_DIR", None) is None:
parser.add_argument("import_base_dir",
metavar="MJ_IMPORT_DIR",
help="Base directory for imports. "
"Can be omitted by assigning the environment variable MJ_IMPORT_DIR.")
parser.add_argument("dst_file", metavar="DEST", help="Destination file, '-' if it should be printed on std out.")
parser.add_argument("--log_level", action=LogLevelChoices, default="warn", const="log_level",
help="Logging level (error, warn, info or debug)")
args = vars(parser.parse_args())
if os.getenv("MJ_IMPORT_DIR", None) is not None:
args["import_base_dir"] = os.getenv("MJ_IMPORT_DIR")
LOG_LEVELS = {
"info": logging.INFO,
"error": logging.ERROR,
"warn": logging.WARN,
"debug": logging.DEBUG
}
logging.basicConfig(level=LOG_LEVELS[args["log_level"]])
try:
PreProcessor(args["src_file"], args["import_base_dir"], args["dst_file"]).preprocess()
except PreProcessorError as err:
_LOG.error(str(err))
return 1
\ No newline at end of file
from collections import defaultdict
import logging
import os
from os.path import relpath
from pprint import pprint
import re
import sys
_LOG = logging.getLogger("preprocessor")
class PreProcessorError(BaseException):
def __init__(self, msg):
super().__init__(msg)
class PreProcessor:
def __init__(self, src_file: str, import_base_dir: str, dst_file: str):
self.src_file = src_file
self.import_base_dir = import_base_dir
self.dst_file = dst_file
self.imported_strs = []
self._already_imported_classes = {} # name -> full_name
self._import_regexp = re.compile("import [A-Za-z.]+;")
self._imported_class_regexp = re.compile("[A-Za-z.]+;")
self._imported_classes = defaultdict(lambda: []) # name -> embedding files
if not os.path.isfile(src_file):
raise PreProcessorError("Source file '{}' isn't a file".format(src_file))
if not os.path.isdir(import_base_dir):
raise PreProcessorError("MJ_IMPORT_DIR '{}' isn't a directory".format(import_base_dir))
if not dst_file != "-":
if os.path.isdir(dst_file):
raise PreProcessorError("Destination file '{}' isn't a file or '-'".format(dst_file))
if os.path.realpath(src_file) == os.path.realpath(dst_file):
raise PreProcessorError("Destination file '{}' is equal to the source file".format(dst_file))
def preprocess(self):
#if is_importable_file(self.src_file):
# raise PreProcessorError("Can't pre process importable file '{}'".format(self.src_file))
self._preprocess_file(self.src_file)
self._store_in_dst_file()
def _preprocess_file(self, file: str):
lines = []
middle_lines = []
def add_commented(line: str):
middle_lines.append("/*{}*/".format(line))
with open(file, "r") as f:
for line in f:
line = line.rstrip()
if self._import_regexp.match(line):
_LOG.debug("File '{}': parse '{}'".format(file, line))
full_name = self._imported_class_regexp.search(line).group(0)[:-1]
_LOG.debug("File '{}': import class {}".format(file, full_name))
self._import_file(full_name)
self._imported_classes[relpath(self._file_name_for_full_class_name(full_name))]\
.append(_class_name_for_file(file))
add_commented(line)
elif line.startswith("public class "):
_LOG.debug("File '{}': modify '{}'".format(file, line))
line = line.replace("public class ", "/*public*/ class ", 1)
middle_lines.append(line)
elif line.startswith("package"):
_LOG.debug("File '{}': ignore '{}'".format(file, line))
add_commented(line)
else:
middle_lines.append(line)
if file != self.src_file:
lines.append("/* ##################### \n")
lines.append(" imported from file: {}".format(relpath(file, self.import_base_dir)))
#lines.append(" imported by: {}".format(",".join(sorted(self._imported_classes[relpath(file)]))))
lines.append("\n ##################### \n*/")
lines.extend(middle_lines)
self.imported_strs.append("\n".join(lines))
def _import_file(self, full_name: str):
_LOG.debug("Try to import {}".format(full_name))
path = self._file_name_for_full_class_name(full_name)
name = _class_name_for_file(path)
if name in self._already_imported_classes:
if full_name != self._already_imported_classes[name]:
raise PreProcessorError("Can't import {}, as another class with the same name was already imported: {}"
.format(full_name, self._already_imported_classes[name]))
else:
self._already_imported_classes[name] = full_name
self._preprocess_file(path)
def _file_name_for_full_class_name(self, full_name: str) -> str:
parts = full_name.split(".")
parts[-1] += ".java"
path = os.path.join(self.import_base_dir, *parts)
if not os.path.exists(path):
raise PreProcessorError("File '{}' doesn't exist, can't import class {}".format(path, _class_name_for_file(path)))
if not is_importable_file(path):
raise PreProcessorError("File '{}' isn't importable, can't import class {}".format(path, _class_name_for_file(path)))
return path
def _store_in_dst_file(self):
_LOG.debug("Try to output to '{}'".format(self.dst_file))
if self.dst_file == "-":
for text in reversed(self.imported_strs):
print()
print()
print(text)
else:
with open(self.dst_file, "w") as f:
for text in reversed(self.imported_strs):
f.write(text)
f.write("\n\n")
f.flush()
def _class_name_for_file(file: str):
return os.path.basename(file).split(".")[0]
def is_importable_file(file: str) -> bool:
name = _class_name_for_file(file)
has_package = False
has_public_class = False
has_main_method = False
with open(file, "r") as f:
for line in f:
if line.startswith("package "):
has_package = True
elif line.startswith("public class "):
match = re.search("[A-Za-z_]+", line.replace("public class ", ""))
if match:
has_public_class = True
if match.group(0) != name:
raise PreProcessorError("File '{}' has invalid format: expected a public class {}, got {}"
.format(file, name, match.group(0)))
elif "String[]" in line and "main" in line and "void" in line and "static" in line and "public" in line:
has_main_method = True
if all([has_package, has_public_class, not has_main_method]):
return True
if (has_package or has_public_class) == has_main_method:
raise PreProcessorError("File '{}' has invalid format: "
"package={}, 'public class'={}, main method={}"
.format(file, has_package, has_public_class, has_main_method))
return False
\ No newline at end of file
#!/usr/bin/env python3
import sys
from os.path import dirname, realpath
p = dirname(realpath(__file__))
sys.path.append(p)
from preproc.cli import run
run()
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment