Add tests with input files

And fix test timeouts.

mjtest/environment.py

@@ -8,7 +8,7 @@ import sys
 from datetime import datetime
 import time
 from threading import Timer
-from typing import Dict
+from typing import Dict, Optional
 
 import subprocess
 
@@ -165,13 +165,13 @@ class Environment:
         cmd = [self.mj_run_cmd, mode_flag] + list(args)
         return execute(cmd, timeout=timeout or self.timeout)
 
-    def run_command(self, cmd: str, *args: Tuple[str], timeout: float = None) -> Tuple[bytes, bytes, int]:
+    def run_command(self, cmd: str, *args: Tuple[str], timeout: float = None, input_str: Optional[str] = None) -> Tuple[bytes, bytes, int]:
         """
         Execute the passend command with its arguments
 
         :return: (out, err, return code)
         """
-        return execute([cmd] + list(args), timeout=timeout or self.timeout)
+        return execute([cmd] + list(args), timeout=timeout or self.timeout, input_str=input_str)
 
     def has_to_preprocess(self, file: str) -> bool:
         return os.path.relpath(file, self.test_dir).startswith("exec")

mjtest/test/exec_tests.py

@@ -4,6 +4,10 @@ import os
 import shutil
 import signal
 from os import path
+
+import re
+from typing import Optional
+
 from mjtest.environment import TestMode, Environment
 from mjtest.test.syntax_tests import BasicSyntaxTest
 from mjtest.test.tests import TestCase, BasicDiffTestResult, BasicTestResult, ExtensibleTestResult
@@ -18,12 +22,16 @@ class JavaExecTest(BasicSyntaxTest):
     """
 
     FILE_ENDINGS = [".java", ".mj"]
-    INVALID_FILE_ENDINGS = [".inf.java", ".inf.mj"]
+    INVALID_FILE_ENDINGS = [".inf.java", ".inf.mj", ".input.mj", ".input.java"]
     OUTPUT_FILE_ENDING = ".out"
     MODE = TestMode.compile_firm
+    INPUT_FILE_REGEXP = r'(\.[0-9]+)\.input(c?)$'
 
     def __init__(self, env: Environment, type: str, file: str, preprocessed_file: str):
         super().__init__(env, type, file, preprocessed_file)
+        self._has_input_file = bool(re.search(self.INPUT_FILE_REGEXP, file))
+        self._input_file = file if self._has_input_file else None
+        self._has_character_input = file.endswith("c") if self._has_input_file else False
         self._expected_output_file = file + self.OUTPUT_FILE_ENDING
         prev_out_dir = path.join(path.dirname(file), ".java_output")
         if not path.exists(prev_out_dir):
@@ -48,10 +56,25 @@ class JavaExecTest(BasicSyntaxTest):
         cwd = os.getcwd()
         os.chdir(tmp_dir)
         exp_out = None
+        input_str = None
         #print(base_filename, get_main_class_name(base_filename + ".java"))
 
         test_result = ExtensibleTestResult(self)
 
+        if self._has_input_file:
+            if self._has_character_input:
+                with open(self._input_file, "r") as f:
+                    input_str = f.read()
+            else:
+                with open(self._input_file, "r") as f:
+                    chars = bytearray('ascii', 'ignore')  # type: bytearray
+                    for line in f.readlines():
+                        for part in line.split(" "):
+                            part = part.strip()
+                            if len(part) > 0:
+                                chars.append(int(part))
+                input_str = chars.decode()
+
         if not self._has_expected_output_file:
             _, err, javac_rtcode = \
                 self.env.run_command("javac", base_filename + ".java", timeout=timeout)
@@ -68,13 +91,15 @@ class JavaExecTest(BasicSyntaxTest):
                 _LOG.debug("Can't find a main class, using the file name instead")
                 main_class = base_filename
             exp_out, err, java_rtcode = \
-                self.env.run_command("java", get_main_class_name(base_filename + ".java"), timeout=timeout)
+                self.env.run_command("java", get_main_class_name(base_filename + ".java"), timeout=timeout, input_str=input_str)
             if javac_rtcode != 0:
                 test_result.add_long_text("Java output: ", exp_out.decode())
                 test_result.incorrect_msg = "java runtime error"
                 test_result.set_error_code(java_rtcode)
                 test_result.add_long_text("Java error message", err.decode())
                 test_result.add_file("Source file", self.preprocessed_file)
+                if self._input_file:
+                    test_result.add_file("Input file", self.file)
                 os.chdir(cwd)
                 return test_result
             exp_out = exp_out.decode()
@@ -111,19 +136,23 @@ class JavaExecTest(BasicSyntaxTest):
                 os.chdir(cwd)
                 raise
             try:
-                out, err, rtcode = self.env.run_command("./a.out", timeout=timeout)
+                out, err, rtcode = self.env.run_command("./a.out", timeout=timeout, input_str=input_str)
                 if rtcode != 0:
                     test_result.incorrect_msg = "binary can't be run, non zero error code"
                     test_result.set_error_code(rtcode)
                     test_result.add_long_text("Error output", err.decode())
                     test_result.add_long_text("Output", out.decode())
                     test_result.add_file("Source file", self.preprocessed_file)
+                    if self._input_file:
+                        test_result.add_file("Input file", self.file)
                     os.chdir(cwd)
                     return test_result
             except SigKill as sig:
                 test_result.incorrect_msg = "binary can't be run: " + sig.name.strip()
                 test_result.set_error_code(sig.retcode)
                 test_result.add_file("Source file", self.preprocessed_file)
+                if self._input_file:
+                    test_result.add_file("Input file", self.file)
                 os.chdir(cwd)
                 return test_result
             except:
@@ -140,6 +169,8 @@ class JavaExecTest(BasicSyntaxTest):
                     test_result.add_long_text("Actual output", out)
                     test_result.add_diff("Output diff [expected <-> actual]", exp_out, out, with_line_numbers=True)
                     test_result.add_file("Source file", self.preprocessed_file)
+                    if self._input_file:
+                        test_result.add_file("Input file", self.file)
                 os.chdir(cwd)
                 return test_result
             return BasicTestResult(self, rtcode, out, err.decode())
@@ -159,10 +190,34 @@ class JavaExecTest(BasicSyntaxTest):
                 _LOG.exception("Decoding a hash sum for java output caching failed")
         return self._hash_sum_for_file(file) == old_hash
 
+    @classmethod
     def _hash_sum_for_file(self, file: str) -> str:
         with open(file, "r") as f:
             return hashlib.sha256(f.read().encode()).hexdigest()
 
+    @classmethod
+    def _test_case_file_for_input_file(cls, input_file: str) -> Optional[str]:
+        base = re.sub(cls.INPUT_FILE_REGEXP, "", input_file)
+        if path.exists(base + ".input.mj"):
+            return base + ".input.mj"
+        if path.exists(base + ".input.java"):
+            return base + ".input.java"
+        return None
+
+    @staticmethod
+    def is_file_ending_valid(cls, file: str) -> bool:
+        if re.search(cls.INPUT_FILE_REGEXP, file) and ".java_output" not in file and "precprocessor" not in file:
+            if not JavaExecTest._test_case_file_for_input_file(file):
+                _LOG.error("Skip {} that hasn't a corresponding test case file".format(file))
+            return True
+        return super().is_file_ending_valid(cls, file)
+
+    @staticmethod
+    def _get_test_case_file(cls, file: str) -> Optional[str]:
+        if re.search(JavaExecTest.INPUT_FILE_REGEXP, file):
+            return JavaExecTest._test_case_file_for_input_file(file)
+        return file
+
 TestCase.TEST_CASE_CLASSES[TestMode.compile_firm].append(JavaExecTest)
 
 

mjtest/test/tests.py

@@ -78,7 +78,7 @@ class TestSuite:
                 if not p.startswith("."):
                     file_names.append(p)
         for file in sorted(file_names):
-            if not TestCase.has_valid_file_ending(self.env.mode, file):
+            if not TestCase.has_valid_file_ending(self.env.mode, join(dir, file)):
                 _LOG.debug("Skip file " + file)
             elif self.env.only_incorrect_tests and file in correct_test_cases:
                 _LOG.info("Skip file {} as its test case was executed correctly the last run")
@@ -87,7 +87,7 @@ class TestSuite:
                 if self.env.has_to_preprocess(file_path) and self.env.is_lib_file(file_path):
                     _LOG.debug("Skip lib file '{}'".format(file))
                     continue
-                preprocessed = self.env.preprocess(join(dir, file))
+                preprocessed = self.env.preprocess(join(dir, TestCase.get_test_case_file(self.env, join(dir, file))))
                 test_case = TestCase.create_from_file(self.env, m, join(dir, file), preprocessed)
                 if not test_case:
                     pass
@@ -261,8 +261,7 @@ class TestCase:
     @classmethod
     def _test_case_class_for_file(cls, type: str, file: str):
         for t in cls.TEST_CASE_CLASSES[type]:
-            if any(file.endswith(e) for e in t.FILE_ENDINGS) and \
-                    not any(file.endswith(e) for e in t.INVALID_FILE_ENDINGS):
+            if t.is_file_ending_valid(t, file):
                 return t
         return False
 
@@ -271,6 +270,19 @@ class TestCase:
     def has_valid_file_ending(cls, type: str, file: str):
         return cls._test_case_class_for_file(type, file) != False
 
+    @staticmethod
+    def is_file_ending_valid(cls, file: str):
+        return any(file.endswith(e) for e in cls.FILE_ENDINGS) and \
+               not any(file.endswith(e) for e in cls.INVALID_FILE_ENDINGS)
+
+
+    @staticmethod
+    def _get_test_case_file(cls, file: str) -> Optional[str]:
+        return file
+
+    @classmethod
+    def get_test_case_file(cls,  env: Environment, file: str) -> Optional[str]:
+        return cls._test_case_class_for_file(env.mode, file)._get_test_case_file(cls, file)
 
 class TestResult:
 
@@ -373,6 +385,7 @@ class ExtensibleTestResult(TestResult):
         self.has_succeeded = error_code == 0
         self.error_code = error_code
 
+
 class TestResultMessage:
 
     def __init__(self, title: str, content: str, multiline: bool, with_line_numbers: bool):

mjtest/util/shell.py

@@ -43,13 +43,14 @@ def _lower_rlimit(res, limit):
 
 
 class _Execute(object):
-    def __init__(self, cmd, timeout, env, rlimit):
+    def __init__(self, cmd, timeout, env, rlimit, input_str):
         self.cmd       = cmd
         self.timeout   = timeout
         self.env       = env
         self.proc      = None
         self.exception = None
         self.rlimit    = rlimit
+        self.input_str = input_str
         MB = 1024 * 1024
         if not 'RLIMIT_CORE' in rlimit:
             rlimit['RLIMIT_CORE'] = 0
@@ -76,35 +77,25 @@ class _Execute(object):
             self.proc = subprocess.Popen(self.cmd,
                                          stdout=subprocess.PIPE,
                                          stderr=subprocess.PIPE,
+                                         stdin=subprocess.PIPE if self.input_str else None,
                                          env=self.env,
                                          shell=True,
                                          **prexec_args)
-            x = self.proc.communicate()
+            #if self.input_str:
+            #    self.proc.stdin.write(self.input_str.decode)
+            input_bytes = self.input_str.encode() if self.input_str else None
+            x = self.proc.communicate(input=input_bytes, timeout=self.timeout if self.timeout > 0.0 else None)
             self.out, self.err = x
             self.returncode = self.proc.returncode
-        except Exception as e:
-            self.exception = e
+        except subprocess.TimeoutExpired as t:
+            self.returncode = -signal.SIGXCPU
+            self.out = bytearray()
+            self.err = bytearray()
+        #except Exception as e:
+        #    self.exception = e
 
     def run(self):
-        if self.timeout > 0.0:
-            _LOG.debug("run with timeout %.1f: %s" % (float(self.timeout), self.cmd))
-            thread = threading.Thread(target=self._run_process)
-            thread.start()
-            thread.join(float(self.timeout))
-            if self.exception is not None:
-                raise self.exception
-            if thread.is_alive():
-                _LOG.debug("timeout %.1f reached, terminate process" % self.timeout)
-                if self.proc is not None:
-                    self.proc.terminate()
-                    thread.join(1.0)
-                    if thread.is_alive():
-                        _LOG.debug("termination ignored, now kill process")
-                        self.proc.kill()
-                        thread.join()
-                raise SigKill(signal.SIGXCPU, "SIGXCPU")
-        else:
-            self._run_process()
+        self._run_process()
 
         # Usually python can recognize application terminations triggered by
         # signals, but somehow it doesn't do this for java (I suspect, that java
@@ -117,7 +108,7 @@ class _Execute(object):
             raise SigKill(-self.returncode, _EXIT_CODES[self.returncode] + ": " + os.strerror(-self.returncode))
         return (self.out, self.err, self.returncode)
 
-def execute(cmd, env=None, timeout=0, rlimit=None, propagate_sigint=True):
+def execute(cmd, env=None, timeout=0, rlimit=None, propagate_sigint=True, input_str = None):
     """Execute a command and return stderr and stdout data"""
     if not rlimit:
         rlimit = dict()
@@ -127,7 +118,7 @@ def execute(cmd, env=None, timeout=0, rlimit=None, propagate_sigint=True):
     else:
         #cmd = shlex.split(cmd[0]) + cmd[1:]
         cmd = " ".join(shlex.quote(c) for c in cmd)
-    exc = _Execute(cmd, timeout, env, rlimit)
+    exc = _Execute(cmd, timeout, env, rlimit, input_str)
     (out, err, returncode) = exc.run()
     if returncode == -signal.SIGINT:
         raise KeyboardInterrupt