Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
uwdkn
mjtest
Commits
fcaf1778
Commit
fcaf1778
authored
Dec 28, 2016
by
Johannes Bechberger
Browse files
Add tests with input files
And fix test timeouts.
parent
4a6e643d
Changes
4
Hide whitespace changes
Inline
Side-by-side
mjtest/environment.py
View file @
fcaf1778
...
...
@@ -8,7 +8,7 @@ import sys
from
datetime
import
datetime
import
time
from
threading
import
Timer
from
typing
import
Dict
from
typing
import
Dict
,
Optional
import
subprocess
...
...
@@ -165,13 +165,13 @@ class Environment:
cmd
=
[
self
.
mj_run_cmd
,
mode_flag
]
+
list
(
args
)
return
execute
(
cmd
,
timeout
=
timeout
or
self
.
timeout
)
def
run_command
(
self
,
cmd
:
str
,
*
args
:
Tuple
[
str
],
timeout
:
float
=
None
)
->
Tuple
[
bytes
,
bytes
,
int
]:
def
run_command
(
self
,
cmd
:
str
,
*
args
:
Tuple
[
str
],
timeout
:
float
=
None
,
input_str
:
Optional
[
str
]
=
None
)
->
Tuple
[
bytes
,
bytes
,
int
]:
"""
Execute the passend command with its arguments
:return: (out, err, return code)
"""
return
execute
([
cmd
]
+
list
(
args
),
timeout
=
timeout
or
self
.
timeout
)
return
execute
([
cmd
]
+
list
(
args
),
timeout
=
timeout
or
self
.
timeout
,
input_str
=
input_str
)
def
has_to_preprocess
(
self
,
file
:
str
)
->
bool
:
return
os
.
path
.
relpath
(
file
,
self
.
test_dir
).
startswith
(
"exec"
)
...
...
mjtest/test/exec_tests.py
View file @
fcaf1778
...
...
@@ -4,6 +4,10 @@ import os
import
shutil
import
signal
from
os
import
path
import
re
from
typing
import
Optional
from
mjtest.environment
import
TestMode
,
Environment
from
mjtest.test.syntax_tests
import
BasicSyntaxTest
from
mjtest.test.tests
import
TestCase
,
BasicDiffTestResult
,
BasicTestResult
,
ExtensibleTestResult
...
...
@@ -18,12 +22,16 @@ class JavaExecTest(BasicSyntaxTest):
"""
FILE_ENDINGS
=
[
".java"
,
".mj"
]
INVALID_FILE_ENDINGS
=
[
".inf.java"
,
".inf.mj"
]
INVALID_FILE_ENDINGS
=
[
".inf.java"
,
".inf.mj"
,
".input.mj"
,
".input.java"
]
OUTPUT_FILE_ENDING
=
".out"
MODE
=
TestMode
.
compile_firm
INPUT_FILE_REGEXP
=
r
'(\.[0-9]+)\.input(c?)$'
def
__init__
(
self
,
env
:
Environment
,
type
:
str
,
file
:
str
,
preprocessed_file
:
str
):
super
().
__init__
(
env
,
type
,
file
,
preprocessed_file
)
self
.
_has_input_file
=
bool
(
re
.
search
(
self
.
INPUT_FILE_REGEXP
,
file
))
self
.
_input_file
=
file
if
self
.
_has_input_file
else
None
self
.
_has_character_input
=
file
.
endswith
(
"c"
)
if
self
.
_has_input_file
else
False
self
.
_expected_output_file
=
file
+
self
.
OUTPUT_FILE_ENDING
prev_out_dir
=
path
.
join
(
path
.
dirname
(
file
),
".java_output"
)
if
not
path
.
exists
(
prev_out_dir
):
...
...
@@ -48,10 +56,25 @@ class JavaExecTest(BasicSyntaxTest):
cwd
=
os
.
getcwd
()
os
.
chdir
(
tmp_dir
)
exp_out
=
None
input_str
=
None
#print(base_filename, get_main_class_name(base_filename + ".java"))
test_result
=
ExtensibleTestResult
(
self
)
if
self
.
_has_input_file
:
if
self
.
_has_character_input
:
with
open
(
self
.
_input_file
,
"r"
)
as
f
:
input_str
=
f
.
read
()
else
:
with
open
(
self
.
_input_file
,
"r"
)
as
f
:
chars
=
bytearray
(
'ascii'
,
'ignore'
)
# type: bytearray
for
line
in
f
.
readlines
():
for
part
in
line
.
split
(
" "
):
part
=
part
.
strip
()
if
len
(
part
)
>
0
:
chars
.
append
(
int
(
part
))
input_str
=
chars
.
decode
()
if
not
self
.
_has_expected_output_file
:
_
,
err
,
javac_rtcode
=
\
self
.
env
.
run_command
(
"javac"
,
base_filename
+
".java"
,
timeout
=
timeout
)
...
...
@@ -68,13 +91,15 @@ class JavaExecTest(BasicSyntaxTest):
_LOG
.
debug
(
"Can't find a main class, using the file name instead"
)
main_class
=
base_filename
exp_out
,
err
,
java_rtcode
=
\
self
.
env
.
run_command
(
"java"
,
get_main_class_name
(
base_filename
+
".java"
),
timeout
=
timeout
)
self
.
env
.
run_command
(
"java"
,
get_main_class_name
(
base_filename
+
".java"
),
timeout
=
timeout
,
input_str
=
input_str
)
if
javac_rtcode
!=
0
:
test_result
.
add_long_text
(
"Java output: "
,
exp_out
.
decode
())
test_result
.
incorrect_msg
=
"java runtime error"
test_result
.
set_error_code
(
java_rtcode
)
test_result
.
add_long_text
(
"Java error message"
,
err
.
decode
())
test_result
.
add_file
(
"Source file"
,
self
.
preprocessed_file
)
if
self
.
_input_file
:
test_result
.
add_file
(
"Input file"
,
self
.
file
)
os
.
chdir
(
cwd
)
return
test_result
exp_out
=
exp_out
.
decode
()
...
...
@@ -111,19 +136,23 @@ class JavaExecTest(BasicSyntaxTest):
os
.
chdir
(
cwd
)
raise
try
:
out
,
err
,
rtcode
=
self
.
env
.
run_command
(
"./a.out"
,
timeout
=
timeout
)
out
,
err
,
rtcode
=
self
.
env
.
run_command
(
"./a.out"
,
timeout
=
timeout
,
input_str
=
input_str
)
if
rtcode
!=
0
:
test_result
.
incorrect_msg
=
"binary can't be run, non zero error code"
test_result
.
set_error_code
(
rtcode
)
test_result
.
add_long_text
(
"Error output"
,
err
.
decode
())
test_result
.
add_long_text
(
"Output"
,
out
.
decode
())
test_result
.
add_file
(
"Source file"
,
self
.
preprocessed_file
)
if
self
.
_input_file
:
test_result
.
add_file
(
"Input file"
,
self
.
file
)
os
.
chdir
(
cwd
)
return
test_result
except
SigKill
as
sig
:
test_result
.
incorrect_msg
=
"binary can't be run: "
+
sig
.
name
.
strip
()
test_result
.
set_error_code
(
sig
.
retcode
)
test_result
.
add_file
(
"Source file"
,
self
.
preprocessed_file
)
if
self
.
_input_file
:
test_result
.
add_file
(
"Input file"
,
self
.
file
)
os
.
chdir
(
cwd
)
return
test_result
except
:
...
...
@@ -140,6 +169,8 @@ class JavaExecTest(BasicSyntaxTest):
test_result
.
add_long_text
(
"Actual output"
,
out
)
test_result
.
add_diff
(
"Output diff [expected <-> actual]"
,
exp_out
,
out
,
with_line_numbers
=
True
)
test_result
.
add_file
(
"Source file"
,
self
.
preprocessed_file
)
if
self
.
_input_file
:
test_result
.
add_file
(
"Input file"
,
self
.
file
)
os
.
chdir
(
cwd
)
return
test_result
return
BasicTestResult
(
self
,
rtcode
,
out
,
err
.
decode
())
...
...
@@ -159,10 +190,34 @@ class JavaExecTest(BasicSyntaxTest):
_LOG
.
exception
(
"Decoding a hash sum for java output caching failed"
)
return
self
.
_hash_sum_for_file
(
file
)
==
old_hash
@
classmethod
def
_hash_sum_for_file
(
self
,
file
:
str
)
->
str
:
with
open
(
file
,
"r"
)
as
f
:
return
hashlib
.
sha256
(
f
.
read
().
encode
()).
hexdigest
()
@
classmethod
def
_test_case_file_for_input_file
(
cls
,
input_file
:
str
)
->
Optional
[
str
]:
base
=
re
.
sub
(
cls
.
INPUT_FILE_REGEXP
,
""
,
input_file
)
if
path
.
exists
(
base
+
".input.mj"
):
return
base
+
".input.mj"
if
path
.
exists
(
base
+
".input.java"
):
return
base
+
".input.java"
return
None
@
staticmethod
def
is_file_ending_valid
(
cls
,
file
:
str
)
->
bool
:
if
re
.
search
(
cls
.
INPUT_FILE_REGEXP
,
file
)
and
".java_output"
not
in
file
and
"precprocessor"
not
in
file
:
if
not
JavaExecTest
.
_test_case_file_for_input_file
(
file
):
_LOG
.
error
(
"Skip {} that hasn't a corresponding test case file"
.
format
(
file
))
return
True
return
super
().
is_file_ending_valid
(
cls
,
file
)
@
staticmethod
def
_get_test_case_file
(
cls
,
file
:
str
)
->
Optional
[
str
]:
if
re
.
search
(
JavaExecTest
.
INPUT_FILE_REGEXP
,
file
):
return
JavaExecTest
.
_test_case_file_for_input_file
(
file
)
return
file
TestCase
.
TEST_CASE_CLASSES
[
TestMode
.
compile_firm
].
append
(
JavaExecTest
)
...
...
mjtest/test/tests.py
View file @
fcaf1778
...
...
@@ -78,7 +78,7 @@ class TestSuite:
if
not
p
.
startswith
(
"."
):
file_names
.
append
(
p
)
for
file
in
sorted
(
file_names
):
if
not
TestCase
.
has_valid_file_ending
(
self
.
env
.
mode
,
file
):
if
not
TestCase
.
has_valid_file_ending
(
self
.
env
.
mode
,
join
(
dir
,
file
)
)
:
_LOG
.
debug
(
"Skip file "
+
file
)
elif
self
.
env
.
only_incorrect_tests
and
file
in
correct_test_cases
:
_LOG
.
info
(
"Skip file {} as its test case was executed correctly the last run"
)
...
...
@@ -87,7 +87,7 @@ class TestSuite:
if
self
.
env
.
has_to_preprocess
(
file_path
)
and
self
.
env
.
is_lib_file
(
file_path
):
_LOG
.
debug
(
"Skip lib file '{}'"
.
format
(
file
))
continue
preprocessed
=
self
.
env
.
preprocess
(
join
(
dir
,
file
))
preprocessed
=
self
.
env
.
preprocess
(
join
(
dir
,
TestCase
.
get_test_case_file
(
self
.
env
,
join
(
dir
,
file
))
))
test_case
=
TestCase
.
create_from_file
(
self
.
env
,
m
,
join
(
dir
,
file
),
preprocessed
)
if
not
test_case
:
pass
...
...
@@ -261,8 +261,7 @@ class TestCase:
@
classmethod
def
_test_case_class_for_file
(
cls
,
type
:
str
,
file
:
str
):
for
t
in
cls
.
TEST_CASE_CLASSES
[
type
]:
if
any
(
file
.
endswith
(
e
)
for
e
in
t
.
FILE_ENDINGS
)
and
\
not
any
(
file
.
endswith
(
e
)
for
e
in
t
.
INVALID_FILE_ENDINGS
):
if
t
.
is_file_ending_valid
(
t
,
file
):
return
t
return
False
...
...
@@ -271,6 +270,19 @@ class TestCase:
def
has_valid_file_ending
(
cls
,
type
:
str
,
file
:
str
):
return
cls
.
_test_case_class_for_file
(
type
,
file
)
!=
False
@
staticmethod
def
is_file_ending_valid
(
cls
,
file
:
str
):
return
any
(
file
.
endswith
(
e
)
for
e
in
cls
.
FILE_ENDINGS
)
and
\
not
any
(
file
.
endswith
(
e
)
for
e
in
cls
.
INVALID_FILE_ENDINGS
)
@
staticmethod
def
_get_test_case_file
(
cls
,
file
:
str
)
->
Optional
[
str
]:
return
file
@
classmethod
def
get_test_case_file
(
cls
,
env
:
Environment
,
file
:
str
)
->
Optional
[
str
]:
return
cls
.
_test_case_class_for_file
(
env
.
mode
,
file
).
_get_test_case_file
(
cls
,
file
)
class
TestResult
:
...
...
@@ -373,6 +385,7 @@ class ExtensibleTestResult(TestResult):
self
.
has_succeeded
=
error_code
==
0
self
.
error_code
=
error_code
class
TestResultMessage
:
def
__init__
(
self
,
title
:
str
,
content
:
str
,
multiline
:
bool
,
with_line_numbers
:
bool
):
...
...
mjtest/util/shell.py
View file @
fcaf1778
...
...
@@ -43,13 +43,14 @@ def _lower_rlimit(res, limit):
class
_Execute
(
object
):
def
__init__
(
self
,
cmd
,
timeout
,
env
,
rlimit
):
def
__init__
(
self
,
cmd
,
timeout
,
env
,
rlimit
,
input_str
):
self
.
cmd
=
cmd
self
.
timeout
=
timeout
self
.
env
=
env
self
.
proc
=
None
self
.
exception
=
None
self
.
rlimit
=
rlimit
self
.
input_str
=
input_str
MB
=
1024
*
1024
if
not
'RLIMIT_CORE'
in
rlimit
:
rlimit
[
'RLIMIT_CORE'
]
=
0
...
...
@@ -76,35 +77,25 @@ class _Execute(object):
self
.
proc
=
subprocess
.
Popen
(
self
.
cmd
,
stdout
=
subprocess
.
PIPE
,
stderr
=
subprocess
.
PIPE
,
stdin
=
subprocess
.
PIPE
if
self
.
input_str
else
None
,
env
=
self
.
env
,
shell
=
True
,
**
prexec_args
)
x
=
self
.
proc
.
communicate
()
#if self.input_str:
# self.proc.stdin.write(self.input_str.decode)
input_bytes
=
self
.
input_str
.
encode
()
if
self
.
input_str
else
None
x
=
self
.
proc
.
communicate
(
input
=
input_bytes
,
timeout
=
self
.
timeout
if
self
.
timeout
>
0.0
else
None
)
self
.
out
,
self
.
err
=
x
self
.
returncode
=
self
.
proc
.
returncode
except
Exception
as
e
:
self
.
exception
=
e
except
subprocess
.
TimeoutExpired
as
t
:
self
.
returncode
=
-
signal
.
SIGXCPU
self
.
out
=
bytearray
()
self
.
err
=
bytearray
()
#except Exception as e:
# self.exception = e
def
run
(
self
):
if
self
.
timeout
>
0.0
:
_LOG
.
debug
(
"run with timeout %.1f: %s"
%
(
float
(
self
.
timeout
),
self
.
cmd
))
thread
=
threading
.
Thread
(
target
=
self
.
_run_process
)
thread
.
start
()
thread
.
join
(
float
(
self
.
timeout
))
if
self
.
exception
is
not
None
:
raise
self
.
exception
if
thread
.
is_alive
():
_LOG
.
debug
(
"timeout %.1f reached, terminate process"
%
self
.
timeout
)
if
self
.
proc
is
not
None
:
self
.
proc
.
terminate
()
thread
.
join
(
1.0
)
if
thread
.
is_alive
():
_LOG
.
debug
(
"termination ignored, now kill process"
)
self
.
proc
.
kill
()
thread
.
join
()
raise
SigKill
(
signal
.
SIGXCPU
,
"SIGXCPU"
)
else
:
self
.
_run_process
()
self
.
_run_process
()
# Usually python can recognize application terminations triggered by
# signals, but somehow it doesn't do this for java (I suspect, that java
...
...
@@ -117,7 +108,7 @@ class _Execute(object):
raise
SigKill
(
-
self
.
returncode
,
_EXIT_CODES
[
self
.
returncode
]
+
": "
+
os
.
strerror
(
-
self
.
returncode
))
return
(
self
.
out
,
self
.
err
,
self
.
returncode
)
def
execute
(
cmd
,
env
=
None
,
timeout
=
0
,
rlimit
=
None
,
propagate_sigint
=
True
):
def
execute
(
cmd
,
env
=
None
,
timeout
=
0
,
rlimit
=
None
,
propagate_sigint
=
True
,
input_str
=
None
):
"""Execute a command and return stderr and stdout data"""
if
not
rlimit
:
rlimit
=
dict
()
...
...
@@ -127,7 +118,7 @@ def execute(cmd, env=None, timeout=0, rlimit=None, propagate_sigint=True):
else
:
#cmd = shlex.split(cmd[0]) + cmd[1:]
cmd
=
" "
.
join
(
shlex
.
quote
(
c
)
for
c
in
cmd
)
exc
=
_Execute
(
cmd
,
timeout
,
env
,
rlimit
)
exc
=
_Execute
(
cmd
,
timeout
,
env
,
rlimit
,
input_str
)
(
out
,
err
,
returncode
)
=
exc
.
run
()
if
returncode
==
-
signal
.
SIGINT
:
raise
KeyboardInterrupt
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment