Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
uwdkn
mjtest
Commits
caa946ba
Commit
caa946ba
authored
Jan 22, 2017
by
Johannes Bechberger
Browse files
Improve README and add invalid exec test case handler
parent
7d3a9a85
Changes
3
Hide whitespace changes
Inline
Side-by-side
README.mdwn
View file @
caa946ba
...
...
@@ -113,9 +113,23 @@ Test types for the compile-firm and compile mode
if a <code>[test file].out</code> exists: the content of this file<br/>
if no such file exists: the same as the execution of the file with <code>java</code></td>
</tr>
<tr>
<td><code>.[number].inputc</code></td>
<td>This a test case with an input. The input is interpreted as ASCII chars. It requires a corresponding
<code>.input.java</code> or <code>.input.mj</code> file that is executed with this input.
The rest is equivalent to normal test cases from the row above</td>
</tr>
<tr>
<td><code>.[number].input</code></td>
<td>This is an input test case whichs input file contains space seperated decimal numbers that are used as input data.</td>
</tr>
<tr>
<td><code>.invalid.mj</code> <code>.invalid.java</code></td>
<td>The test file should compile. The execution of the resulting binary should abort width a return code <code>> 0</code></td>
</tr>
</table>
__All compile-firm and compile mode tests have to be semantically correct__
_
__All compile-firm and compile mode tests have to be semantically correct__
__All bigger test cases (e.g. for benchmarking) should be placed into the `big` folder__
...
...
mjtest/test/exec_tests.py
View file @
caa946ba
...
...
@@ -24,7 +24,7 @@ class JavaExecTest(BasicSyntaxTest):
"""
FILE_ENDINGS
=
[
".java"
,
".mj"
]
INVALID_FILE_ENDINGS
=
[
".inf.java"
,
".inf.mj"
,
".input.mj"
,
".input.java"
]
INVALID_FILE_ENDINGS
=
[
".inf.java"
,
".inf.mj"
,
".input.mj"
,
".input.java"
,
".invalid.java"
,
".invalid.mj"
]
OUTPUT_FILE_ENDING
=
".out"
MODE
=
TestMode
.
compile_firm
INPUT_FILE_REGEXP
=
r
'(\.[0-9]+)\.input(c?)$'
...
...
@@ -289,7 +289,7 @@ class JavaInfiniteLoopTest(BasicSyntaxTest):
os
.
chdir
(
cwd
)
return
test_result
else
:
test_result
.
incorrect_msg
=
"
sdf
binary can't be run: "
+
sig
.
name
test_result
.
incorrect_msg
=
"binary can't be run: "
+
sig
.
name
test_result
.
set_error_code
(
sig
.
retcode
)
test_result
.
add_file
(
"Source file"
,
self
.
preprocessed_file
)
os
.
chdir
(
cwd
)
...
...
@@ -308,3 +308,86 @@ class JavaInfiniteLoopTest(BasicSyntaxTest):
TestCase
.
TEST_CASE_CLASSES
[
TestMode
.
compile_firm
].
append
(
JavaInfiniteLoopTest
)
class
InvalidJavaExecTest
(
BasicSyntaxTest
):
"""
The compiled binary should fail to execute properly
"""
FILE_ENDINGS
=
[
".invalid.java"
,
".invalid.mj"
]
MODE
=
TestMode
.
compile_firm
def
__init__
(
self
,
env
:
Environment
,
type
:
str
,
file
:
str
,
preprocessed_file
:
str
,
log_file_mode
:
str
):
super
().
__init__
(
env
,
type
,
file
,
preprocessed_file
,
log_file_mode
)
self
.
_should_succeed
=
False
def
run
(
self
)
->
BasicDiffTestResult
:
is_big_testcase
=
"big"
in
self
.
file
timeout
=
self
.
env
.
big_timeout
if
is_big_testcase
else
self
.
env
.
timeout
base_filename
=
path
.
basename
(
self
.
file
).
split
(
"."
)[
0
]
tmp_dir
=
self
.
env
.
create_pid_local_tmpdir
()
shutil
.
copy
(
self
.
preprocessed_file
,
path
.
join
(
tmp_dir
,
base_filename
+
".java"
))
cwd
=
os
.
getcwd
()
os
.
chdir
(
tmp_dir
)
#print(base_filename, get_main_class_name(base_filename + ".java"))
test_result
=
ExtensibleTestResult
(
self
)
test_result
.
require_error_string_in_error_case
=
False
try
:
out
,
err
,
rtcode
=
None
,
None
,
None
try
:
out
,
err
,
rtcode
=
self
.
env
.
run_mj_command
(
self
.
MODE
,
base_filename
+
".java"
,
timeout
=-
1
)
if
rtcode
!=
0
:
test_result
.
incorrect_msg
=
"file can't be compiled"
test_result
.
set_error_code
(
-
rtcode
)
test_result
.
add_long_text
(
"Error output"
,
err
.
decode
())
test_result
.
add_long_text
(
"Output"
,
out
.
decode
())
test_result
.
add_file
(
"Source file"
,
self
.
preprocessed_file
)
os
.
chdir
(
cwd
)
return
test_result
except
SigKill
as
sig
:
test_result
.
incorrect_msg
=
"file can't be compiled: "
+
sig
.
name
test_result
.
set_error_code
(
sig
.
retcode
)
test_result
.
add_file
(
"Source file"
,
self
.
preprocessed_file
)
os
.
chdir
(
cwd
)
return
test_result
except
:
os
.
chdir
(
cwd
)
raise
try
:
out
,
err
,
rtcode
=
self
.
env
.
run_command
(
"./a.out"
,
timeout
=
timeout
)
if
rtcode
!=
0
:
test_result
.
incorrect_msg
=
"binary can't be run, non zero error code"
test_result
.
set_error_code
(
rtcode
)
test_result
.
add_long_text
(
"Error output"
,
err
.
decode
())
test_result
.
add_long_text
(
"Output"
,
out
.
decode
())
test_result
.
add_file
(
"Source file"
,
self
.
preprocessed_file
)
os
.
chdir
(
cwd
)
return
test_result
except
SigKill
as
sig
:
test_result
.
incorrect_msg
=
"binary can't be run: "
+
sig
.
name
.
strip
()
test_result
.
set_error_code
(
sig
.
retcode
)
test_result
.
add_file
(
"Source file"
,
self
.
preprocessed_file
)
os
.
chdir
(
cwd
)
return
test_result
except
:
os
.
chdir
(
cwd
)
raise
test_result
=
BasicTestResult
(
self
,
rtcode
,
out
.
decode
(),
err
.
decode
())
test_result
.
require_error_string_in_error_case
=
False
return
test_result
except
SigKill
as
sig
:
os
.
chdir
(
cwd
)
assert
False
except
BaseException
:
os
.
chdir
(
cwd
)
raise
TestCase
.
TEST_CASE_CLASSES
[
TestMode
.
compile_firm
].
append
(
InvalidJavaExecTest
)
class
InvalidJavaCompileExecTest
(
InvalidJavaExecTest
):
MODE
=
TestMode
.
compile
TestCase
.
TEST_CASE_CLASSES
[
TestMode
.
compile
].
append
(
InvalidJavaExecTest
)
mjtest/test/tests.py
View file @
caa946ba
...
...
@@ -439,6 +439,7 @@ class BasicTestResult(TestResult):
if
error_output
:
self
.
add_additional_text
(
"Error output"
,
error_output
)
self
.
has_succeeded
=
error_code
==
0
self
.
require_error_string_in_error_case
=
True
def
succeeded
(
self
):
return
self
.
has_succeeded
...
...
@@ -447,13 +448,14 @@ class BasicTestResult(TestResult):
if
self
.
succeeded
():
return
super
().
is_correct
()
else
:
return
super
().
is_correct
()
and
self
.
_contains_error_str
return
super
().
is_correct
()
and
(
self
.
_contains_error_str
or
not
self
.
require_error_string_in_error_case
)
def
short_message
(
self
)
->
str
:
if
self
.
is_correct
():
return
"correct"
else
:
if
not
self
.
succeeded
()
and
not
self
.
test_case
.
should_succeed
()
and
not
self
.
_contains_error_str
:
if
not
self
.
succeeded
()
and
not
self
.
test_case
.
should_succeed
()
and
not
self
.
_contains_error_str
and
\
not
self
.
require_error_string_in_error_case
:
return
"the error output doesn't contain the word
\"
error
\"
"
return
self
.
_incorrect_msg
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment