Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
10
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Open sidebar
uceme
mjtest
Commits
d3ccbe56
Commit
d3ccbe56
authored
Nov 03, 2016
by
Johannes Bechberger
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Add `ast` mode
parent
4bd5b32b
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
111 additions
and
20 deletions
+111
-20
README.mdwn
README.mdwn
+22
-3
mjtest/environment.py
mjtest/environment.py
+6
-3
mjtest/test/ast_tests.py
mjtest/test/ast_tests.py
+38
-0
mjtest/test/tests.py
mjtest/test/tests.py
+45
-14
No files found.
README.mdwn
View file @
d3ccbe56
...
...
@@ -15,6 +15,7 @@ Test modes
The test cases are divided in 3 'modes':
- __syntax__: Test cases that just check whether `./run --parsecheck` accepts as correct or rejects
them.
- __ast__: Test cases that check the generated ast.
- __semantic__: Test cases that check semantic checking of MiniJava programs
- __exec__: Test cases that check the correct compilation of MiniJava programs.
...
...
@@ -27,13 +28,14 @@ The different types a test cases are differentiated by their file endings.
Side note: An error code greater than 0 should result in an errror message on error output containing the word `error`.
Test types for the syntax mode
------------------------------
<table>
<tr><th>File ending(s) of test cases</th><th>Expected behaviour to complete a test of this type</th></tr>
<tr>
<td><code>.valid.mj</code><code>.mj</code>
<td><code>.valid.mj</code>
<code>.mj</code>
<td>Return code is <code>0</code>, i.e. the MiniJava is accepted as syntactically correct</td>
</tr>
<tr>
...
...
@@ -47,6 +49,23 @@ Test types for the syntax mode
</tr>
</table>
Test types for the ast mode
------------------------------
<table>
<tr><th>File ending(s) of test cases</th><th>Expected behaviour to complete a test of this type</th></tr>
<tr>
<td><code>.valid.mj</code> <code>.mj</code>
<td>Return code is <code>0</code> and the output matches the expected output (located in the file `[test file].out`</td>
</tr>
<tr>
<td><code>.invalid.mj</code>
<td>Return code is <code>> 0</code> and the error output contains the word <code>error</code></td>
</tr>
</table>
Test runner
-----------
...
...
@@ -73,12 +92,12 @@ Output of the `./mjt.py --help`
```
usage: mjt.py [-h] [--only_incorrect_tests] [--produce_no_reports]
[--parallel] [--log_level LOG_LEVEL]
{syntax,semantic,exec} MJ_RUN
{syntax,
ast,
semantic,exec} MJ_RUN
MiniJava test runner
positional arguments:
{syntax,semantic,exec}
{syntax,
ast,
semantic,exec}
What do you want to test?
MJ_RUN Command to run your MiniJava implementation, e.g.
`mj/run`, can be omitted by assigning the environment
...
...
mjtest/environment.py
View file @
d3ccbe56
...
...
@@ -11,12 +11,14 @@ from typing import Tuple, List
class
TestMode
:
syntax
=
"syntax"
ast
=
"ast"
semantic
=
"semantic"
exec
=
"exec"
""" All 'success' tests of the n.th mode can used as 'success' tests for the n-1.th mode"""
TEST_MODES
=
[
TestMode
.
syntax
,
TestMode
.
semantic
,
TestMode
.
exec
]
TEST_MODES
=
[
TestMode
.
syntax
,
TestMode
.
ast
,
TestMode
.
semantic
,
TestMode
.
exec
]
class
Environment
:
...
...
@@ -49,7 +51,7 @@ class Environment:
self
.
test_dir
=
os
.
path
.
join
(
get_mjtest_basedir
(),
"tests"
)
if
not
os
.
path
.
exists
(
self
.
test_dir
):
os
.
mkdir
(
self
.
test_dir
)
for
d
in
[
TestMode
.
syntax
,
TestMode
.
semantic
,
TestMode
.
exec
]
:
for
d
in
TEST_MODES
:
os
.
mkdir
(
os
.
path
.
join
(
self
.
test_dir
,
d
))
self
.
only_incorrect_tests
=
only_incorrect_tests
...
...
@@ -92,7 +94,8 @@ class Environment:
:return: (out, err, return code)
"""
mode_flag
=
{
TestMode
.
syntax
:
"--parsetest"
TestMode
.
syntax
:
"--parsetest"
,
TestMode
.
ast
:
"--parse-ast"
}[
mode
]
cmd
=
[
self
.
mj_run_cmd
,
mode_flag
]
+
list
(
args
)
return
execute
(
cmd
,
timeout
=
self
.
timeout
)
...
...
mjtest/test/ast_tests.py
0 → 100644
View file @
d3ccbe56
import
shutil
,
logging
from
mjtest.environment
import
Environment
,
TestMode
from
mjtest.test.tests
import
TestCase
,
BasicDiffTestResult
from
os
import
path
_LOG
=
logging
.
getLogger
(
"tests"
)
class
ASTDiffTest
(
TestCase
):
FILE_ENDINGS
=
[
".invalid.mj"
,
".valid.mj"
,
".mj"
]
OUTPUT_FILE_ENDING
=
".out"
MODE
=
TestMode
.
ast
def
__init__
(
self
,
env
:
Environment
,
type
:
str
,
file
:
str
):
super
().
__init__
(
env
,
type
,
file
)
self
.
_should_succeed
=
not
file
.
endswith
(
".invalid.mj"
)
self
.
_expected_output_file
=
file
+
self
.
OUTPUT_FILE_ENDING
self
.
_has_expected_output_file
=
path
.
exists
(
self
.
_expected_output_file
)
def
should_succeed
(
self
)
->
bool
:
return
self
.
_should_succeed
def
short_name
(
self
)
->
str
:
return
path
.
basename
(
self
.
file
)
def
run
(
self
)
->
BasicDiffTestResult
:
out
,
err
,
rtcode
=
self
.
env
.
run_mj_command
(
self
.
MODE
,
self
.
file
)
exp_out
=
""
if
rtcode
>
0
and
self
.
should_succeed
():
if
self
.
_has_expected_output_file
:
with
open
(
self
.
_expected_output_file
,
"r"
)
as
f
:
exp_out
=
f
.
read
()
else
:
_LOG
.
error
(
"Expected output file for test case {}:{} is missing."
.
format
(
self
.
MODE
,
self
.
short_name
()))
return
BasicDiffTestResult
(
self
,
rtcode
,
out
.
decode
(),
err
.
decode
(),
exp_out
)
TestCase
.
TEST_CASE_CLASSES
[
TestMode
.
ast
].
append
(
ASTDiffTest
)
\ No newline at end of file
mjtest/test/tests.py
View file @
d3ccbe56
...
...
@@ -11,6 +11,7 @@ from mjtest.util.parallelism import available_cpu_count
from
mjtest.util.utils
import
cprint
,
colored
from
pprint
import
pprint
import
shutil
import
difflib
_LOG
=
logging
.
getLogger
(
"tests"
)
...
...
@@ -40,7 +41,7 @@ class TestSuite:
if
exists
(
dir
):
self
.
_load_test_case_dir
(
type
,
dir
)
else
:
_LOG
.
warn
(
"Test folder {} doesn't exist"
.
format
(
dir
))
_LOG
.
info
(
"Test folder {} doesn't exist"
.
format
(
dir
))
def
_load_test_case_dir
(
self
,
mode
:
str
,
dir
:
str
):
self
.
test_cases
[
mode
]
=
[]
...
...
@@ -175,10 +176,7 @@ class TestCase:
A single test case.
"""
TEST_CASE_CLASSES
=
{
TestMode
.
syntax
:
[],
TestMode
.
semantic
:
[]
}
TEST_CASE_CLASSES
=
dict
((
k
,
[])
for
k
in
TEST_MODES
)
FILE_ENDINGS
=
[]
def
__init__
(
self
,
env
:
Environment
,
type
:
str
,
file
:
str
):
...
...
@@ -192,14 +190,8 @@ class TestCase:
def
can_run
(
self
,
mode
:
str
=
""
)
->
bool
:
mode
=
mode
or
self
.
env
.
mode
if
mode
==
TestMode
.
exec
:
return
self
.
type
==
TestMode
.
exec
if
mode
==
TestMode
.
semantic
:
return
self
.
type
==
TestMode
.
semantic
\
or
(
self
.
type
==
TestMode
.
exec
and
self
.
should_succeed
())
if
mode
==
TestMode
.
syntax
:
return
self
.
type
==
TestMode
.
syntax
or
\
(
self
.
can_run
(
TestMode
.
semantic
)
and
self
.
should_succeed
())
return
self
.
type
==
mode
or
\
(
self
.
type
in
TEST_MODES
[
TEST_MODES
.
index
(
self
.
env
.
mode
):]
and
self
.
should_succeed
())
def
run
(
self
)
->
'TestResult'
:
raise
NotImplementedError
()
...
...
@@ -324,4 +316,43 @@ Return code: {}
arr
=
[
"[{:04d}] {:s}"
.
format
(
i
+
1
,
l
)
for
(
i
,
l
)
in
enumerate
(
arr
)]
return
"
\n
"
.
join
(
arr
)
import
mjtest.test.syntax_tests
\ No newline at end of file
class
BasicDiffTestResult
(
BasicTestResult
):
def
__init__
(
self
,
test_case
:
TestCase
,
error_code
:
int
,
output
:
str
,
error_output
:
str
,
expected_output
:
str
):
super
().
__init__
(
test_case
,
error_code
,
output
,
error_output
)
self
.
expected_output
=
expected_output
self
.
_is_output_correct
=
self
.
expected_output
.
strip
()
==
self
.
output
if
self
.
is_correct
():
self
.
add_additional_text
(
"Expected and actual output"
,
self
.
output
)
elif
self
.
succeeded
()
and
self
.
test_case
.
should_succeed
():
self
.
add_additional_text
(
"Diff[expected output, actual output]"
,
self
.
_output_diff
())
self
.
add_additional_text
(
"Expected output"
,
self
.
expected_output
)
self
.
add_additional_text
(
"Actual output"
,
self
.
output
)
def
is_correct
(
self
):
if
self
.
succeeded
():
return
super
().
is_correct
()
and
self
.
is_output_correct
()
else
:
return
super
().
is_correct
()
and
self
.
_contains_error_str
def
_output_diff
(
self
)
->
str
:
return
difflib
.
Differ
().
compare
(
self
.
expected_output
,
self
.
output
)
def
is_output_correct
(
self
)
->
str
:
return
self
.
_is_output_correct
def
short_message
(
self
)
->
str
:
if
self
.
is_correct
():
return
"correct"
else
:
if
not
self
.
succeeded
()
and
not
self
.
test_case
.
should_succeed
()
and
not
self
.
_contains_error_str
:
return
"the error output doesn't contain the word
\"
error
\"
"
if
self
.
succeeded
()
and
self
.
test_case
.
should_succeed
():
return
"the actual output differs from the expected"
return
"incorrect return code"
import
mjtest.test.syntax_tests
import
mjtest.test.ast_tests
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment