Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
ufebl
mjtest
Commits
d3ccbe56
Commit
d3ccbe56
authored
Nov 03, 2016
by
Johannes Bechberger
Browse files
Add `ast` mode
parent
4bd5b32b
Changes
4
Hide whitespace changes
Inline
Side-by-side
README.mdwn
View file @
d3ccbe56
...
...
@@ -15,6 +15,7 @@ Test modes
The test cases are divided in 3 'modes':
- __syntax__: Test cases that just check whether `./run --parsecheck` accepts as correct or rejects
them.
- __ast__: Test cases that check the generated ast.
- __semantic__: Test cases that check semantic checking of MiniJava programs
- __exec__: Test cases that check the correct compilation of MiniJava programs.
...
...
@@ -27,13 +28,14 @@ The different types a test cases are differentiated by their file endings.
Side note: An error code greater than 0 should result in an errror message on error output containing the word `error`.
Test types for the syntax mode
------------------------------
<table>
<tr><th>File ending(s) of test cases</th><th>Expected behaviour to complete a test of this type</th></tr>
<tr>
<td><code>.valid.mj</code><code>.mj</code>
<td><code>.valid.mj</code>
<code>.mj</code>
<td>Return code is <code>0</code>, i.e. the MiniJava is accepted as syntactically correct</td>
</tr>
<tr>
...
...
@@ -47,6 +49,23 @@ Test types for the syntax mode
</tr>
</table>
Test types for the ast mode
------------------------------
<table>
<tr><th>File ending(s) of test cases</th><th>Expected behaviour to complete a test of this type</th></tr>
<tr>
<td><code>.valid.mj</code> <code>.mj</code>
<td>Return code is <code>0</code> and the output matches the expected output (located in the file `[test file].out`</td>
</tr>
<tr>
<td><code>.invalid.mj</code>
<td>Return code is <code>> 0</code> and the error output contains the word <code>error</code></td>
</tr>
</table>
Test runner
-----------
...
...
@@ -73,12 +92,12 @@ Output of the `./mjt.py --help`
```
usage: mjt.py [-h] [--only_incorrect_tests] [--produce_no_reports]
[--parallel] [--log_level LOG_LEVEL]
{syntax,semantic,exec} MJ_RUN
{syntax,
ast,
semantic,exec} MJ_RUN
MiniJava test runner
positional arguments:
{syntax,semantic,exec}
{syntax,
ast,
semantic,exec}
What do you want to test?
MJ_RUN Command to run your MiniJava implementation, e.g.
`mj/run`, can be omitted by assigning the environment
...
...
mjtest/environment.py
View file @
d3ccbe56
...
...
@@ -11,12 +11,14 @@ from typing import Tuple, List
class
TestMode
:
syntax
=
"syntax"
ast
=
"ast"
semantic
=
"semantic"
exec
=
"exec"
""" All 'success' tests of the n.th mode can used as 'success' tests for the n-1.th mode"""
TEST_MODES
=
[
TestMode
.
syntax
,
TestMode
.
semantic
,
TestMode
.
exec
]
TEST_MODES
=
[
TestMode
.
syntax
,
TestMode
.
ast
,
TestMode
.
semantic
,
TestMode
.
exec
]
class
Environment
:
...
...
@@ -49,7 +51,7 @@ class Environment:
self
.
test_dir
=
os
.
path
.
join
(
get_mjtest_basedir
(),
"tests"
)
if
not
os
.
path
.
exists
(
self
.
test_dir
):
os
.
mkdir
(
self
.
test_dir
)
for
d
in
[
TestMode
.
syntax
,
TestMode
.
semantic
,
TestMode
.
exec
]
:
for
d
in
TEST_MODES
:
os
.
mkdir
(
os
.
path
.
join
(
self
.
test_dir
,
d
))
self
.
only_incorrect_tests
=
only_incorrect_tests
...
...
@@ -92,7 +94,8 @@ class Environment:
:return: (out, err, return code)
"""
mode_flag
=
{
TestMode
.
syntax
:
"--parsetest"
TestMode
.
syntax
:
"--parsetest"
,
TestMode
.
ast
:
"--parse-ast"
}[
mode
]
cmd
=
[
self
.
mj_run_cmd
,
mode_flag
]
+
list
(
args
)
return
execute
(
cmd
,
timeout
=
self
.
timeout
)
...
...
mjtest/test/ast_tests.py
0 → 100644
View file @
d3ccbe56
import
shutil
,
logging
from
mjtest.environment
import
Environment
,
TestMode
from
mjtest.test.tests
import
TestCase
,
BasicDiffTestResult
from
os
import
path
_LOG
=
logging
.
getLogger
(
"tests"
)
class
ASTDiffTest
(
TestCase
):
FILE_ENDINGS
=
[
".invalid.mj"
,
".valid.mj"
,
".mj"
]
OUTPUT_FILE_ENDING
=
".out"
MODE
=
TestMode
.
ast
def
__init__
(
self
,
env
:
Environment
,
type
:
str
,
file
:
str
):
super
().
__init__
(
env
,
type
,
file
)
self
.
_should_succeed
=
not
file
.
endswith
(
".invalid.mj"
)
self
.
_expected_output_file
=
file
+
self
.
OUTPUT_FILE_ENDING
self
.
_has_expected_output_file
=
path
.
exists
(
self
.
_expected_output_file
)
def
should_succeed
(
self
)
->
bool
:
return
self
.
_should_succeed
def
short_name
(
self
)
->
str
:
return
path
.
basename
(
self
.
file
)
def
run
(
self
)
->
BasicDiffTestResult
:
out
,
err
,
rtcode
=
self
.
env
.
run_mj_command
(
self
.
MODE
,
self
.
file
)
exp_out
=
""
if
rtcode
>
0
and
self
.
should_succeed
():
if
self
.
_has_expected_output_file
:
with
open
(
self
.
_expected_output_file
,
"r"
)
as
f
:
exp_out
=
f
.
read
()
else
:
_LOG
.
error
(
"Expected output file for test case {}:{} is missing."
.
format
(
self
.
MODE
,
self
.
short_name
()))
return
BasicDiffTestResult
(
self
,
rtcode
,
out
.
decode
(),
err
.
decode
(),
exp_out
)
TestCase
.
TEST_CASE_CLASSES
[
TestMode
.
ast
].
append
(
ASTDiffTest
)
\ No newline at end of file
mjtest/test/tests.py
View file @
d3ccbe56
...
...
@@ -11,6 +11,7 @@ from mjtest.util.parallelism import available_cpu_count
from
mjtest.util.utils
import
cprint
,
colored
from
pprint
import
pprint
import
shutil
import
difflib
_LOG
=
logging
.
getLogger
(
"tests"
)
...
...
@@ -40,7 +41,7 @@ class TestSuite:
if
exists
(
dir
):
self
.
_load_test_case_dir
(
type
,
dir
)
else
:
_LOG
.
warn
(
"Test folder {} doesn't exist"
.
format
(
dir
))
_LOG
.
info
(
"Test folder {} doesn't exist"
.
format
(
dir
))
def
_load_test_case_dir
(
self
,
mode
:
str
,
dir
:
str
):
self
.
test_cases
[
mode
]
=
[]
...
...
@@ -175,10 +176,7 @@ class TestCase:
A single test case.
"""
TEST_CASE_CLASSES
=
{
TestMode
.
syntax
:
[],
TestMode
.
semantic
:
[]
}
TEST_CASE_CLASSES
=
dict
((
k
,
[])
for
k
in
TEST_MODES
)
FILE_ENDINGS
=
[]
def
__init__
(
self
,
env
:
Environment
,
type
:
str
,
file
:
str
):
...
...
@@ -192,14 +190,8 @@ class TestCase:
def
can_run
(
self
,
mode
:
str
=
""
)
->
bool
:
mode
=
mode
or
self
.
env
.
mode
if
mode
==
TestMode
.
exec
:
return
self
.
type
==
TestMode
.
exec
if
mode
==
TestMode
.
semantic
:
return
self
.
type
==
TestMode
.
semantic
\
or
(
self
.
type
==
TestMode
.
exec
and
self
.
should_succeed
())
if
mode
==
TestMode
.
syntax
:
return
self
.
type
==
TestMode
.
syntax
or
\
(
self
.
can_run
(
TestMode
.
semantic
)
and
self
.
should_succeed
())
return
self
.
type
==
mode
or
\
(
self
.
type
in
TEST_MODES
[
TEST_MODES
.
index
(
self
.
env
.
mode
):]
and
self
.
should_succeed
())
def
run
(
self
)
->
'TestResult'
:
raise
NotImplementedError
()
...
...
@@ -324,4 +316,43 @@ Return code: {}
arr
=
[
"[{:04d}] {:s}"
.
format
(
i
+
1
,
l
)
for
(
i
,
l
)
in
enumerate
(
arr
)]
return
"
\n
"
.
join
(
arr
)
import
mjtest.test.syntax_tests
\ No newline at end of file
class
BasicDiffTestResult
(
BasicTestResult
):
def
__init__
(
self
,
test_case
:
TestCase
,
error_code
:
int
,
output
:
str
,
error_output
:
str
,
expected_output
:
str
):
super
().
__init__
(
test_case
,
error_code
,
output
,
error_output
)
self
.
expected_output
=
expected_output
self
.
_is_output_correct
=
self
.
expected_output
.
strip
()
==
self
.
output
if
self
.
is_correct
():
self
.
add_additional_text
(
"Expected and actual output"
,
self
.
output
)
elif
self
.
succeeded
()
and
self
.
test_case
.
should_succeed
():
self
.
add_additional_text
(
"Diff[expected output, actual output]"
,
self
.
_output_diff
())
self
.
add_additional_text
(
"Expected output"
,
self
.
expected_output
)
self
.
add_additional_text
(
"Actual output"
,
self
.
output
)
def
is_correct
(
self
):
if
self
.
succeeded
():
return
super
().
is_correct
()
and
self
.
is_output_correct
()
else
:
return
super
().
is_correct
()
and
self
.
_contains_error_str
def
_output_diff
(
self
)
->
str
:
return
difflib
.
Differ
().
compare
(
self
.
expected_output
,
self
.
output
)
def
is_output_correct
(
self
)
->
str
:
return
self
.
_is_output_correct
def
short_message
(
self
)
->
str
:
if
self
.
is_correct
():
return
"correct"
else
:
if
not
self
.
succeeded
()
and
not
self
.
test_case
.
should_succeed
()
and
not
self
.
_contains_error_str
:
return
"the error output doesn't contain the word
\"
error
\"
"
if
self
.
succeeded
()
and
self
.
test_case
.
should_succeed
():
return
"the actual output differs from the expected"
return
"incorrect return code"
import
mjtest.test.syntax_tests
import
mjtest.test.ast_tests
\ No newline at end of file
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment