Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
ufebl
mjtest
Commits
fec32835
Commit
fec32835
authored
Nov 08, 2016
by
Johannes Bechberger
Browse files
Add asts test runner
parent
00d02cb3
Changes
4
Hide whitespace changes
Inline
Side-by-side
README.mdwn
View file @
fec32835
...
...
@@ -16,7 +16,7 @@ The test cases are divided in 5 'modes':
- __lexer__: Test cases that check the lexed token (and their correct output)
- __syntax__: Test cases that just check whether `./run --parsecheck` accepts as correct or rejects
them.
- __ast__: Test cases that check the generated ast.
- __ast__: Test cases that check the generated ast
by using the pretty printing functionality
.
- __semantic__: Test cases that check semantic checking of MiniJava programs
- __exec__: Test cases that check the correct compilation of MiniJava programs.
...
...
@@ -74,14 +74,13 @@ Test types for the ast mode
<tr><th>File ending(s) of test cases</th><th>Expected behaviour to complete a test of this type</th></tr>
<tr>
<td><code>.valid.mj</code> <code>.mj</code>
<td>Return code is <code>0</code> and the output matches the expected output (located in the file `[test file].out`</td>
</tr>
<tr>
<td><code>.invalid.mj</code>
<td>Return code is <code>> 0</code> and the error output contains the word <code>error</code></td>
<td>Pretty printing the source file should result in the same output as pretty printing the already pretty printed file.
The sorted lexer output for the last should be the same as the sorted lexer output for the source file.</td>
</tr>
</table>
It uses all syntax mode tests implicitly.
Test runner
-----------
...
...
mjtest/environment.py
View file @
fec32835
...
...
@@ -20,6 +20,10 @@ class TestMode:
exec
=
"exec"
USE_TESTS_OF_OTHER
=
{
ast
:
[
syntax
]
}
""" All 'success' tests of the n.th mode can used as 'success' tests for the n-1.th mode"""
TEST_MODES
=
[
TestMode
.
lexer
,
TestMode
.
syntax
,
TestMode
.
ast
,
TestMode
.
semantic
,
TestMode
.
exec
]
...
...
@@ -102,7 +106,7 @@ class Environment:
mode_flag
=
{
TestMode
.
lexer
:
"--lextest"
,
TestMode
.
syntax
:
"--parsetest"
,
TestMode
.
ast
:
"--p
arse-as
t"
TestMode
.
ast
:
"--p
retty-prin
t"
}[
mode
]
cmd
=
[
self
.
mj_run_cmd
,
mode_flag
]
+
list
(
args
)
return
execute
(
cmd
,
timeout
=
self
.
timeout
)
...
...
mjtest/test/ast_tests.py
View file @
fec32835
import
difflib
import
os
import
shutil
,
logging
from
typing
import
Tuple
from
mjtest.environment
import
Environment
,
TestMode
from
mjtest.test.syntax_tests
import
BasicSyntaxTest
from
mjtest.test.tests
import
TestCase
,
BasicDiffTestResult
,
BasicTestResult
from
os
import
path
_LOG
=
logging
.
getLogger
(
"tests"
)
class
AST
DiffTest
(
TestCase
):
class
AST
PrettyPrintTest
(
BasicSyntaxTest
):
FILE_ENDINGS
=
[
".invalid.mj"
,
".valid.mj"
,
".mj"
]
OUTPUT_FILE_ENDING
=
".out"
MODE
=
TestMode
.
ast
FILE_ENDINGS
=
[
".mj"
,
".valid.mj"
]
INVALID_FILE_ENDINGS
=
[
".invalid.mj"
]
def
__init__
(
self
,
env
:
Environment
,
type
:
str
,
file
:
str
):
super
().
__init__
(
env
,
type
,
file
)
self
.
_should_succeed
=
not
file
.
endswith
(
".invalid.mj"
)
self
.
_expected_output_file
=
file
+
self
.
OUTPUT_FILE_ENDING
self
.
_has_expected_output_file
=
path
.
exists
(
self
.
_expected_output_file
)
def
should_succeed
(
self
)
->
bool
:
return
self
.
_should_succeed
def
short_name
(
self
)
->
str
:
return
path
.
basename
(
self
.
file
)
def
run
(
self
)
->
BasicDiffTestResult
:
out
,
err
,
rtcode
=
self
.
env
.
run_mj_command
(
self
.
MODE
,
self
.
file
)
exp_out
=
""
if
rtcode
==
0
and
self
.
should_succeed
():
if
self
.
_has_expected_output_file
and
self
.
type
==
self
.
MODE
and
self
.
env
.
mode
==
self
.
MODE
:
with
open
(
self
.
_expected_output_file
,
"r"
)
as
f
:
exp_out
=
f
.
read
()
#else:
# _LOG.error("Expected output file for test case {}:{} is missing.".format(self.MODE, self.short_name()))
if
self
.
type
==
self
.
MODE
and
self
.
env
.
mode
==
self
.
MODE
:
return
BasicDiffTestResult
(
self
,
rtcode
,
out
.
decode
(),
err
.
decode
(),
exp_out
)
return
BasicTestResult
(
self
,
rtcode
,
out
.
decode
(),
err
.
decode
())
class
LexerDiffTest
(
ASTDiffTest
):
MODE
=
TestMode
.
lexer
TestCase
.
TEST_CASE_CLASSES
[
TestMode
.
ast
].
append
(
ASTDiffTest
)
TestCase
.
TEST_CASE_CLASSES
[
TestMode
.
lexer
].
append
(
LexerDiffTest
)
\ No newline at end of file
def
run
(
self
)
->
BasicTestResult
:
tmp_file
=
self
.
env
.
create_tmpfile
()
rtcode
,
out
,
err
=
self
.
_pretty_print
(
self
.
file
,
tmp_file
)
if
rtcode
>
0
:
os
.
remove
(
tmp_file
)
return
BasicTestResult
(
self
,
rtcode
,
out
,
err
)
_file
=
self
.
file
tmp_file2
=
self
.
env
.
create_tmpfile
()
rtcode
,
out2
,
err2
=
self
.
_pretty_print
(
tmp_file
,
tmp_file2
)
if
rtcode
>
0
:
os
.
remove
(
tmp_file2
)
btr
=
BasicTestResult
(
self
,
rtcode
,
out2
,
err2
)
btr
.
add_additional_text
(
"Prior out"
,
out
)
btr
.
add_additional_text
(
"Prior err"
,
err
)
return
btr
rtcode_lex
,
out_lex
,
err_lex
=
self
.
env
.
run_mj_command
(
TestMode
.
lexer
,
self
.
file
)
rtcode_lex2
,
out_lex2
,
err_lex2
=
self
.
env
.
run_mj_command
(
TestMode
.
lexer
,
tmp_file2
)
os
.
remove
(
tmp_file2
)
out_lex
=
self
.
_sort_lexed
(
out_lex
.
decode
())
out_lex2
=
self
.
_sort_lexed
(
out_lex2
.
decode
())
incorrect_msg
,
rtcode
=
""
,
0
if
rtcode_lex
+
rtcode_lex2
:
incorrect_msg
,
rtcode
=
"Lexing failed"
,
1
elif
out
!=
out2
:
incorrect_msg
,
rtcode
=
"Not idempotent"
,
1
elif
out_lex
!=
out_lex2
:
incorrect_msg
,
rtcode
=
"Sorted and lexed second pretty print differs from original"
,
1
btr
=
BasicTestResult
(
self
,
rtcode
,
incorrect_msg
=
incorrect_msg
)
btr
.
add_additional_text
(
"First round output"
,
out
)
btr
.
add_additional_text
(
"Second round output"
,
out2
)
btr
.
add_additional_text
(
"Diff"
,
self
.
_diff
(
out
,
out2
))
btr
.
add_additional_text
(
"Original file, sorted and lexed"
,
out_lex
)
btr
.
add_additional_text
(
"Second round output, sorted and lexed"
,
out_lex2
)
btr
.
add_additional_text
(
"Diff"
,
self
.
_diff
(
out_lex
,
out_lex2
))
return
btr
def
_diff
(
self
,
first
:
str
,
second
:
str
)
->
str
:
return
""
.
join
(
difflib
.
Differ
().
compare
(
first
.
splitlines
(
True
),
second
.
splitlines
(
True
)))
def
_sort_lexed
(
self
,
lexed
:
str
)
->
str
:
#return "".join(difflib.Differ().compare(self.expected_output.splitlines(True), self.output.splitlines(True)))
return
""
.
join
(
sorted
(
lexed
.
splitlines
(
True
)))
def
_pretty_print
(
self
,
input_file
:
str
,
output_file
:
str
)
->
Tuple
[
int
,
str
,
str
]:
out
,
err
,
rtcode
=
self
.
env
.
run_mj_command
(
TestMode
.
ast
,
input_file
)
with
open
(
output_file
,
"w"
)
as
f
:
print
(
out
,
file
=
f
)
return
rtcode
,
out
.
decode
(),
err
.
decode
()
TestCase
.
TEST_CASE_CLASSES
[
"ast"
].
append
(
ASTPrettyPrintTest
)
\ No newline at end of file
mjtest/test/tests.py
View file @
fec32835
...
...
@@ -31,7 +31,11 @@ class TestSuite:
self
.
_load_test_cases
()
def
_load_test_cases
(
self
):
types
=
TEST_MODES
#TEST_MODES[TEST_MODES.index(self.env.mode):]
types
=
[
self
.
env
.
mode
]
#TEST_MODES#TEST_MODES[TEST_MODES.index(self.env.mode):]
if
self
.
env
.
ci_testing
:
types
=
TEST_MODES
elif
self
.
env
.
mode
in
TestMode
.
USE_TESTS_OF_OTHER
:
types
+=
TestMode
.
USE_TESTS_OF_OTHER
[
self
.
env
.
mode
]
for
type
in
types
:
self
.
_load_test_case_type
(
type
)
...
...
@@ -54,21 +58,27 @@ class TestSuite:
if
len
(
t
)
>
0
:
self
.
correct_test_cases
[
mode
].
add
(
t
)
correct_test_cases
.
add
(
t
)
m
=
mode
if
m
!=
self
.
env
.
mode
and
self
.
env
.
mode
in
TestMode
.
USE_TESTS_OF_OTHER
and
\
m
in
TestMode
.
USE_TESTS_OF_OTHER
[
self
.
env
.
mode
]:
m
=
self
.
env
.
mode
for
file
in
sorted
(
os
.
listdir
(
dir
)):
if
not
TestCase
.
has_valid_file_ending
(
m
ode
,
file
):
if
not
TestCase
.
has_valid_file_ending
(
m
,
file
):
_LOG
.
debug
(
"Skip file "
+
file
)
elif
self
.
env
.
only_incorrect_tests
and
file
in
correct_test_cases
:
_LOG
.
info
(
"Skip file {} as its test case was executed correctly the last run"
)
else
:
test_case
=
TestCase
.
create_from_file
(
self
.
env
,
mode
,
join
(
dir
,
file
))
if
not
test_case
.
can_run
():
test_case
=
TestCase
.
create_from_file
(
self
.
env
,
m
,
join
(
dir
,
file
))
if
not
test_case
:
pass
elif
not
test_case
.
can_run
():
_LOG
.
debug
(
"Skip test case '{}' because it isn't suited"
.
format
(
test_case
.
name
()))
else
:
if
m
ode
not
in
self
.
test_cases
:
self
.
test_cases
[
m
ode
]
=
[]
self
.
test_cases
[
m
ode
].
append
(
test_case
)
if
m
ode
in
self
.
test_cases
and
len
(
self
.
test_cases
[
m
ode
])
==
0
:
del
self
.
test_cases
[
m
ode
]
if
m
not
in
self
.
test_cases
:
self
.
test_cases
[
m
]
=
[]
self
.
test_cases
[
m
].
append
(
test_case
)
if
m
in
self
.
test_cases
and
len
(
self
.
test_cases
[
m
])
==
0
:
del
self
.
test_cases
[
m
]
def
_log_file_for_type
(
self
,
type
:
str
):
return
join
(
self
.
env
.
test_dir
,
type
,
".mjtest_correct_testcases_"
+
self
.
env
.
mode
)
...
...
@@ -185,6 +195,7 @@ class TestCase:
TEST_CASE_CLASSES
=
dict
((
k
,
[])
for
k
in
TEST_MODES
)
FILE_ENDINGS
=
[]
INVALID_FILE_ENDINGS
=
[]
def
__init__
(
self
,
env
:
Environment
,
type
:
str
,
file
:
str
):
self
.
env
=
env
...
...
@@ -196,13 +207,14 @@ class TestCase:
def
can_run
(
self
,
mode
:
str
=
""
)
->
bool
:
mode
=
mode
or
self
.
env
.
mode
same_mode
=
self
.
type
==
mode
types
=
TEST_MODES
[
TEST_MODES
.
index
(
self
.
env
.
mode
):]
if
self
.
env
.
ci_testing
:
return
s
elf
.
type
==
mode
or
\
return
s
ame_
mode
or
\
(
self
.
type
in
types
and
self
.
should_succeed
())
or
\
(
self
.
type
not
in
types
and
not
self
.
should_succeed
())
else
:
return
s
elf
.
type
==
mode
return
s
ame_
mode
def
run
(
self
)
->
'TestResult'
:
raise
NotImplementedError
()
...
...
@@ -222,7 +234,8 @@ class TestCase:
@
classmethod
def
_test_case_class_for_file
(
cls
,
type
:
str
,
file
:
str
):
for
t
in
cls
.
TEST_CASE_CLASSES
[
type
]:
if
any
(
file
.
endswith
(
e
)
for
e
in
t
.
FILE_ENDINGS
):
if
any
(
file
.
endswith
(
e
)
for
e
in
t
.
FILE_ENDINGS
)
and
\
not
any
(
file
.
endswith
(
e
)
for
e
in
t
.
INVALID_FILE_ENDINGS
):
return
t
return
False
...
...
@@ -257,12 +270,18 @@ class TestResult:
class
BasicTestResult
(
TestResult
):
def
__init__
(
self
,
test_case
:
TestCase
,
error_code
:
int
,
output
:
str
,
error_output
:
str
):
def
__init__
(
self
,
test_case
:
TestCase
,
error_code
:
int
,
output
:
str
=
None
,
error_output
:
str
=
None
,
incorrect_msg
:
str
=
"incorrect return code"
):
super
().
__init__
(
test_case
,
error_code
)
self
.
_incorrect_msg
=
incorrect_msg
self
.
_contains_error_str
=
"error"
in
error_output
self
.
error_output
=
error_output
self
.
output
=
output
self
.
other_texts
=
[]
# type: List[Tuple[str, str, bool]]
if
output
:
self
.
add_additional_text
(
"Output"
,
output
)
if
error_output
:
self
.
add_additional_text
(
"Error output"
,
error_output
)
def
is_correct
(
self
):
if
self
.
succeeded
():
...
...
@@ -276,7 +295,7 @@ class BasicTestResult(TestResult):
else
:
if
not
self
.
succeeded
()
and
not
self
.
test_case
.
should_succeed
()
and
not
self
.
_contains_error_str
:
return
"the error output doesn't contain the word
\"
error
\"
"
return
"
incorrect
return code"
return
self
.
_
incorrect
_msg
def
long_message
(
self
)
->
str
:
file_content
=
[]
...
...
@@ -299,19 +318,10 @@ Source file:
{}
Output:
{}
Error output:
{}
Return code: {}
{}
"""
.
format
(
self
.
short_message
().
capitalize
(),
self
.
_ident
(
file_content
),
self
.
_ident
(
self
.
output
),
self
.
_ident
(
self
.
error_output
),
self
.
error_code
,
"""
.
format
(
self
.
short_message
().
capitalize
(),
self
.
_ident
(
file_content
),
self
.
error_code
,
"
\n
"
.
join
(
others
))
def
add_additional_text
(
self
,
title
:
str
,
content
:
str
):
...
...
@@ -364,6 +374,43 @@ class BasicDiffTestResult(BasicTestResult):
return
"incorrect return code"
class
DiffTest
(
TestCase
):
FILE_ENDINGS
=
[
".invalid.mj"
,
".valid.mj"
,
".mj"
]
OUTPUT_FILE_ENDING
=
".out"
MODE
=
TestMode
.
ast
def
__init__
(
self
,
env
:
Environment
,
type
:
str
,
file
:
str
):
super
().
__init__
(
env
,
type
,
file
)
self
.
_should_succeed
=
not
file
.
endswith
(
".invalid.mj"
)
self
.
_expected_output_file
=
file
+
self
.
OUTPUT_FILE_ENDING
self
.
_has_expected_output_file
=
exists
(
self
.
_expected_output_file
)
def
should_succeed
(
self
)
->
bool
:
return
self
.
_should_succeed
def
short_name
(
self
)
->
str
:
return
basename
(
self
.
file
)
def
run
(
self
)
->
BasicDiffTestResult
:
out
,
err
,
rtcode
=
self
.
env
.
run_mj_command
(
self
.
MODE
,
self
.
file
)
exp_out
=
""
if
rtcode
==
0
and
self
.
should_succeed
():
if
self
.
_has_expected_output_file
and
self
.
type
==
self
.
MODE
and
self
.
env
.
mode
==
self
.
MODE
:
with
open
(
self
.
_expected_output_file
,
"r"
)
as
f
:
exp_out
=
f
.
read
()
#else:
# _LOG.error("Expected output file for test case {}:{} is missing.".format(self.MODE, self.short_name()))
if
self
.
type
==
self
.
MODE
and
self
.
env
.
mode
==
self
.
MODE
:
return
BasicDiffTestResult
(
self
,
rtcode
,
out
.
decode
(),
err
.
decode
(),
exp_out
)
return
BasicTestResult
(
self
,
rtcode
,
out
.
decode
(),
err
.
decode
())
class
LexerDiffTest
(
DiffTest
):
MODE
=
TestMode
.
lexer
TestCase
.
TEST_CASE_CLASSES
[
TestMode
.
lexer
].
append
(
LexerDiffTest
)
import
mjtest.test.syntax_tests
import
mjtest.test.ast_tests
\ No newline at end of file
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment