Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
10
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Open sidebar
uceme
mjtest
Commits
fbbd2749
Commit
fbbd2749
authored
Jan 16, 2019
by
Johannes Bucher
Browse files
Options
Browse Files
Download
Plain Diff
Update mjtest and mjtest-tests
Merge remote-tracking branch 'gitlab-original/master'
parents
d771772b
529180eb
Changes
12
Hide whitespace changes
Inline
Side-by-side
Showing
12 changed files
with
160 additions
and
102 deletions
+160
-102
README.md
README.md
+21
-5
mjtest/cli.py
mjtest/cli.py
+8
-1
mjtest/environment.py
mjtest/environment.py
+14
-6
mjtest/test/ast_tests.py
mjtest/test/ast_tests.py
+3
-1
mjtest/test/bench.py
mjtest/test/bench.py
+6
-6
mjtest/test/exec_tests.py
mjtest/test/exec_tests.py
+70
-53
mjtest/test/syntax_tests.py
mjtest/test/syntax_tests.py
+4
-2
mjtest/test/tests.py
mjtest/test/tests.py
+12
-12
mjtest/util/shell.py
mjtest/util/shell.py
+9
-10
mjtest/util/utils.py
mjtest/util/utils.py
+9
-2
preproc/preproc/preprocessor.py
preproc/preproc/preprocessor.py
+3
-3
tests
tests
+1
-1
No files found.
README.md
View file @
fbbd2749
...
...
@@ -23,8 +23,10 @@ The test cases are divided in 6 'modes':
them.
-
__ast__: Test cases that check the generated ast by using the pretty printing functionality.
-
__semantic__: Test cases that check semantic checking of MiniJava programs
-
__compile-firm__: Test cases that check the correct compilation and execution of MiniJava programs with the libfirm backend.
-
__compile__: Test cases that check the correct compilation and execution of MiniJava programs with the self implemented backend.
-
__compile-firm-only__: Test cases that check the compilation of MiniJava programs with the libfirm backend.
-
__compile-firm__ (alias
`exec-firm`
): Test cases that check the correct compilation and execution of MiniJava programs with the libfirm backend.
-
__compile-only__: Test cases that check the compilation MiniJava programs with the self implemented backend.
-
__compile__ (alias
`exec`
): Test cases that check the correct compilation and execution of MiniJava programs with the self implemented backend.
The test different test cases for each mode are located in a folder with the same name.
Except the compile-firm test cases which are located in the
`exec`
folder.
...
...
@@ -103,6 +105,19 @@ Test types for the semantic mode
__All semantic mode tests have to be syntactically correct__
Test types for the compile-firm-only and compile-only mode
----------------------------------------------------------
<table>
<tr><th>
File ending(s) of test cases
</th><th>
Expected behaviour to complete a test of this type
</th></tr>
<tr>
<td><code>
.valid.mj
</code>
<code>
.mj
</code>
<code>
.valid.java
</code>
<code>
.java
</code></td>
<td>
Return code is
<code>
0
</code>
and the compiler produces an
`a.out`
file, i.e. the MiniJava code can be compiled
</td>
</tr>
</table>
__
All valid semantic test-cases are also used, the non-semantic compile-(firm)-only
should be placed into the
`compile-only`
folder__
Test types for the compile-firm and compile mode
------------------------------------------------
...
...
@@ -112,7 +127,7 @@ Test types for the compile-firm and compile mode
<tr>
<td><code>
.java
</code>
<code>
.mj
</code></td>
<td>
The test file should compile. The resulting binary should be runnable and its output should be:
<br/>
if a
<code>
[test file].out
</code>
exists: the content of this file
<br/>
if a
<code>
[input file].out
</code>
(for test cases with input) or
<code>
[test file].out
</code>
(else)
exists: the content of this file
<br/>
if no such file exists: the same as the execution of the file with
<code>
java
</code></td>
</tr>
<tr>
...
...
@@ -171,12 +186,13 @@ usage: mjt.py [-h] [--only_incorrect_tests] [--all_exec_tests]
[
--produce_no_reports
]
[
--produce_all_reports
]
[
--parallel
]
[
--output_no_incorrect_reports
]
[
--color
]
[
--ci_testing
]
[
--log_level
LOG_LEVEL]
{
all,lexer,syntax,ast,semantic,compile-firm,compile
}
MJ_RUN
{
all,lexer,syntax,ast,semantic,compile-firm-only,compile-only,
compile-firm,compile,exec,exec-firm
}
MJ_RUN
MiniJava
test
runner
positional arguments:
{
all,lexer,syntax,ast,semantic,compile-firm
,exec
}
{
all,lexer,syntax,ast,semantic,compile-firm
-only,compile-only,compile-firm,exec,exec-firm
}
What
do
you want to
test
?
MJ_RUN Command to run your MiniJava implementation, e.g.
`
mj/run
`
, can be omitted by assigning the environment
...
...
mjtest/cli.py
View file @
fbbd2749
...
...
@@ -27,7 +27,7 @@ class LogLevelChoices(argparse.Action):
if
True
:
#__name__ == '__main__':
parser
=
argparse
.
ArgumentParser
(
description
=
"MiniJava test runner"
,
add_help
=
True
)
parser
.
add_argument
(
"mode"
,
choices
=
[
"all"
]
+
TEST_MODES
,
choices
=
[
"all"
]
+
TEST_MODES
+
[
"exec-firm"
]
,
help
=
"What do you want to test?"
)
if
os
.
getenv
(
"MJ_RUN"
,
None
)
is
None
:
parser
.
add_argument
(
"mj_run"
,
...
...
@@ -84,9 +84,16 @@ if True:#__name__ == '__main__':
if
ret
is
not
None
:
failed
+=
ret
.
failed
count
+=
ret
.
count
if
args
[
"mode"
]
in
[
"exec"
,
"exec-firm"
]:
new_mode
=
args
[
"mode"
].
replace
(
"exec"
,
"compile"
)
print
(
colored
(
"
\"
{}
\"
is just an alias for
\"
{}
\"
, using the latter instead"
.
format
(
args
[
"mode"
],
new_mode
),
"yellow"
))
args
[
"mode"
]
=
new_mode
if
args
[
"mode"
]
==
"all"
:
report_subdir
=
datetime
.
now
().
strftime
(
"%d-%m-%y_%H-%M-%S"
)
for
mode
in
[
TestMode
.
lexer
,
TestMode
.
syntax
,
TestMode
.
ast
,
TestMode
.
semantic
,
TestMode
.
compile_firm_only
,
TestMode
.
compile_only
,
TestMode
.
compile_firm
,
TestMode
.
compile
]:
args
[
"all_exec_tests"
]
=
True
args
[
"report_subdir"
]
=
report_subdir
+
"_"
+
mode
...
...
mjtest/environment.py
View file @
fbbd2749
...
...
@@ -34,6 +34,10 @@ class TestMode:
compile
=
"compile"
compile_firm_only
=
"compile-firm-only"
compile_only
=
"compile-only"
exec
=
"exec"
bench
=
"bench"
...
...
@@ -42,12 +46,14 @@ class TestMode:
ast
:
[
syntax
],
compile_firm
:
[
exec
],
compile
:
[
exec
],
compile_firm_only
:
[
semantic
,
compile_only
],
compile_only
:
[
semantic
],
bench
:
[
exec
]
}
""" All 'success' tests of the n.th mode can used as 'success' tests for the n-1.th mode"""
TEST_MODES
=
[
TestMode
.
lexer
,
TestMode
.
syntax
,
TestMode
.
ast
,
TestMode
.
semantic
,
TestMode
.
compile_firm
,
TestMode
.
exec
,
TestMode
.
bench
,
TestMode
.
compile
]
TEST_MODES
=
[
TestMode
.
lexer
,
TestMode
.
syntax
,
TestMode
.
ast
,
TestMode
.
semantic
,
TestMode
.
compile_firm
_only
,
TestMode
.
compile_only
,
TestMode
.
compile_firm
,
TestMode
.
exec
,
TestMode
.
bench
,
TestMode
.
compile
]
def
get_test_dirname
(
mode
:
str
)
->
str
:
...
...
@@ -163,8 +169,10 @@ class Environment:
TestMode
.
syntax
:
"--parsetest"
,
TestMode
.
ast
:
"--print-ast"
,
TestMode
.
semantic
:
"--check"
,
TestMode
.
compile_only
:
None
,
TestMode
.
compile_firm_only
:
"--compile-firm"
,
TestMode
.
compile_firm
:
"--compile-firm"
,
TestMode
.
compile
:
None
TestMode
.
compile
:
None
,
}[
mode
]
cmd
=
[
self
.
mj_run_cmd
]
if
mode_flag
:
...
...
@@ -172,13 +180,13 @@ class Environment:
cmd
.
extend
(
list
(
args
))
return
execute
(
cmd
,
timeout
=
timeout
or
self
.
timeout
)
def
run_command
(
self
,
cmd
:
str
,
*
args
:
Tuple
[
str
],
timeout
:
float
=
None
,
input_
str
:
Optional
[
str
]
=
None
)
->
Tuple
[
bytes
,
bytes
,
int
]:
def
run_command
(
self
,
cmd
:
str
,
*
args
:
Tuple
[
str
],
timeout
:
float
=
None
,
input_
bytes
:
Optional
[
bytes
]
=
None
)
->
Tuple
[
bytes
,
bytes
,
int
]:
"""
Execute the passend command with its arguments
:return: (out, err, return code)
"""
return
execute
([
cmd
]
+
list
(
args
),
timeout
=
timeout
or
self
.
timeout
,
input_
str
=
input_
str
)
return
execute
([
cmd
]
+
list
(
args
),
timeout
=
timeout
or
self
.
timeout
,
input_
bytes
=
input_
bytes
)
def
has_to_preprocess
(
self
,
file
:
str
)
->
bool
:
return
os
.
path
.
relpath
(
file
,
self
.
test_dir
).
startswith
(
"exec"
)
...
...
@@ -241,4 +249,4 @@ class Environment:
timer
.
cancel
()
else
:
stdout
,
stderr
=
proc
.
communicate
()
return
stdout
,
stderr
,
proc
.
returncode
,
timeouted
\ No newline at end of file
return
stdout
,
stderr
,
proc
.
returncode
,
timeouted
mjtest/test/ast_tests.py
View file @
fbbd2749
...
...
@@ -7,6 +7,8 @@ from mjtest.test.syntax_tests import BasicSyntaxTest
from
mjtest.test.tests
import
TestCase
,
BasicDiffTestResult
,
BasicTestResult
from
os
import
path
from
mjtest.util.utils
import
decode
_LOG
=
logging
.
getLogger
(
"tests"
)
...
...
@@ -59,6 +61,6 @@ class ASTPrettyPrintTest(BasicSyntaxTest):
with
open
(
output_file
,
"wb"
)
as
f
:
f
.
write
(
out
)
f
.
flush
()
return
rtcode
,
out
.
decode
(),
err
.
decode
()
return
rtcode
,
decode
(
out
),
decode
(
err
)
TestCase
.
TEST_CASE_CLASSES
[
"ast"
].
append
(
ASTPrettyPrintTest
)
mjtest/test/bench.py
View file @
fbbd2749
...
...
@@ -16,7 +16,7 @@ from mjtest.environment import TestMode, Environment
from
mjtest.test.syntax_tests
import
BasicSyntaxTest
from
mjtest.test.tests
import
TestCase
,
BasicDiffTestResult
,
BasicTestResult
,
ExtensibleTestResult
from
mjtest.util.shell
import
SigKill
from
mjtest.util.utils
import
get_main_class_name
,
InsertionTimeOrderedDict
from
mjtest.util.utils
import
get_main_class_name
,
InsertionTimeOrderedDict
,
decode
_LOG
=
logging
.
getLogger
(
"bench_tests"
)
...
...
@@ -49,8 +49,8 @@ class BenchExecTest(BasicSyntaxTest):
INVALID_FILE_ENDINGS
=
[
".inf.java"
,
".inf.mj"
]
MODE
=
TestMode
.
compile_firm
def
__init__
(
self
,
env
:
Environment
,
type
:
str
,
file
:
str
,
preprocessed_file
:
str
):
super
().
__init__
(
env
,
type
,
file
,
preprocessed_file
)
def
__init__
(
self
,
env
:
Environment
,
type
:
str
,
file
:
str
,
preprocessed_file
:
str
,
log_file_mode
:
str
=
""
):
super
().
__init__
(
env
,
type
,
file
,
preprocessed_file
,
log_file_mode
)
self
.
_should_succeed
=
True
def
_bench_command
(
self
,
cmd
:
str
,
*
args
:
Tuple
[
str
])
->
_RunResult
:
...
...
@@ -85,7 +85,7 @@ class BenchExecTest(BasicSyntaxTest):
_LOG
.
error
(
"File
\"
{}
\"
isn't valid Java"
.
format
(
self
.
preprocessed_file
))
test_result
.
incorrect_msg
=
"invalid java code, but output file missing"
test_result
.
set_error_code
(
javac_rtcode
)
test_result
.
add_long_text
(
"Javac error message"
,
err
.
decode
())
test_result
.
add_long_text
(
"Javac error message"
,
decode
(
err
))
test_result
.
add_file
(
"Source file"
,
self
.
preprocessed_file
)
os
.
chdir
(
cwd
)
return
test_result
...
...
@@ -102,8 +102,8 @@ class BenchExecTest(BasicSyntaxTest):
if
rtcode
!=
0
:
test_result
.
incorrect_msg
=
"file can't be compiled"
test_result
.
set_error_code
(
rtcode
)
test_result
.
add_long_text
(
"Error output"
,
err
.
decode
())
test_result
.
add_long_text
(
"Output"
,
out
.
decode
())
test_result
.
add_long_text
(
"Error output"
,
decode
(
err
))
test_result
.
add_long_text
(
"Output"
,
decode
(
out
))
test_result
.
add_file
(
"Source file"
,
self
.
preprocessed_file
)
os
.
chdir
(
cwd
)
return
test_result
...
...
mjtest/test/exec_tests.py
View file @
fbbd2749
...
...
@@ -14,7 +14,7 @@ from mjtest.environment import TestMode, Environment
from
mjtest.test.syntax_tests
import
BasicSyntaxTest
from
mjtest.test.tests
import
TestCase
,
BasicDiffTestResult
,
BasicTestResult
,
ExtensibleTestResult
from
mjtest.util.shell
import
SigKill
from
mjtest.util.utils
import
get_main_class_name
,
InsertionTimeOrderedDict
from
mjtest.util.utils
import
get_main_class_name
,
InsertionTimeOrderedDict
,
decode
_LOG
=
logging
.
getLogger
(
"exec_tests"
)
...
...
@@ -48,6 +48,7 @@ class JavaExecTest(BasicSyntaxTest):
self
.
_expected_output_file
=
self
.
_prev_out_file
_LOG
.
info
(
"Reuse old java output file
\"
{}
\"
"
.
format
(
path
.
relpath
(
self
.
_prev_out_file
)))
self
.
_should_succeed
=
True
self
.
_only_compile
=
self
.
MODE
.
endswith
(
"-only"
)
def
run
(
self
)
->
BasicDiffTestResult
:
is_big_testcase
=
"big"
in
self
.
file
...
...
@@ -58,33 +59,32 @@ class JavaExecTest(BasicSyntaxTest):
cwd
=
os
.
getcwd
()
os
.
chdir
(
tmp_dir
)
exp_out
=
None
input_
str
=
None
input_
bytes
=
None
#print(base_filename, get_main_class_name(base_filename + ".java"))
test_result
=
ExtensibleTestResult
(
self
)
if
self
.
_has_input_file
:
if
self
.
_has_character_input
:
with
open
(
self
.
_input_file
,
"r"
)
as
f
:
input_
str
=
f
.
read
()
with
open
(
self
.
_input_file
,
"r
b
"
)
as
f
:
input_
bytes
=
f
.
read
()
else
:
with
open
(
self
.
_input_file
,
"r"
)
as
f
:
char
s
=
bytearray
(
'ascii'
,
'ignore'
)
# type: bytearray
with
open
(
self
.
_input_file
,
"r"
,
errors
=
"backslashreplace"
)
as
f
:
input_byte
s
=
bytearray
(
encoding
=
'ascii'
,
errors
=
'ignore'
)
# type: bytearray
for
line
in
f
.
readlines
():
for
part
in
line
.
split
(
" "
):
part
=
part
.
strip
()
if
len
(
part
)
>
0
:
chars
.
append
(
int
(
part
))
input_str
=
chars
.
decode
()
if
not
self
.
_has_expected_output_file
:
if
not
self
.
_has_expected_output_file
and
not
self
.
_only_compile
:
_
,
err
,
javac_rtcode
=
\
self
.
env
.
run_command
(
"javac"
,
base_filename
+
".java"
,
timeout
=
timeout
)
if
javac_rtcode
!=
0
:
_LOG
.
error
(
"File
\"
{}
\"
isn't valid Java"
.
format
(
self
.
preprocessed_file
))
test_result
.
incorrect_msg
=
"invalid java code, but output file missing"
test_result
.
set_error_code
(
javac_rtcode
)
test_result
.
add_long_text
(
"Javac error message"
,
err
.
decode
())
test_result
.
add_long_text
(
"Javac error message"
,
decode
(
err
))
test_result
.
add_file
(
"Source file"
,
self
.
preprocessed_file
)
os
.
chdir
(
cwd
)
return
test_result
...
...
@@ -92,30 +92,31 @@ class JavaExecTest(BasicSyntaxTest):
if
not
main_class
:
_LOG
.
debug
(
"Can't find a main class, using the file name instead"
)
main_class
=
base_filename
exp_out
,
err
,
java_rtcode
=
\
self
.
env
.
run_command
(
"java"
,
get_main_class_name
(
base_filename
+
".java"
),
timeout
=
timeout
,
input_str
=
input_str
)
if
javac_rtcode
!=
0
:
test_result
.
add_long_text
(
"Java output: "
,
exp_out
.
decode
())
test_result
.
incorrect_msg
=
"java runtime error"
test_result
.
set_error_code
(
java_rtcode
)
test_result
.
add_long_text
(
"Java error message"
,
err
.
decode
())
test_result
.
add_file
(
"Source file"
,
self
.
preprocessed_file
)
if
self
.
_input_file
:
test_result
.
add_file
(
"Input file"
,
self
.
file
)
os
.
chdir
(
cwd
)
return
test_result
exp_out
=
exp_out
.
decode
()
with
open
(
self
.
_prev_out_file
,
"w"
)
as
f
:
f
.
write
(
exp_out
)
f
.
flush
()
with
open
(
self
.
_prev_out_hash_file
,
"w"
)
as
f
:
f
.
write
(
self
.
_hash_sum_for_file
(
base_filename
+
".java"
))
f
.
flush
()
if
self
.
_has_expected_output_file
and
self
.
type
==
self
.
MODE
and
self
.
env
.
mode
==
self
.
MODE
:
with
open
(
self
.
_expected_output_file
,
"r"
)
as
f
:
if
not
self
.
_only_compile
:
exp_out
,
err
,
java_rtcode
=
\
self
.
env
.
run_command
(
"java"
,
get_main_class_name
(
base_filename
+
".java"
),
timeout
=
timeout
,
input_bytes
=
input_bytes
)
if
javac_rtcode
!=
0
:
test_result
.
add_long_text
(
"Java output: "
,
decode
(
exp_out
))
test_result
.
incorrect_msg
=
"java runtime error"
test_result
.
set_error_code
(
java_rtcode
)
test_result
.
add_long_text
(
"Java error message"
,
decode
(
err
))
test_result
.
add_file
(
"Source file"
,
self
.
preprocessed_file
)
if
self
.
_input_file
:
test_result
.
add_file
(
"Input file"
,
self
.
file
)
os
.
chdir
(
cwd
)
return
test_result
exp_out
=
decode
(
exp_out
)
with
open
(
self
.
_prev_out_file
,
"w"
,
errors
=
"backslashreplace"
)
as
f
:
f
.
write
(
exp_out
)
f
.
flush
()
with
open
(
self
.
_prev_out_hash_file
,
"w"
,
errors
=
"backslashreplace"
)
as
f
:
f
.
write
(
self
.
_hash_sum_for_file
(
base_filename
+
".java"
))
f
.
flush
()
if
self
.
_has_expected_output_file
and
self
.
type
==
self
.
MODE
and
self
.
env
.
mode
==
self
.
MODE
\
and
not
self
.
_only_compile
:
with
open
(
self
.
_expected_output_file
,
"r"
,
errors
=
"backslashreplace"
)
as
f
:
exp_out
=
f
.
read
()
try
:
out
,
err
,
rtcode
=
None
,
None
,
None
try
:
...
...
@@ -123,8 +124,8 @@ class JavaExecTest(BasicSyntaxTest):
if
rtcode
!=
0
:
test_result
.
incorrect_msg
=
"file can't be compiled"
test_result
.
set_error_code
(
rtcode
)
test_result
.
add_long_text
(
"Error output"
,
err
.
decode
())
test_result
.
add_long_text
(
"Output"
,
out
.
decode
())
test_result
.
add_long_text
(
"Error output"
,
decode
(
err
))
test_result
.
add_long_text
(
"Output"
,
decode
(
out
))
test_result
.
add_file
(
"Source file"
,
self
.
preprocessed_file
)
os
.
chdir
(
cwd
)
return
test_result
...
...
@@ -137,13 +138,19 @@ class JavaExecTest(BasicSyntaxTest):
except
:
os
.
chdir
(
cwd
)
raise
if
self
.
_only_compile
:
if
not
os
.
path
.
exists
(
"a.out"
):
test_result
.
incorrect_msg
=
"binary cannot be found"
test_result
.
set_error_code
(
1
)
test_result
.
add_file
(
"Source file"
,
self
.
preprocessed_file
)
return
test_result
try
:
out
,
err
,
rtcode
=
self
.
env
.
run_command
(
"./a.out"
,
timeout
=
timeout
,
input_
str
=
input_
str
)
out
,
err
,
rtcode
=
self
.
env
.
run_command
(
"./a.out"
,
timeout
=
timeout
,
input_
bytes
=
input_
bytes
)
if
rtcode
!=
0
:
test_result
.
incorrect_msg
=
"binary can't be run, non zero error code"
test_result
.
set_error_code
(
rtcode
)
test_result
.
add_long_text
(
"Error output"
,
err
.
decode
())
test_result
.
add_long_text
(
"Output"
,
out
.
decode
())
test_result
.
add_long_text
(
"Error output"
,
decode
(
err
))
test_result
.
add_long_text
(
"Output"
,
decode
(
out
))
test_result
.
add_file
(
"Source file"
,
self
.
preprocessed_file
)
if
self
.
_input_file
:
test_result
.
add_file
(
"Input file"
,
self
.
file
)
...
...
@@ -160,7 +167,7 @@ class JavaExecTest(BasicSyntaxTest):
except
:
os
.
chdir
(
cwd
)
raise
out
=
out
.
decode
()
out
=
decode
(
out
)
if
self
.
type
==
self
.
MODE
and
self
.
env
.
mode
==
self
.
MODE
:
if
exp_out
.
strip
()
!=
out
.
strip
():
test_result
.
incorrect_msg
=
"incorrect output"
...
...
@@ -176,7 +183,7 @@ class JavaExecTest(BasicSyntaxTest):
test_result
.
add_long_text
(
"Output"
,
out
)
os
.
chdir
(
cwd
)
return
test_result
return
BasicTestResult
(
self
,
rtcode
,
out
,
err
.
decode
())
return
BasicTestResult
(
self
,
rtcode
,
out
,
decode
(
err
))
except
SigKill
as
sig
:
os
.
chdir
(
cwd
)
assert
False
...
...
@@ -186,7 +193,7 @@ class JavaExecTest(BasicSyntaxTest):
def
_check_hash_sum
(
self
,
file
:
str
,
hash_sum_file
:
str
)
->
bool
:
old_hash
=
""
with
open
(
hash_sum_file
,
"r"
)
as
f
:
with
open
(
hash_sum_file
,
"r"
,
errors
=
"backslashreplace"
)
as
f
:
try
:
old_hash
=
f
.
readline
().
strip
()
except
UnicodeDecodeError
:
...
...
@@ -195,7 +202,7 @@ class JavaExecTest(BasicSyntaxTest):
@
classmethod
def
_hash_sum_for_file
(
self
,
file
:
str
)
->
str
:
with
open
(
file
,
"r"
)
as
f
:
with
open
(
file
,
"r"
,
errors
=
"backslashreplace"
)
as
f
:
return
hashlib
.
sha256
(
f
.
read
().
encode
()).
hexdigest
()
@
classmethod
...
...
@@ -228,6 +235,16 @@ class JavaCompileExecTest(JavaExecTest):
TestCase
.
TEST_CASE_CLASSES
[
TestMode
.
compile
].
append
(
JavaCompileExecTest
)
class
JavaCompileOnlyTest
(
JavaExecTest
):
MODE
=
TestMode
.
compile_only
TestCase
.
TEST_CASE_CLASSES
[
TestMode
.
compile_only
].
append
(
JavaCompileOnlyTest
)
class
JavaCompileFirmOnlyTest
(
JavaExecTest
):
MODE
=
TestMode
.
compile_firm_only
TestCase
.
TEST_CASE_CLASSES
[
TestMode
.
compile_firm_only
].
append
(
JavaCompileFirmOnlyTest
)
class
JavaInfiniteLoopTest
(
BasicSyntaxTest
):
FILE_ENDINGS
=
[
".inf.java"
,
".inf.mj"
]
...
...
@@ -255,8 +272,8 @@ class JavaInfiniteLoopTest(BasicSyntaxTest):
if
rtcode
!=
0
:
test_result
.
incorrect_msg
=
"file can't be compiled"
test_result
.
set_error_code
(
rtcode
)
test_result
.
add_long_text
(
"Error output"
,
err
.
decode
())
test_result
.
add_long_text
(
"Output"
,
out
.
decode
())
test_result
.
add_long_text
(
"Error output"
,
decode
(
err
))
test_result
.
add_long_text
(
"Output"
,
decode
(
out
))
test_result
.
add_file
(
"Source file"
,
self
.
preprocessed_file
)
os
.
chdir
(
cwd
)
return
test_result
...
...
@@ -264,19 +281,19 @@ class JavaInfiniteLoopTest(BasicSyntaxTest):
if
rtcode
!=
0
:
test_result
.
incorrect_msg
=
"binary can't be run, non zero error code"
test_result
.
set_error_code
(
rtcode
)
test_result
.
add_long_text
(
"Error output"
,
err
.
decode
())
test_result
.
add_long_text
(
"Output"
,
out
.
decode
())
test_result
.
add_long_text
(
"Error output"
,
decode
(
err
))
test_result
.
add_long_text
(
"Output"
,
decode
(
out
))
test_result
.
add_file
(
"Source file"
,
self
.
preprocessed_file
)
os
.
chdir
(
cwd
)
return
test_result
except
SigKill
as
sig
:
if
sig
.
retcode
==
signal
.
SIGXCPU
:
out
,
_
,
_
,
_
=
self
.
env
.
run_command_till_timeout
(
"./a.out"
,
timeout
=
1
)
test_result
.
add_long_text
(
"Output"
,
out
.
decode
())
test_result
.
add_long_text
(
"Output"
,
decode
(
out
))
if
self
.
_has_output_file
:
out
=
out
.
decode
().
strip
()
out
=
decode
(
out
).
strip
()
exp_out
=
""
with
open
(
self
.
_output_file
,
"r"
)
as
f
:
with
open
(
self
.
_output_file
,
"r"
,
errors
=
"backslashreplace"
)
as
f
:
exp_out
=
f
.
read
().
strip
()
test_result
.
add_long_text
(
"Expected output start"
,
exp_out
)
if
not
out
.
startswith
(
exp_out
):
...
...
@@ -342,8 +359,8 @@ class InvalidJavaExecTest(BasicSyntaxTest):
if
rtcode
!=
0
:
test_result
.
incorrect_msg
=
"file can't be compiled"
test_result
.
set_error_code
(
-
rtcode
)
test_result
.
add_long_text
(
"Error output"
,
err
.
decode
())
test_result
.
add_long_text
(
"Output"
,
out
.
decode
())
test_result
.
add_long_text
(
"Error output"
,
decode
(
err
))
test_result
.
add_long_text
(
"Output"
,
decode
(
out
))
test_result
.
add_file
(
"Source file"
,
self
.
preprocessed_file
)
os
.
chdir
(
cwd
)
return
test_result
...
...
@@ -361,8 +378,8 @@ class InvalidJavaExecTest(BasicSyntaxTest):
if
rtcode
!=
0
:
test_result
.
incorrect_msg
=
"binary can't be run, non zero error code"
test_result
.
set_error_code
(
rtcode
)
test_result
.
add_long_text
(
"Error output"
,
err
.
decode
())
test_result
.
add_long_text
(
"Output"
,
out
.
decode
())
test_result
.
add_long_text
(
"Error output"
,
decode
(
err
))
test_result
.
add_long_text
(
"Output"
,
decode
(
out
))
test_result
.
add_file
(
"Source file"
,
self
.
preprocessed_file
)
os
.
chdir
(
cwd
)
return
test_result
...
...
@@ -375,7 +392,7 @@ class InvalidJavaExecTest(BasicSyntaxTest):
except
:
os
.
chdir
(
cwd
)
raise
test_result
=
BasicTestResult
(
self
,
rtcode
,
out
.
decode
(),
err
.
decode
())
test_result
=
BasicTestResult
(
self
,
rtcode
,
decode
(
out
),
decode
(
err
))
test_result
.
require_error_string_in_error_case
=
False
return
test_result
except
SigKill
as
sig
:
...
...
mjtest/test/syntax_tests.py
View file @
fbbd2749
...
...
@@ -3,6 +3,8 @@ from mjtest.environment import Environment, TestMode, TEST_MODES, get_test_dirna
from
mjtest.test.tests
import
TestCase
,
BasicTestResult
from
os
import
path
from
mjtest.util.utils
import
decode
class
BasicSyntaxTest
(
TestCase
):
...
...
@@ -35,7 +37,7 @@ class BasicSyntaxTest(TestCase):
def
run
(
self
)
->
BasicTestResult
:
out
,
err
,
rtcode
=
self
.
env
.
run_mj_command
(
self
.
MODE
,
self
.
preprocessed_file
)
return
BasicTestResult
(
self
,
rtcode
,
out
.
decode
(),
err
.
decode
())
return
BasicTestResult
(
self
,
rtcode
,
decode
(
out
),
decode
(
err
))
TestCase
.
TEST_CASE_CLASSES
[
TestMode
.
syntax
].
append
(
BasicSyntaxTest
)
...
...
@@ -53,7 +55,7 @@ class JavaCompileTest(BasicSyntaxTest):
tmp_dir
=
self
.
env
.
create_pid_local_tmpdir
()
_
,
self
.
javac_err
,
self
.
javac_rtcode
=
\
self
.
env
.
run_command
(
"javac"
,
path
.
relpath
(
preprocessed_file
),
"-d"
,
tmp_dir
,
"-verbose"
)
self
.
javac_err
=
self
.
javac_err
.
decode
(
)
# type: str
self
.
javac_err
=
decode
(
self
.
javac_err
)
# type: str
self
.
_should_succeed
=
self
.
_is_file_syntactically_correct
()
if
self
.
SYNTAX_TEST
else
self
.
javac_rtcode
==
0
def
_is_file_syntactically_correct
(
self
):
...
...
mjtest/test/tests.py
View file @
fbbd2749
from
collections
import
namedtuple
import
shutil
from
typing
import
Optional
,
List
,
Tuple
,
T
,
Union
,
Dict
from
typing
import
Optional
,
List
,
Tuple
,
Union
,
Dict
import
collections
from
mjtest.environment
import
Environment
,
TestMode
,
TEST_MODES
,
get_test_dirname
from
os.path
import
join
,
exists
,
basename
...
...
@@ -8,7 +8,7 @@ import logging
import
os
import
multiprocessing
from
mjtest.util.parallelism
import
available_cpu_count
from
mjtest.util.utils
import
cprint
,
colored
from
mjtest.util.utils
import
cprint
,
colored
,
decode
from
pprint
import
pprint
import
shutil
import
difflib
...
...
@@ -50,7 +50,7 @@ class TestSuite:
correct_test_cases
=
set
()
log_file
=
self
.
_log_file_for_type
(
mode
)
if
exists
(
log_file
):
with
open
(
log_file
)
as
f
:
with
open
(
log_file
,
errors
=
"backslashreplace"
)
as
f
:
correct_test_cases
=
set
()
for
t
in
f
.
readlines
():
t
=
t
.
strip
()
...
...
@@ -223,7 +223,7 @@ class TestSuite:
os
.
mkdir
(
os
.
path
.
dirname
(
log_file
))
except
IOError
:
pass
with
open
(
log_file
,
"w"
)
as
f
:
with
open
(
log_file
,
"w"
,
errors
=
"backslashreplace"
)
as
f
:
f
.
write
(
"
\n
"
.
join
(
self
.
correct_test_cases
[
mode
]))
except
IOError
as
e
:
_LOG
.
exception
(
"Caught i/o error while storing {}"
.
format
(
log_file
))
...
...
@@ -314,7 +314,7 @@ class TestResult:
def
store_at
(
self
,
file
:
str
):
os
.
makedirs
(
os
.
path
.
dirname
(
file
),
exist_ok
=
True
)
with
open
(
file
,
"w"
)
as
f
:
with
open
(
file
,
"w"
,
errors
=
"backslashreplace"
)
as
f
:
print
(
self
.
long_message
(),
file
=
f
)
def
short_message
(
self
)
->
str
:
...
...
@@ -348,8 +348,8 @@ class ExtensibleTestResult(TestResult):
self
.
messages
.
append
(
TestResultMessage
(
title
,
content
,
multiline
=
False
,
with_line_numbers
=
False
))
def
add_file
(
self
,
title
:
str
,
file_name
:
str
,
with_line_numbers
:
bool
=
True
):
with
open
(
file_name
,
"r"
)
as
f
:
file_content
=
os
.
linesep
.
join
([
line
.
rstrip
()
for
line
in
f
])
with
open
(
file_name
,
"r"
,
errors
=
"backslashreplace"
)
as
f
:
file_content
=
os
.
linesep
.
join
([
line
.
rstrip
()
for
line
in
f
.
readlines
()
])
self
.
add_long_text
(
title
,
file_content
,
with_line_numbers
)
def
succeeded
(
self
):
...
...
@@ -461,7 +461,7 @@ class BasicTestResult(TestResult):
def
long_message
(
self
)
->
str
:
file_content
=
[]
with
open
(
self
.
test_case
.
preprocessed_file
,
"r"
)
as
f
:
with
open
(
self
.
test_case
.
preprocessed_file
,
"r"
,
errors
=
"backslashreplace"
)
as
f
:
file_content
=
[
line
.
rstrip
()
for
line
in
f
]
others
=
[]
for
title
,
content
,
long_text
in
self
.
other_texts
:
...
...
@@ -563,13 +563,13 @@ class DiffTest(TestCase):
exp_out
=
""
if
rtcode
==
0
and
self
.
should_succeed
():
if
self
.
_has_expected_output_file
and
self
.
type
==
self
.
MODE
and
self
.
env
.
mode
==
self
.
MODE
:
with
open
(
self
.
_expected_output_file
,
"r"
)
as
f
:
with
open
(
self
.
_expected_output_file
,
"r"
,
errors
=
"backslashreplace"
)
as
f
:
exp_out
=
f
.
read
()
#else:
# _LOG.error("Expected output file for test case {}:{} is missing.".format(self.MODE, self.short_name()))
if
self
.
type
==
self
.
MODE
and
self
.
env
.
mode
==
self
.
MODE
:
return
BasicDiffTestResult
(
self
,
rtcode
,
out
.
decode
(),
err
.
decode
(),
exp_out
)
return
BasicTestResult
(
self
,
rtcode
,
out
.
decode
(),
err
.
decode
())
return
BasicDiffTestResult
(
self
,
rtcode
,
decode
(
out
),
decode
(
err
),
exp_out
)
return
BasicTestResult
(
self
,
rtcode
,
decode
(
out
),
decode
(
err
))
class
LexerDiffTest
(
DiffTest
):
...
...
@@ -582,4 +582,4 @@ import mjtest.test.syntax_tests
import
mjtest.test.ast_tests
import
mjtest.test.semantic_tests
import
mjtest.test.exec_tests
import
mjtest.test.bench
\ No newline at end of file
import
mjtest.test.bench
mjtest/util/shell.py
View file @
fbbd2749
...
...
@@ -16,7 +16,7 @@ try:
except
ImportError
:
has_resource_module
=
False
pass
import
sys
import
signal
import
threading
...
...
@@ -46,14 +46,14 @@ def _lower_rlimit(res, limit):
class
_Execute
(
object
):
def
__init__
(
self
,
cmd
,
timeout
,
env
,
rlimit
,
input_
str
):
def
__init__
(
self
,
cmd
,
timeout
,
env
,
rlimit
,
input_
bytes
):
self
.
cmd
=
cmd
self
.
timeout
=
timeout
self
.
env
=
env
self
.
proc
=
None
self
.
exception
=
None
self
.
rlimit
=
rlimit
self
.
input_
str
=
input_
str
self
.
input_
bytes
=
input_
bytes
MB
=
1024
*
1024
if
not
'RLIMIT_CORE'
in
rlimit
:
rlimit
[
'RLIMIT_CORE'
]
=
0
...
...
@@ -80,14 +80,13 @@ class _Execute(object):
self
.
proc
=
subprocess
.
Popen
(
self
.
cmd
,
stdout
=
subprocess
.
PIPE
,
stderr
=
subprocess
.
PIPE
,
stdin
=
subprocess
.
PIPE
if
self
.
input_
str
else
None
,