Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
ufebl
mjtest
Commits
8e19d593
Commit
8e19d593
authored
Nov 02, 2016
by
Johannes Bechberger
Browse files
Fix "only_incorrect_testcases" bug
And add "--produce_no_reports" option
parent
4a370c9f
Changes
4
Hide whitespace changes
Inline
Side-by-side
README.mdwn
View file @
8e19d593
...
...
@@ -68,8 +68,8 @@ To get colored output install the python3 module `termcolor`.
Output of the `./mjt.py --help`
```
usage: mjt.py [-h] [--only_incorrect_tests] [--p
arallel
]
[--log_level LOG_LEVEL]
usage: mjt.py [-h] [--only_incorrect_tests] [--p
roduce_no_reports
]
[--parallel]
[--log_level LOG_LEVEL]
{syntax,semantic,exec} MJ_RUN
MiniJava test runner
...
...
@@ -85,6 +85,8 @@ optional arguments:
-h, --help show this help message and exit
--only_incorrect_tests
Only run the tests that were incorrect the last run
--produce_no_reports Produce no long reports besides the command line
output
--parallel Run the tests in parallel
--log LOG Logging level (error, warn, info or debug)
```
...
...
mjtest/cli.py
View file @
8e19d593
...
...
@@ -36,6 +36,8 @@ if True:#__name__ == '__main__':
# help="Directory that contains all test cases, default is the 'tests' directory")
parser
.
add_argument
(
"--only_incorrect_tests"
,
action
=
"store_true"
,
default
=
False
,
help
=
"Only run the tests that were incorrect the last run"
)
parser
.
add_argument
(
"--produce_no_reports"
,
action
=
"store_true"
,
default
=
False
,
help
=
"Produce no long reports besides the command line output"
)
parser
.
add_argument
(
"--parallel"
,
action
=
"store_true"
,
default
=
False
,
help
=
"Run the tests in parallel"
)
#parser.add_argument("--timeout", action="store_const", default=30, const="timeout",
...
...
@@ -54,7 +56,8 @@ if True:#__name__ == '__main__':
finally
:
suite
.
env
.
clean_up
()
suite
.
store
()
if
ret
is
None
or
ret
:
if
ret
is
None
or
ret
.
failed
>
0
:
sys
.
exit
(
1
)
else
:
sys
.
exit
(
0
)
mjtest/environment.py
View file @
8e19d593
...
...
@@ -29,7 +29,8 @@ class Environment:
def
__init__
(
self
,
mode
,
mj_run
:
str
,
tmp_dir
:
str
=
""
,
test_dir
:
str
=
""
,
only_incorrect_tests
:
bool
=
False
,
parallel
:
bool
=
False
,
timeout
:
int
=
30
,
report_dir
:
str
=
""
,
log_level
:
str
=
"warn"
):
timeout
:
int
=
30
,
report_dir
:
str
=
""
,
log_level
:
str
=
"warn"
,
produce_no_reports
:
bool
=
True
):
self
.
mode
=
mode
self
.
mj_run_cmd
=
os
.
path
.
realpath
(
mj_run
)
...
...
@@ -54,18 +55,21 @@ class Environment:
self
.
only_incorrect_tests
=
only_incorrect_tests
self
.
parallel
=
parallel
self
.
timeout
=
timeout
if
report_dir
:
self
.
report_dir
=
os
.
path
.
abspath
(
os
.
path
.
expandvars
(
report_dir
))
if
not
os
.
path
.
exists
(
report_dir
):
os
.
mkdir
(
self
.
report_dir
)
if
not
produce_no_reports
:
if
report_dir
:
self
.
report_dir
=
os
.
path
.
abspath
(
os
.
path
.
expandvars
(
report_dir
))
if
not
os
.
path
.
exists
(
report_dir
):
os
.
mkdir
(
self
.
report_dir
)
else
:
self
.
report_dir
=
os
.
path
.
join
(
get_mjtest_basedir
(),
"reports"
)
if
not
os
.
path
.
exists
(
self
.
report_dir
):
os
.
mkdir
(
self
.
report_dir
)
self
.
report_dir
=
os
.
path
.
join
(
self
.
report_dir
,
datetime
.
now
().
strftime
(
"%d-%m-%y_%H-%M-%S"
))
os
.
mkdir
(
self
.
report_dir
)
else
:
self
.
report_dir
=
os
.
path
.
join
(
get_mjtest_basedir
(),
"reports"
)
if
not
os
.
path
.
exists
(
self
.
report_dir
):
os
.
mkdir
(
self
.
report_dir
)
self
.
report_dir
=
os
.
path
.
join
(
self
.
report_dir
,
datetime
.
now
().
strftime
(
"%d-%m-%y_%H-%M-%S"
))
os
.
mkdir
(
self
.
report_dir
)
self
.
report_dir
=
None
logging
.
basicConfig
(
level
=
self
.
LOG_LEVELS
[
log_level
])
self
.
produce_reports
=
not
produce_no_reports
# type: bool
def
create_tmpfile
(
self
)
->
str
:
return
os
.
path
.
join
(
self
.
tmp_dir
,
str
(
os
.
times
()))
...
...
mjtest/test/tests.py
View file @
8e19d593
from
collections
import
namedtuple
import
shutil
from
typing
import
Optional
,
List
,
Tuple
,
T
,
Union
,
Dict
import
collections
from
mjtest.environment
import
Environment
,
TestMode
,
TEST_MODES
from
os.path
import
join
,
exists
,
basename
import
logging
...
...
@@ -25,7 +26,7 @@ class TestSuite:
def
__init__
(
self
,
env
:
Environment
):
self
.
env
=
env
self
.
test_cases
=
{}
# type: Dict[str, List[TestCase]]
self
.
correct_test_cases
=
{}
# type: Dict[str,
Lis
t[str]]
self
.
correct_test_cases
=
collections
.
defaultdict
(
set
)
# type: Dict[str,
Se
t[str]]
self
.
_load_test_cases
()
def
_load_test_cases
(
self
):
...
...
@@ -47,7 +48,12 @@ class TestSuite:
log_file
=
self
.
_log_file_for_type
(
mode
)
if
exists
(
log_file
):
with
open
(
log_file
)
as
f
:
correct_test_cases
=
set
(
f
.
readlines
())
correct_test_cases
=
set
()
for
t
in
f
.
readlines
():
t
=
t
.
strip
()
if
len
(
t
)
>
0
:
self
.
correct_test_cases
[
mode
].
add
(
t
)
correct_test_cases
.
add
(
t
)
for
file
in
sorted
(
os
.
listdir
(
dir
)):
if
not
TestCase
.
has_valid_file_ending
(
mode
,
file
):
_LOG
.
debug
(
"Skip file "
+
file
)
...
...
@@ -66,7 +72,7 @@ class TestSuite:
return
join
(
self
.
env
.
test_dir
,
type
,
".mjtest_correct_testcases"
)
def
_add_correct_test_case
(
self
,
test_case
:
'TestCase'
):
self
.
correct_test_cases
[
test_case
.
type
].
a
ppen
d
(
basename
(
test_case
.
file
))
self
.
correct_test_cases
[
test_case
.
type
].
a
d
d
(
basename
(
test_case
.
file
))
def
run
(
self
)
->
RunResult
:
ret
=
RunResult
(
0
,
0
)
...
...
@@ -87,10 +93,11 @@ class TestSuite:
colored
(
"{} failed."
.
format
(
ret
.
failed
),
"red"
,
attrs
=
[
"bold"
]))
else
:
cprint
(
"All {} run tests succeeded"
.
format
(
ret
.
count
),
"green"
)
report_dir
=
self
.
env
.
report_dir
+
"."
+
(
"successful"
if
ret
.
failed
==
0
else
"failed"
)
os
.
rename
(
self
.
env
.
report_dir
,
report_dir
)
print
(
"A full report for each test can be found at {}"
.
format
(
os
.
path
.
relpath
(
report_dir
)))
if
self
.
env
.
produce_reports
:
report_dir
=
self
.
env
.
report_dir
+
"."
+
(
"successful"
if
ret
.
failed
==
0
else
"failed"
)
os
.
rename
(
self
.
env
.
report_dir
,
report_dir
)
print
(
"A full report for each test can be found at {}"
.
format
(
os
.
path
.
relpath
(
report_dir
)))
return
ret
...
...
@@ -109,13 +116,18 @@ class TestSuite:
def
_func
(
self
,
test_case
:
'TestCase'
):
ret
=
self
.
_run_test_case
(
test_case
)
if
ret
is
not
False
and
ret
.
is_correct
():
return
0
,
[
test_case
]
return
1
,
[]
return
0
,
test_case
return
1
,
test_case
def
_run_parallel
(
self
,
mode
:
str
,
parallel_jobs
:
int
)
->
RunResult
:
pool
=
multiprocessing
.
Pool
(
parallel_jobs
)
rets
=
pool
.
map
(
self
.
_func
,
self
.
test_cases
[
mode
])
return
RunResult
(
len
(
rets
),
sum
(
map
(
lambda
x
:
x
[
0
],
rets
)))
result
=
RunResult
(
len
(
rets
),
sum
(
map
(
lambda
x
:
x
[
0
],
rets
)))
for
(
suc
,
test_case
)
in
rets
:
if
suc
==
0
:
self
.
_add_correct_test_case
(
test_case
)
return
result
def
_run_test_case
(
self
,
test_case
:
'TestCase'
)
->
Optional
[
'TestResult'
]:
try
:
...
...
@@ -126,13 +138,14 @@ class TestSuite:
tc
=
test_case
.
name
()),
color
,
attrs
=
[
"bold"
])
+
colored
(
""
if
ret
.
is_correct
()
else
ret
.
short_message
(),
color
))
try
:
if
not
exists
(
self
.
env
.
report_dir
):
os
.
mkdir
(
self
.
env
.
report_dir
)
rep_dir
=
join
(
self
.
env
.
report_dir
,
test_case
.
type
)
if
not
exists
(
rep_dir
):
os
.
mkdir
(
rep_dir
)
suffix
=
".correct"
if
ret
.
is_correct
()
else
".incorrect"
ret
.
store_at
(
join
(
rep_dir
,
test_case
.
short_name
()
+
suffix
))
if
self
.
env
.
produce_reports
:
if
not
exists
(
self
.
env
.
report_dir
):
os
.
mkdir
(
self
.
env
.
report_dir
)
rep_dir
=
join
(
self
.
env
.
report_dir
,
test_case
.
type
)
if
not
exists
(
rep_dir
):
os
.
mkdir
(
rep_dir
)
suffix
=
".correct"
if
ret
.
is_correct
()
else
".incorrect"
ret
.
store_at
(
join
(
rep_dir
,
test_case
.
short_name
()
+
suffix
))
return
ret
except
IOError
:
_LOG
.
exception
(
"Caught i/o error while trying to store the report for '{}'"
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment