spec.py 14.2 KB
Newer Older
Andreas Zwinkau's avatar
Andreas Zwinkau committed
1
2
3
4
#!/usr/bin/env python
import os
import platform
import optparse # TODO argparse from python 2.7 on
5
6
import subprocess
import resource
Andreas Zwinkau's avatar
Andreas Zwinkau committed
7
from datetime import datetime
Andreas Zwinkau's avatar
Andreas Zwinkau committed
8
9

SCRIPT_VERSION="0.1"
Andreas Zwinkau's avatar
Andreas Zwinkau committed
10
11
AUTHOR="Andreas Zwinkau <zwinkau@kit.edu>"

Andreas Zwinkau's avatar
Andreas Zwinkau committed
12
13
14
15
16
17
(KERNEL,HOSTNAME,KERNEL_VERSION,VERSION,ARCH,PROCESSOR) = platform.uname()
assert not PROCESSOR, PROCESSOR
LINUX_NAME, LINUX_VERSION, LINUX_CODENAME = platform.dist()
BITSIZE = {
	"i686": 32,
	"x86_64": 64,
Andreas Zwinkau's avatar
Andreas Zwinkau committed
18
	"sparc64": 64,
Andreas Zwinkau's avatar
Andreas Zwinkau committed
19
20
21
22
23
}.get(ARCH, "unspecified for "+ARCH)

# assert spec dir
for subdir in "benchspec config bin result".split():
	assert os.path.isdir(subdir), "subdir missing: "+subdir
Andreas Zwinkau's avatar
Andreas Zwinkau committed
24
for filename in "shrc README LICENSE MANIFEST".split():
Andreas Zwinkau's avatar
Andreas Zwinkau committed
25
26
	assert os.path.isfile(filename), "file missing: "+filename

Andreas Zwinkau's avatar
Andreas Zwinkau committed
27
28
29
def identify_suite():
	"""Which SPEC suite are we in?"""
	for line in file("README"):
Andreas Zwinkau's avatar
Andreas Zwinkau committed
30
31
		possibilities = ["SPEC CPU2000", "SPEC CPU2006", "SPEC OMP2001"]
		if line.startswith("---"): break
Andreas Zwinkau's avatar
Andreas Zwinkau committed
32
33
34
		for p in possibilities:
			if p in line:
				return p
Andreas Zwinkau's avatar
Andreas Zwinkau committed
35
	print "Error: Unknown SPEC suite (if at all)"
Andreas Zwinkau's avatar
Andreas Zwinkau committed
36
37
38
	exit(-1)
SPEC_SUITE = identify_suite()

Andreas Zwinkau's avatar
Andreas Zwinkau committed
39
parser = optparse.OptionParser(version=SCRIPT_VERSION)
Andreas Zwinkau's avatar
Andreas Zwinkau committed
40
41
parser.add_option("--config-only", action="store_true", default=False,
		help="Just generate config file, but do not build or run")
42
43
parser.add_option("--build-only", action="store_true", default=False,
		help="Just generate config file and build benchmark programs, but do not run")
44
45
46
47
48
49
50
51
parser.add_option("--no-cpp", action="store_true", default=False,
		help="Exlude benchmark programs using C++")
parser.add_option("--no-fortran", action="store_true", default=False,
		help="Exlude benchmark programs using Fortran")
parser.add_option("--no-medium", action="store_true", default=False,
		help="Exlude OMPM programs")
parser.add_option("--no-large", action="store_true", default=False,
		help="Exlude OMPL progras")
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
parser.add_option("-i", "--iterations", metavar="ITR", default=3,
		help="Evaluate ITR iterations per program, default=3")

parser.add_option("--c-compiler", metavar="CC", default="cparser",
		help="Use CC to build benchmark programs, default=cparser")
parser.add_option("--fortran-compiler", metavar="FC", default="gfortran",
		help="Use FC to build benchmark programs, default=gfortran")
parser.add_option("--cpp-compiler", metavar="CXX", default="g++",
		help="Use CXX to build benchmark programs, default=g++")

parser.add_option("--compiler-args", metavar="FLAGS", default="-O3",
		help="Append FLAGS to build benchmark programs for all compilers, default=-O3")
parser.add_option("--c-compiler-args", metavar="CFLAGS", default="",
		help="Append CFLAGS to build benchmark programs for C compiler, default empty")
parser.add_option("--fortran-compiler-args", metavar="FFLAGS", default="",
		help="Append FFLAGS to build benchmark programs for Fortran compiler, default empty")
parser.add_option("--cpp-compiler-args", metavar="CFLAGS", default="",
		help="Append CFLAGS to build benchmark programs for C++ compiler, default empty")
Andreas Zwinkau's avatar
Andreas Zwinkau committed
70

71

Andreas Zwinkau's avatar
Andreas Zwinkau committed
72
73
74
75
76
77
78
79
# http://www.spec.org/cpu2006/Docs/config.html

_CONFIG_HEAD = """\
#### Automatically generated by spec.py script ####

output_format = csv,html,raw,screen,config
ignore_errors = yes
makeflags = -j4        # allows multiple simultaneous compiles
Andreas Zwinkau's avatar
Andreas Zwinkau committed
80
test_sponsor = IPD Snelting
Andreas Zwinkau's avatar
Andreas Zwinkau committed
81
82
83
tester_name = IPD Snelting
company_name = Karlsruhe Institute of Technology KIT
prepared_by = spec.py script
Andreas Zwinkau's avatar
Andreas Zwinkau committed
84

Andreas Zwinkau's avatar
Andreas Zwinkau committed
85
## just assuming:
Andreas Zwinkau's avatar
Andreas Zwinkau committed
86
87
88
89
hw_fpu = Integrated
sw_state = Multi-user
"""

90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
class SigKill(Exception):
	def __init__(self, retcode, name):
		self.retcode = retcode
		self.name = name

_EXIT_CODES = {
	-2: "keyboard interrupt",
	-3: "keyboard quit",
	-4: "illegal instruction",
	-6: "aborted",
	-8: "floating point exception",
	-9: "kill",
	-11: "segfault",
	-13: "broken pipe",
	-14: "timeout",
	-15: "termination",
}

def execute(cmd, env=None, timeout=0):
	"""Execute a command and return stderr and/or stdout data"""
	preexec_fn = None
	if timeout > 0.0:
		def set_timeout():
			 resource.setrlimit(resource.RLIMIT_CPU, (timeout, timeout))
		preexec_fn = set_timeout
	cmd = filter(lambda x: x, cmd.split(' '))
	proc = subprocess.Popen(cmd,
							stdout=subprocess.PIPE,
							stderr=subprocess.STDOUT,
							preexec_fn = preexec_fn,
							env=env)
	out, err = proc.communicate()
	try:
		for line in out.splitlines():
			yield line
	except Exception, e:
		print "'%s' -> %s" % (c, e)
	if proc.returncode in _EXIT_CODES:
		raise SigKill(proc.returncode, _EXIT_CODES[proc.returncode])
	elif proc.returncode < 0: # program aborted
		raise Exception(proc.returncode, cmd)

Andreas Zwinkau's avatar
Andreas Zwinkau committed
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
def path_to_filesystem(path):
	for line in execute("stat -f ."):
		if "Type:" in line:
			i = line.index("Type:")
			return line[i+6:].strip()

def _get_proc_info(path):
	"""Extract data from proc files"""
	data = dict()
	for line in file(path):
		try:
			key, value = line.split(":")
		except ValueError:
			continue
		data[key.strip()] = value.strip()
	return data

def get_mem_info():
	"""Return data about Memory as dict"""
	return _get_proc_info("/proc/meminfo")

def get_cpu_info(): # TODO ignores multiple cpus
	"""Return data about CPU as dict"""
	return _get_proc_info("/proc/cpuinfo")

def get_compiler_version(compiler):
	"""Get the version from a compiler executable"""
159
160
161
162
163
164
	try:
		d = execute(compiler+" --version")
		for line in d:
			return line
	except OSError:
		return "No compiler '%s' found." % compiler
Andreas Zwinkau's avatar
Andreas Zwinkau committed
165

Andreas Zwinkau's avatar
Andreas Zwinkau committed
166
PORTABILITY_FLAGS = dict()
167

Andreas Zwinkau's avatar
Andreas Zwinkau committed
168
169
170
171
172
173
174
175
176
177
PORTABILITY_FLAGS[("SPEC CPU2000", "Ubuntu", 32)] = """\
176.gcc=default=default=default:
CPORTABILITY =
notes003 = CPORTABILITY 176.gcc: -fno-strict-aliasing
186.crafty=default=default=default:
CPORTABILITY = -DLINUX_i386
notes004 = CPORTABILITY 186.crafty: -DLINUX_i386
252.eon=default=default=default:
CXXPORTABILITY = -DHAS_ERRLIST
notes005 = CXXPORTABILITY 252.eon: -DHAS_ERRLIST
Andreas Zwinkau's avatar
Andreas Zwinkau committed
178
253.perlbmk=default=default=default:
Andreas Zwinkau's avatar
Andreas Zwinkau committed
179
180
181
182
183
184
185
186
CPORTABILITY = -DSPEC_CPU2000_NEED_BOOL -DSPEC_CPU2000_LINUX_I386
notes006 = CPORTABILITY 253.perlbmk: -DSPEC_CPU2000_NEED_BOOL -DSPEC_CPU2000_LINUX_I386
254.gap=default=default=default:
CPORTABILITY = -DSYS_HAS_SIGNAL_PROTO -DSYS_HAS_MALLOC_PROTO -DSYS_HAS_CALLOC_PROTO -DSYS_IS_USG -DSYS_HAS_IOCTL_PROTO -DSYS_HAS_TIME_PROTO
notes007 = CPORTABILITY 254.gap: -DSYS_HAS_SIGNAL_PROTO -DSYS_HAS_MALLOC_PROTO -DSYS_HAS_CALLOC_PROTO -DSYS_IS_USG -DSYS_HAS_IOCTL_PROTO -DSYS_HAS_TIME_PROTO
255.vortex=default=default=default:
CPORTABILITY = -fno-strict-aliasing
notes008 = CPORTABILITY 255.vortex: -fno-strict-aliasing
Andreas Zwinkau's avatar
Andreas Zwinkau committed
187
187.facerec=default=default=default:
Andreas Zwinkau's avatar
Andreas Zwinkau committed
188
189
EXTRA_FFLAGS= -I../src
"""
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211

PORTABILITY_FLAGS[("SPEC OMP2001", "Ubuntu", 32)] = """\
310.wupwise_m=default=default=default:
FPORTABILITY=-ffixed-form
312.swim_m=default=default=default:
FPORTABILITY=-ffixed-form
314.mgrid_m=default=default=default:
FPORTABILITY=-ffixed-form
# no lastprivate error on i44pc50, needs extra download
srcalt = intel
316.applu_m=default=default=default:
FPORTABILITY=-ffixed-form
318.galgel_m=default=default=default:
FPORTABILITY=-ffixed-form -ffixed-line-length-80
324.apsi_m=default=default=default:
FPORTABILITY=-ffixed-form
326.gafort_m=default=default=default:
FPORTABILITY=-qsuffix=f=f90
328.fma3d_m=default=default=default:
FPORTABILITY=-qsuffix=f=f90
"""

Andreas Zwinkau's avatar
Andreas Zwinkau committed
212
PORTABILITY_FLAGS[("SPEC CPU2006", "Ubuntu", 32)] = """\
Andreas Zwinkau's avatar
Andreas Zwinkau committed
213
214
215
216
217
218
219
220
400.perlbench=default=default=default:
CPORTABILITY= -DSPEC_CPU_LINUX_IA32
462.libquantum=default=default=default:
CPORTABILITY= -DSPEC_CPU_LINUX
483.xalancbmk=default=default=default:
CXXPORTABILITY= -DSPEC_CPU_LINUX
481.wrf=default=default=default:
CPORTABILITY = -DSPEC_CPU_CASE_FLAG -DSPEC_CPU_LINUX
Andreas Zwinkau's avatar
Andreas Zwinkau committed
221
"""
222

Andreas Zwinkau's avatar
Andreas Zwinkau committed
223
PORTABILITY_FLAGS[("SPEC CPU2006", "Ubuntu", 64)] = """\
Andreas Zwinkau's avatar
Andreas Zwinkau committed
224
225
226
227
228
229
230
231
232
233
default=base=default=default:
PORTABILITY = -DSPEC_CPU_LP64 
400.perlbench=default=default=default:
CPORTABILITY = -DSPEC_CPU_LINUX_X64
462.libquantum=default=default=default:
CPORTABILITY=  -DSPEC_CPU_LINUX
483.xalancbmk=default=default=default:
CXXPORTABILITY= -DSPEC_CPU_LINUX
481.wrf=default=default=default:
CPORTABILITY = -DSPEC_CPU_CASE_FLAG -DSPEC_CPU_LINUX
Andreas Zwinkau's avatar
Andreas Zwinkau committed
234
235
236
237
238
239
240
241
242
243
"""

def get_portability():
	"""Get necessary portability flags"""
	contents = ["\n\n#### Portability Flags ####"]
	for (suite,os,bitsize),text in PORTABILITY_FLAGS.items():
		if suite != SPEC_SUITE: continue
		if os != LINUX_NAME: continue
		if bitsize != BITSIZE: continue
		contents.append(text)
Andreas Zwinkau's avatar
Andreas Zwinkau committed
244
245
	return "\n".join(contents)

246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
BENCHMARK_LANGUAGES = [
	("SPEC CPU2000", "164.gzip", "c"),
	("SPEC CPU2000", "175.vpr", "c"),
	("SPEC CPU2000", "176.gcc", "c"),
	("SPEC CPU2000", "181.mcf", "c"),
	("SPEC CPU2000", "186.crafty", "c"),
	("SPEC CPU2000", "197.parser", "c"),
	("SPEC CPU2000", "252.eon", "cpp"),
	("SPEC CPU2000", "253.perlbmk", "c"),
	("SPEC CPU2000", "254.gap", "c"),
	("SPEC CPU2000", "255.vortex", "c"),
	("SPEC CPU2000", "256.bzip2", "c"),
	("SPEC CPU2000", "300.twolf", "c"),
	("SPEC CPU2000", "168.wupwise", "fortran77"),
	("SPEC CPU2000", "171.swim", "fortran77"),
	("SPEC CPU2000", "172.mgrid", "fortran77"),
	("SPEC CPU2000", "173.applu", "fortran77"),
	("SPEC CPU2000", "177.mesa", "c"),
	("SPEC CPU2000", "178.galgel", "fortran90"),
	("SPEC CPU2000", "179.art", "c"),
	("SPEC CPU2000", "183.equake", "c"),
	("SPEC CPU2000", "187.facerec", "fortran90"),
	("SPEC CPU2000", "188.ammp", "c"),
	("SPEC CPU2000", "189.lucas", "fortran90"),
	("SPEC CPU2000", "191.fma3d", "fortran90"),
	("SPEC CPU2000", "200.sixtrack", "fortran77"),
	("SPEC CPU2000", "301.apsi", "fortran77"),
	("SPEC OMP2001", "310.wupwise_m", "?"),
	("SPEC OMP2001", "311.wupwise_l", "?"),
	("SPEC OMP2001", "312.swim_m", "?"),
	("SPEC OMP2001", "313.swim_l", "?"),
	("SPEC OMP2001", "314.mgrid_m", "?"),
	("SPEC OMP2001", "315.mgrid_l", "?"),
	("SPEC OMP2001", "316.applu_m", "?"),
	("SPEC OMP2001", "317.applu_l", "?"),
	("SPEC OMP2001", "318.galgel_m", "?"),
	("SPEC OMP2001", "330.art_m", "?"),
	("SPEC OMP2001", "331.art_l", "?"),
	("SPEC OMP2001", "320.equake_m", "?"),
	("SPEC OMP2001", "321.equake_l", "?"),
	("SPEC OMP2001", "332.ammp_m", "?"),
	("SPEC OMP2001", "328.fma3d_m", "?"),
	("SPEC OMP2001", "329.fma3d_l", "?"),
	("SPEC OMP2001", "324.apsi_m", "?"),
	("SPEC OMP2001", "325.apsi_l", "?"),
	("SPEC OMP2001", "326.gafort_m", "?"),
	("SPEC OMP2001", "327.gafort_l", "?"),
	("SPEC CPU2006", "400.perlbench", "c"),
	("SPEC CPU2006", "401.bzip2", "c"),
	("SPEC CPU2006", "403.gcc", "c"),
	("SPEC CPU2006", "429.mcf", "c"),
	("SPEC CPU2006", "445.gobmk", "c"),
	("SPEC CPU2006", "456.hmmer", "c"),
	("SPEC CPU2006", "458.sjeng", "c"),
	("SPEC CPU2006", "462.libquantum", "c"),
	("SPEC CPU2006", "464.h264ref", "c"),
	("SPEC CPU2006", "471.omnetpp", "cpp"),
	("SPEC CPU2006", "473.astar", "cpp"),
	("SPEC CPU2006", "483.xalancbmk", "cpp"),
	("SPEC CPU2006", "410.bwaves", "fortran"),
	("SPEC CPU2006", "416.gamess", "fortran"),
	("SPEC CPU2006", "433.milc", "c"),
	("SPEC CPU2006", "434.zeusmp", "fortran"),
	("SPEC CPU2006", "435.gromacs", "c,fortran"),
	("SPEC CPU2006", "436.cactusADM", "c,fortran"),
	("SPEC CPU2006", "437.leslie3d", "fortran"),
	("SPEC CPU2006", "444.namd", "cpp"),
	("SPEC CPU2006", "447.dealII", "cpp"),
	("SPEC CPU2006", "450.soplex", "cpp"),
	("SPEC CPU2006", "453.povray", "cpp"),
	("SPEC CPU2006", "454.calculix", "c,fortran"),
	("SPEC CPU2006", "459.GemsFDTD", "fortran"),
	("SPEC CPU2006", "465.tonto", "fortran"),
	("SPEC CPU2006", "470.lbm", "c"),
	("SPEC CPU2006", "481.wrf", "c,fortran"),
	("SPEC CPU2006", "482.sphinx3", "c"),
]

def filter_progs(options):
	for suite, prog, langs in BENCHMARK_LANGUAGES:
		if suite != SPEC_SUITE:
			continue
		if options.no_cpp and "cpp" in langs:
			continue
		if options.no_fortran and "fortran" in langs:
			continue
		if options.no_medium and prog.endswith("_m"):
			continue
		if options.no_large and prog.endswith("_l"):
			continue
		yield prog

Andreas Zwinkau's avatar
Andreas Zwinkau committed
338
339
340
341
342
def generate_spec_config(options):
	"""Generate a config file for SPEC"""
	contents = [_CONFIG_HEAD]
	cpuinfo = get_cpu_info()
	meminfo = get_mem_info()
343
344
345
346
	if options.build_only:
		contents.append("action = build")
	else:
		contents.append("action = validate") # includes run
347
	contents.append("iterations = %d" % options.iterations)
Andreas Zwinkau's avatar
Andreas Zwinkau committed
348
349
	if SPEC_SUITE != "SPEC CPU2006":
		contents.append("test_date = %s" % datetime.now())
350
	contents.append("")
Andreas Zwinkau's avatar
Andreas Zwinkau committed
351
	contents.append("sw_os = %s %s %s" % (LINUX_NAME, LINUX_VERSION, LINUX_CODENAME))
352
353
354
	contents.append("sw_compiler1 = %s" % get_compiler_version(options.c_compiler))
	contents.append("sw_compiler2 = %s" % get_compiler_version(options.cpp_compiler))
	contents.append("sw_compiler3 = %s" % get_compiler_version(options.fortran_compiler))
Andreas Zwinkau's avatar
Andreas Zwinkau committed
355
	contents.append("sw_file = %s" % path_to_filesystem("."))
Andreas Zwinkau's avatar
Andreas Zwinkau committed
356
	assert int(BITSIZE), "bitsize is "+BITSIZE
Andreas Zwinkau's avatar
Andreas Zwinkau committed
357
358
359
	contents.append("sw_base_ptrsize = %d-bit" % BITSIZE)
	contents.append("sw_peak_ptrsize = %d-bit" % BITSIZE)
	contents.append("")
Andreas Zwinkau's avatar
Andreas Zwinkau committed
360
361
362
	chips = (1+int(cpuinfo["processor"]))
	cores_per_chip = int(cpuinfo["cpu cores"])
	cores = cores_per_chip * chips
363
364
	if SPEC_SUITE == "SPEC CPU2006":
		contents.append("hw_cpu_name = %s" % cpuinfo["model name"])
Andreas Zwinkau's avatar
Andreas Zwinkau committed
365
366
		contents.append("hw_ncores = %d" % cores)
	elif SPEC_SUITE == "SPEC CPU2000":
367
		contents.append("hw_model = %s" % cpuinfo["model name"])
Andreas Zwinkau's avatar
Andreas Zwinkau committed
368
		contents.append("hw_ncpu = %d" % cores)
Andreas Zwinkau's avatar
Andreas Zwinkau committed
369
	contents.append("hw_cpu_mhz = %s" % cpuinfo["cpu MHz"])
Andreas Zwinkau's avatar
Andreas Zwinkau committed
370
371
	contents.append("hw_nchips = %d" % chips)
	contents.append("hw_ncoresperchip = %d" % cores_per_chip)
Andreas Zwinkau's avatar
Andreas Zwinkau committed
372
373
	contents.append("hw_vendor = %s" % cpuinfo["vendor_id"])
	contents.append("hw_memory = %s" % meminfo["MemTotal"])
Andreas Zwinkau's avatar
Andreas Zwinkau committed
374
	contents.append("hw_pcache = %s" % cpuinfo["cache size"])
Andreas Zwinkau's avatar
Andreas Zwinkau committed
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
	contents.append("")
	contents.append("CC = %s" % options.c_compiler)
	contents.append("CXX = %s" % options.cpp_compiler)
	contents.append("FC = %s" % options.fortran_compiler)
	contents.append("")
	contents.append("OPTIMIZE = %s" % options.compiler_args)
	contents.append("notes_comp_001 OPTIMIZE = %s" % options.compiler_args)
	contents.append("COPTIMIZE = %s" % options.c_compiler_args)
	contents.append("notes_comp_002 COPTIMIZE = %s" % options.c_compiler_args)
	contents.append("CXXOPTIMIZE = %s" % options.cpp_compiler_args)
	contents.append("notes_comp_003 CXXOPTIMIZE = %s" % options.cpp_compiler_args)
	contents.append("FOPTIMIZE = %s" % options.fortran_compiler_args)
	contents.append("notes_comp_004 FOPTIMIZE = %s" % options.fortran_compiler_args)
	contents.append(get_portability())
	return "\n".join(contents)

if __name__ == "__main__":
	(options, args) = parser.parse_args()
	conf = generate_spec_config(options)
Andreas Zwinkau's avatar
Andreas Zwinkau committed
394
	fh = open("config/generated.cfg", 'w')
Andreas Zwinkau's avatar
Andreas Zwinkau committed
395
396
	fh.write(conf)
	fh.close()
Andreas Zwinkau's avatar
Andreas Zwinkau committed
397
398
	if options.config_only:
		exit(0)
399
400
	progs = " ".join(filter_progs(options))
	cmd = "env bash -l -c 'source shrc && runspec %s -c generated'" % progs
Andreas Zwinkau's avatar
Andreas Zwinkau committed
401
402
	os.system(cmd)