Commit b8fe0c75 authored by Christian Baumann's avatar Christian Baumann
Browse files

Changes from Christian Baumann

- esp. logging of RAM utilization
parent eac88411
...@@ -17,6 +17,7 @@ import sys ...@@ -17,6 +17,7 @@ import sys
import traceback import traceback
import math import math
import json import json
import signal
from collections import namedtuple from collections import namedtuple
...@@ -25,6 +26,10 @@ import curses_display as ui ...@@ -25,6 +26,10 @@ import curses_display as ui
from logging import LoggingManager from logging import LoggingManager
from psutil_functions import calculate_cpu_times_percent from psutil_functions import calculate_cpu_times_percent
def signal_handler(signal, frame):
global running
running = False
def get_time(): def get_time():
""" Unified/comparable clock access """ """ Unified/comparable clock access """
...@@ -94,7 +99,7 @@ class Measurement: ...@@ -94,7 +99,7 @@ class Measurement:
self.timespan = self.r2.timestamp - self.r1.timestamp self.timespan = self.r2.timestamp - self.r1.timestamp
self.cpu_times_percent = calculate_cpu_times_percent(self.r1.cpu_times, self.r2.cpu_times, percpu=True) self.cpu_times_percent = calculate_cpu_times_percent(self.r1.cpu_times, self.r2.cpu_times, percpu=True)
self.net_io = self._calculate_net_io() self.net_io = self._calculate_net_io()
self.memory = psutil.virtual_memory()
def _calculate_net_io(self): def _calculate_net_io(self):
ret = dict() ret = dict()
...@@ -133,6 +138,7 @@ def main_loop(): ...@@ -133,6 +138,7 @@ def main_loop():
- Displays the measurements - Displays the measurements
- Logs the measurements with the LoggingManager - Logs the measurements with the LoggingManager
""" """
signal.signal(signal.SIGINT, signal_handler)
## TODO this should be configurable by command line options ## TODO this should be configurable by command line options
sample_interval = float(args.interval) sample_interval = float(args.interval)
...@@ -158,6 +164,8 @@ def main_loop(): ...@@ -158,6 +164,8 @@ def main_loop():
time.sleep(math.ceil(now)-now) time.sleep(math.ceil(now)-now)
display_skip_counter = 0 display_skip_counter = 0
global running
running = True running = True
while running: while running:
# Take a new reading. # Take a new reading.
...@@ -168,6 +176,7 @@ def main_loop(): ...@@ -168,6 +176,7 @@ def main_loop():
# Display/log the measurement. # Display/log the measurement.
running &= logging_manager.log(measurement) running &= logging_manager.log(measurement)
running &= not logging_manager.get_killstate()
if ( display_skip_counter % display_skip < 1 ) and not args.headless: # the display may skip some samples if ( display_skip_counter % display_skip < 1 ) and not args.headless: # the display may skip some samples
running = ui.display( measurement ) running = ui.display( measurement )
display_skip_counter = 0 display_skip_counter = 0
...@@ -232,6 +241,8 @@ if __name__ == "__main__": ...@@ -232,6 +241,8 @@ if __name__ == "__main__":
help="Time between two samples (in seconds). [Default = 0.5]") help="Time between two samples (in seconds). [Default = 0.5]")
parser.add_argument("-d", "--displayinterval", default="1", parser.add_argument("-d", "--displayinterval", default="1",
help="Time between two display updates (in seconds). [Default = 1]") help="Time between two display updates (in seconds). [Default = 1]")
parser.add_argument("-k", "--killswitch", action="store_true",
help="Stop auto-logging after first traffic. Implies --autologging")
# NICs # NICs
...@@ -251,6 +262,10 @@ if __name__ == "__main__": ...@@ -251,6 +262,10 @@ if __name__ == "__main__":
#assert( set(nics).issuperset(args.nics) ) #assert( set(nics).issuperset(args.nics) )
monitored_nics = args.nics monitored_nics = args.nics
## --killswitch implies --autologging
if ( args.killswitch ):
args.autologging = True
## --autologging implies --logging ## --autologging implies --logging
if ( args.autologging ): if ( args.autologging ):
args.logging = True args.logging = True
...@@ -258,7 +273,7 @@ if __name__ == "__main__": ...@@ -258,7 +273,7 @@ if __name__ == "__main__":
## Logging ## Logging
logging_manager = LoggingManager( psutil.cpu_count(), monitored_nics, helpers.get_sysinfo(), args.environment, logging_manager = LoggingManager( psutil.cpu_count(), monitored_nics, helpers.get_sysinfo(), args.environment,
args.comment, args.path, args.autologging, args.watch ) args.comment, args.path, args.autologging, args.watch, args.killswitch )
if args.logging: if args.logging:
logging_manager.enable_measurement_logger() logging_manager.enable_measurement_logger()
......
...@@ -337,7 +337,24 @@ def _display(measurement): ...@@ -337,7 +337,24 @@ def _display(measurement):
stdscr.addstr(y, LABEL_Received, 'Received:', curses.color_pair(2)) stdscr.addstr(y, LABEL_Received, 'Received:', curses.color_pair(2))
stdscr.addstr(y, LABEL_Received+10, '{0} {1}/s'.format(_format_net_speed(sum_receiving),unit), curses.color_pair(3)) stdscr.addstr(y, LABEL_Received+10, '{0} {1}/s'.format(_format_net_speed(sum_receiving),unit), curses.color_pair(3))
'''
#RAM
y += 1
stdscr.hline(y, 1, "-", 78)
y += 1
#vmem(total=135092002816, available=134568681472, percent=0.4, used=1066811392, free=134025191424, active=524701696, inactive=43393024, buffers=153694208, cached=389795840)
stdscr.addstr(y, 1, "RAM", curses.color_pair(1))
for field in measurement.memory._fields:
value = getattr(measurement.memory, field)
if field != 'percent':
value = bytes2human(value)
stdscr.addstr(y, LABEL_Sent, '{0}:'.format( field.capitalize()), curses.color_pair(2))
stdscr.addstr(y, LABEL_Sent+11, '{0:>7}'.format( value ), curses.color_pair(3))
y += 1
'''
## Show logging comment ## Show logging comment
comment = logging_manager.get_logging_comment() comment = logging_manager.get_logging_comment()
...@@ -363,3 +380,20 @@ def close(): ...@@ -363,3 +380,20 @@ def close():
curses.echo() curses.echo()
curses.curs_set(True) curses.curs_set(True)
curses.endwin() curses.endwin()
def bytes2human(n):
# http://code.activestate.com/recipes/578019
# >>> bytes2human(10000)
# '9.8K'
# >>> bytes2human(100001221)
# '95.4M'
symbols = ('K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')
prefix = {}
for i, s in enumerate(symbols):
prefix[s] = 1 << (i + 1) * 10
for s in reversed(symbols):
if n >= prefix[s]:
value = float(n) / prefix[s]
return '%.1f%s' % (value, s)
return "%sB" % n
...@@ -13,7 +13,7 @@ ...@@ -13,7 +13,7 @@
import os import os
import netifaces import netifaces
import operator import operator
import psutil
def get_nics(): def get_nics():
return netifaces.interfaces() return netifaces.interfaces()
...@@ -98,12 +98,12 @@ def get_sysinfo(): ...@@ -98,12 +98,12 @@ def get_sysinfo():
#uname = os.uname() #uname = os.uname()
#input_fields = ("sysname", "nodename", "release", "version", "machine") #input_fields = ("sysname", "nodename", "release", "version", "machine")
output_fields = ("sysname", "hostname", "kernel", "version", "machine") output_fields = ("sysname", "hostname", "kernel", "version", "machine", "memory")
ret = dict() ret = dict()
for out_field, value in zip(output_fields, os.uname()): for out_field, value in zip(output_fields, os.uname()):
ret[out_field] = value ret[out_field] = value
ret["memory"] = psutil.virtual_memory().total
return ret return ret
......
...@@ -46,7 +46,7 @@ class MeasurementLogger: ...@@ -46,7 +46,7 @@ class MeasurementLogger:
self.filename = filename self.filename = filename
## Constants / Characteristics ## Constants / Characteristics
self.class_names = ("Time", "CPU", "NIC") self.class_names = ("Time", "CPU", "NIC", "NICStats", "Memory")
self.type_string = "CPUnetLOG:MeasurementLog" self.type_string = "CPUnetLOG:MeasurementLog"
## Run "outsourced" init functions. ## Run "outsourced" init functions.
...@@ -68,6 +68,8 @@ class MeasurementLogger: ...@@ -68,6 +68,8 @@ class MeasurementLogger:
self.log_functions["Time"] = self._log_time self.log_functions["Time"] = self._log_time
self.log_functions["CPU"] = self._log_cpus self.log_functions["CPU"] = self._log_cpus
self.log_functions["NIC"] = self._log_nics self.log_functions["NIC"] = self._log_nics
self.log_functions["NICStats"] = self._log_nic_stats
self.log_functions["Memory"] = self._log_memory
## Initialize file writer. ## Initialize file writer.
...@@ -96,6 +98,13 @@ class MeasurementLogger: ...@@ -96,6 +98,13 @@ class MeasurementLogger:
description = "Network traffic (Bits/s)" ) description = "Network traffic (Bits/s)" )
class_defs["NIC"] = nic class_defs["NIC"] = nic
# set up "NICStats" class
nicstats = LoggingClass( name = "NICStats",
fields = ("errin", "errout", "dropin", "dropout"),
siblings = nics,
description = "Network Interface stats (errors and drops)" )
class_defs["NICStats"] = nicstats
# set up "Time" class # set up "Time" class
time = LoggingClass( name = "Time", time = LoggingClass( name = "Time",
fields = ("begin", "end", "duration"), fields = ("begin", "end", "duration"),
...@@ -103,6 +112,13 @@ class MeasurementLogger: ...@@ -103,6 +112,13 @@ class MeasurementLogger:
description = "Begin, end, and duration (in seconds) of this measurement." ) description = "Begin, end, and duration (in seconds) of this measurement." )
class_defs["Time"] = time class_defs["Time"] = time
# set up "Memory" class
memory = LoggingClass( name = "Memory",
fields = ("used", "active", "inactive", "buffers", "cached"),
siblings = None,
description = "Memory utilization" )
class_defs["Memory"] = memory
return class_defs return class_defs
...@@ -181,6 +197,14 @@ class MeasurementLogger: ...@@ -181,6 +197,14 @@ class MeasurementLogger:
## TODO: is 0 a good value to log, in this case? ## TODO: is 0 a good value to log, in this case?
out_vector.extend( (0, 0) ) out_vector.extend( (0, 0) )
def _log_nic_stats(self, measurement, out_vector):
for nic in self.nics:
values = measurement.net_io[nic]
out_vector.extend( [values.total["errin"], values.total["errout"], values.total["dropin"], values.total["dropout"]] )
def _log_memory(self, measurement, out_vector):
out_vector.extend( [measurement.memory.used, measurement.memory.active, measurement.memory.inactive, measurement.memory.buffers, measurement.memory.cached] )
def log(self, measurement): def log(self, measurement):
out_vector = list() out_vector = list()
...@@ -268,7 +292,7 @@ class CNLFileWriter: ...@@ -268,7 +292,7 @@ class CNLFileWriter:
class LoggingManager: class LoggingManager:
def __init__(self, num_cpus, nics, system_info, environment, comment, path, autologging, watch_experiment): def __init__(self, num_cpus, nics, system_info, environment, comment, path, autologging, watch_experiment, killswitch):
self.num_cpus = num_cpus self.num_cpus = num_cpus
self.nics = nics self.nics = nics
self.comment = comment self.comment = comment
...@@ -278,9 +302,12 @@ class LoggingManager: ...@@ -278,9 +302,12 @@ class LoggingManager:
self.hostname = system_info["hostname"] self.hostname = system_info["hostname"]
self.environment = environment self.environment = environment
self.watch_experiment = watch_experiment self.watch_experiment = watch_experiment
self.killswitch = killswitch
self.killstate = False
# auto-logging # auto-logging
self.INACTIVITY_THRESHOLD = 30 # seconds self.INACTIVITY_THRESHOLD = 10 # seconds
# self.INACTIVITY_THRESHOLD = 30 # seconds
self.HISTORY_SIZE = 5 # samples self.HISTORY_SIZE = 5 # samples
self.auto_logging = autologging self.auto_logging = autologging
if ( autologging ): if ( autologging ):
...@@ -365,6 +392,9 @@ class LoggingManager: ...@@ -365,6 +392,9 @@ class LoggingManager:
self.measurement_logger = None self.measurement_logger = None
self.auto_comment = None self.auto_comment = None
if(self.killswitch):
self.killstate = True
## experimental "tcp_probe" ## experimental "tcp_probe"
## kill "cat /proc/net/tcpprobe > file" ## kill "cat /proc/net/tcpprobe > file"
#if ( self.use_tcpprobe ): #if ( self.use_tcpprobe ):
...@@ -513,6 +543,8 @@ class LoggingManager: ...@@ -513,6 +543,8 @@ class LoggingManager:
def get_logging_comment(self): def get_logging_comment(self):
return self.auto_comment if self.auto_comment else self.comment return self.auto_comment if self.auto_comment else self.comment
def get_killstate(self):
return self.killstate
def close(self): def close(self):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment