Commit f5f05487 authored by Michael König (Student)'s avatar Michael König (Student)
Browse files

refactoring++ & better auto-scaling

parent 34a7f0e9
...@@ -5,3 +5,4 @@ __pycache__/ ...@@ -5,3 +5,4 @@ __pycache__/
.old .old
*.egg-info *.egg-info
*.pyc *.pyc
todo.txt
TCPlivePLOT is based on Python3. TCPlivePLOT
================================================================================
Realtime-visualization of TCP flows logged by TCPlog.
TCPlivePLOT is based on Python3 and tested on GNU/Linux 4.{1-7}.
For TCPlog see https://git.scc.kit.edu/CPUnetLOG/TCPlog/
Tested on GNU/Linux 4.{1-6}
Requirements TCPlivePLOT: Requirements TCPlivePLOT:
================================================================================ --------------------------------------------------------------------------------
* python3 * python3
* python-matplotlib * matplotlib
* numpy
* matplotlib-interactive backend * matplotlib-interactive backend
recommended:
* recommended backends:
* Qt4Agg or * Qt4Agg or
* Qt5Agg * Qt5Agg
Running TCPplot: Running TCPlivePLOT:
================================================================================ --------------------------------------------------------------------------------
* ./tcpliveplot.py OR * ./tcpliveplot.py OR
* python3 -m TCPlivePLOT OR * python3 -m TCPlivePLOT OR
* tcpliveplot (after installation) * tcpliveplot (after installation)
Installation of TCPplot: Installation of TCPlivePLOT:
================================================================================ --------------------------------------------------------------------------------
* via pip3: * via pip3:
* sudo pip3 install . # system-wide installation * sudo pip3 install . # system-wide installation
* pip3 install --user . # local installation * pip3 install --user . # local installation
* via setup.py:
* python3 setup.py install # system-wide installation
* python3 setup.py install --user # local installation
...@@ -3,9 +3,7 @@ ...@@ -3,9 +3,7 @@
from setuptools import setup from setuptools import setup
with open("VERSION.txt", "rb") as f: version = 0.1.0
version = f.read().decode("utf-8")
desc = "Realtime-visualization of TCP flows logged by TCPlog." desc = "Realtime-visualization of TCP flows logged by TCPlog."
setup( setup(
...@@ -41,7 +39,7 @@ setup( ...@@ -41,7 +39,7 @@ setup(
# #
], ],
extras_require = { extras_require = {
'Logging': ["tcplog>=0.2"] 'Logging': ["tcplog"]
}, },
keywords = ['tcp', 'flow', 'plot', 'visualize', 'graph', 'live', 'analyze', 'network', 'traffic'], keywords = ['tcp', 'flow', 'plot', 'visualize', 'graph', 'live', 'analyze', 'network', 'traffic'],
classifiers = [ classifiers = [
......
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
"""Convenience wrapper for running TCPlivePLOT directly from source tree.""" """Convenience wrapper for running TCPlivePLOT directly from source tree."""
from TCPlivePLOT.main import main from tcpliveplot.main import main
if __name__ == '__main__': if __name__ == '__main__':
main() main()
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# TODO:
# * dynamic flow detection
# * flowIdentifer i/o "port"
# * fix socket-input
# * dynamic value detection
import matplotlib import matplotlib
matplotlib.use('Qt5Agg') matplotlib.use('Qt4Agg')
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import matplotlib.animation as animation import matplotlib.animation as animation
from matplotlib.widgets import Button from matplotlib.widgets import Button, RadioButtons
import numpy as np import numpy as np
import math
import sys
import time import time
import threading
from collections import deque from collections import deque
from .gui_base import GuiBase from .gui_base import GuiBase
VALUES_TO_PLOT = ['cwnd', 'sst', 'rtt', 'bw'] # (only values for Y-axis) VALUES_TO_PLOT = ['cwnd', 'sst', 'rtt', 'smoothedThroughput'] # (only values for Y-axis)
VALUES_TO_PROCESS = ['time'] + VALUES_TO_PLOT #helper to init all data structures VALUES_TO_PROCESS = ['time'] + VALUES_TO_PLOT #helper to init all data structures
# Strings for UI-elements # Strings for UI-elements
...@@ -25,14 +34,19 @@ QUIT = "Quit" ...@@ -25,14 +34,19 @@ QUIT = "Quit"
CLEAR_GAP = 0.2 # gap in s CLEAR_GAP = 0.2 # gap in s
INFINITY_THRESHOLD = 1e8 INFINITY_THRESHOLD = 1e8
class LiveGui(GuiBase): class LiveGui(GuiBase):
def __init__(self, options, infoRegistry): def __init__(self, options, infoRegistry):
self.options = options self.options = options
self.infoRegistry = infoRegistry self.infoRegistry = infoRegistry
self.__stopped = threading.Event()
self.timestampOfLastGuiRefresh = 0
def setConnectionBuffer(self, connectionBuffer):
self.__connectionBuffer = connectionBuffer
def tearDown(self): def tearDown(self):
plt.close()
sys.exit(0)
pass pass
def startUp(self): def startUp(self):
...@@ -53,6 +67,23 @@ class LiveGui(GuiBase): ...@@ -53,6 +67,23 @@ class LiveGui(GuiBase):
self.__plotLines[port][lineID].set_visible(self.__plotLineConfigs[port][lineID]) self.__plotLines[port][lineID].set_visible(self.__plotLineConfigs[port][lineID])
self.drawPlotLegend() self.drawPlotLegend()
def updateValueVisibility(self, label):
for port in self.options.filterPorts:
for i in range(1, len(VALUES_TO_PLOT)+1):
self.__plotLineConfigs[port][VALUES_TO_PLOT[(i-1)]] = False
self.__plotLines[port][(VALUES_TO_PLOT[(i-1)])].set_visible(self.__plotLineConfigs[port][(VALUES_TO_PLOT[(i-1)])])
if label == 'cwnd':
self.toggleVisibility(VALUES_TO_PLOT[0])
elif label == 'sst':
self.toggleVisibility(VALUES_TO_PLOT[1])
elif label == 'rtt':
self.toggleVisibility(VALUES_TO_PLOT[2])
elif label == 'bw':
self.toggleVisibility(VALUES_TO_PLOT[3])
else:
pass
def drawPlotLegend(self): def drawPlotLegend(self):
"""(Re)draws legend with visible lines.""" """(Re)draws legend with visible lines."""
labelObjs = [] labelObjs = []
...@@ -91,7 +122,6 @@ class LiveGui(GuiBase): ...@@ -91,7 +122,6 @@ class LiveGui(GuiBase):
def stopPlotting(self, event): def stopPlotting(self, event):
"""Callback function to stop plotting and the programm.""" """Callback function to stop plotting and the programm."""
self.__tmpTimestamp = time.perf_counter() self.__tmpTimestamp = time.perf_counter()
plt.close()
self.tearDown() self.tearDown()
def plotGraph(self): def plotGraph(self):
...@@ -100,7 +130,6 @@ class LiveGui(GuiBase): ...@@ -100,7 +130,6 @@ class LiveGui(GuiBase):
self.__minVal = 9999999999 self.__minVal = 9999999999
self.__maxVal = 0 self.__maxVal = 0
fig = plt.figure(FIGURE_TITLE) fig = plt.figure(FIGURE_TITLE)
fig.canvas.mpl_connect('key_press_event', self.plotKeyPressCallback) fig.canvas.mpl_connect('key_press_event', self.plotKeyPressCallback)
self.__ax = plt.axes() self.__ax = plt.axes()
...@@ -110,13 +139,19 @@ class LiveGui(GuiBase): ...@@ -110,13 +139,19 @@ class LiveGui(GuiBase):
self.__plotLines = {} self.__plotLines = {}
self.__plotValues = {} self.__plotValues = {}
self.__plotValuesMin = {}
self.__plotValuesMax = {}
self.__plotLineConfigs = {} self.__plotLineConfigs = {}
for port in self.options.filterPorts: for port in self.options.filterPorts:
self.__plotLines[port] = {} self.__plotLines[port] = {}
self.__plotValues[port] = {} self.__plotValues[port] = {}
self.__plotValuesMin[port] = {}
self.__plotValuesMax[port] = {}
self.__plotLineConfigs[port] = {} self.__plotLineConfigs[port] = {}
self.__plotLineConfigs[port]['lastTimestamp'] = 0 self.__plotLineConfigs[port]['lastTimestamp'] = 0
for val in VALUES_TO_PROCESS: for val in VALUES_TO_PROCESS:
self.__plotValuesMin[port][val] = math.inf
self.__plotValuesMax[port][val] = -math.inf
self.__plotValues[port][val] = deque(maxlen=(int(self.options.xDelta / self.options.plotResolution * 10))) self.__plotValues[port][val] = deque(maxlen=(int(self.options.xDelta / self.options.plotResolution * 10)))
index = 1 index = 1
for val in VALUES_TO_PLOT: for val in VALUES_TO_PLOT:
...@@ -137,6 +172,15 @@ class LiveGui(GuiBase): ...@@ -137,6 +172,15 @@ class LiveGui(GuiBase):
quitButton = Button(quitAx, QUIT) quitButton = Button(quitAx, QUIT)
quitButton.on_clicked(self.stopPlotting) quitButton.on_clicked(self.stopPlotting)
# valueCheckboxesAx = plt.axes([0.05, 0.4, 0.1, 0.15])
# valueCheckboxes = CheckButtons(valueCheckboxesAx, ('cwnd', 'sst', 'rtt', 'bw'), (True, False, False, False))
# valueCheckboxes.on_clicked(self.updateValueVisibility)
# valueRadiobuttonsAx = plt.axes([0.020, 0.025, 0.075, 0.15])
# valueRadiobuttons = RadioButtons(valueRadiobuttonsAx, ('cwnd', 'sst', 'rtt', 'bw'))
# valueRadiobuttons.on_clicked(self.updateValueVisibility)
if(self.options.preloadBuffer > 0): if(self.options.preloadBuffer > 0):
self.__preloading = True self.__preloading = True
else: else:
...@@ -154,12 +198,17 @@ class LiveGui(GuiBase): ...@@ -154,12 +198,17 @@ class LiveGui(GuiBase):
self.__apsFixFactor = 1 self.__apsFixFactor = 1
# call update-routine # call update-routine
animation.FuncAnimation(fig, self.plotGraphUpdate, init_func=self.plotInit, print("foo")
frames=self.options.drawFps, interval=self.options.drawIntervall, blit=self.options.blitting, repeat=True) # self.plotInit()
if self.__stopped.isSet(): # self.__plotLine = self.plotGraphUpdate(0)
return animation.FuncAnimation(fig, self.plotGraphUpdate, init_func=self.plotInit, frames=self.options.drawFps, interval=self.options.drawIntervall, blit=self.options.blitting, repeat=True)
else: # if self.__stopped.isSet():
plt.show() # return
# else:
# plt.ioff()
# plt.draw()
plt.show()
# print("bar")
def returnAllLines(self): def returnAllLines(self):
"""Macro to return all lines as they are.""" """Macro to return all lines as they are."""
...@@ -187,7 +236,7 @@ class LiveGui(GuiBase): ...@@ -187,7 +236,7 @@ class LiveGui(GuiBase):
return self.returnAllLines() return self.returnAllLines()
# fill playback-buffer # fill playback-buffer
if(self.__preloading): if(False and self.__preloading):
bufferLength = -1 bufferLength = -1
for port in self.options.filterPorts: for port in self.options.filterPorts:
bufferLength = max(bufferLength, len(self.__connectionBuffer[port])) bufferLength = max(bufferLength, len(self.__connectionBuffer[port]))
...@@ -219,8 +268,8 @@ class LiveGui(GuiBase): ...@@ -219,8 +268,8 @@ class LiveGui(GuiBase):
newXmin = newXmax - self.options.xDelta newXmin = newXmax - self.options.xDelta
self.__ax.set_xlim(newXmin, newXmax) self.__ax.set_xlim(newXmin, newXmax)
maxYval = 0 maxYval = -math.inf
minYval = 9999999 minYval = math.inf
connectionsData = {} connectionsData = {}
# skip this cycle, plot resolution not yet reached # skip this cycle, plot resolution not yet reached
...@@ -237,7 +286,7 @@ class LiveGui(GuiBase): ...@@ -237,7 +286,7 @@ class LiveGui(GuiBase):
whileRun = False whileRun = False
pass pass
else: else:
lineTime = self.__initRealtimeTimestamp + (data['time'] - self.__initSampletimeTimestamp) lineTime = self.__initRealtimeTimestamp + (float(data['time']) - self.__initSampletimeTimestamp)
# time in past # time in past
if(lineTime < newXmin): if(lineTime < newXmin):
continue continue
...@@ -254,8 +303,13 @@ class LiveGui(GuiBase): ...@@ -254,8 +303,13 @@ class LiveGui(GuiBase):
connectionsData[port].append(nanSample) connectionsData[port].append(nanSample)
infinityReached = False infinityReached = False
for val in VALUES_TO_PLOT: for val in VALUES_TO_PLOT:
if(data[val] > INFINITY_THRESHOLD): try:
convertedValue = float(data[val])
except ValueError:
data[val] = np.nan data[val] = np.nan
else:
if(convertedValue > INFINITY_THRESHOLD):
data[val] = np.nan
# nanSample = self.returnNanSample(lineTime) # nanSample = self.returnNanSample(lineTime)
# connectionsData[port].append(nanSample) # connectionsData[port].append(nanSample)
# infinityReached = True # infinityReached = True
...@@ -279,58 +333,81 @@ class LiveGui(GuiBase): ...@@ -279,58 +333,81 @@ class LiveGui(GuiBase):
# copy raw-value into corresponding lists
for connection in connectionsData: for connection in connectionsData:
while(len(connectionsData[connection]) > 0): while(len(connectionsData[connection]) > 0):
data = connectionsData[connection].popleft() data = connectionsData[connection].popleft()
lineTime = self.__initRealtimeTimestamp + (data['time'] - self.__initSampletimeTimestamp) lineTime = self.__initRealtimeTimestamp + (float(data['time']) - self.__initSampletimeTimestamp)
self.__plotLineConfigs[connection]['lastTimestamp'] = data['time'] self.__plotLineConfigs[connection]['lastTimestamp'] = float(data['time'])
for val in VALUES_TO_PROCESS: for val in VALUES_TO_PROCESS:
if(val == 'time'): if(val == 'time'):
self.__plotValues[connection][val].append(lineTime) self.__plotValues[connection][val].append(lineTime)
else: else:
self.__plotValues[connection][val].append(data[val]) try:
currentVal = float(data[val])
except ValueError:
pass
else:
self.__plotValues[connection][val].append(currentVal)
# update axis (xy-tuple) with data from lists
for val in VALUES_TO_PLOT: for val in VALUES_TO_PLOT:
self.__plotLines[connection][val].set_data(self.__plotValues[connection]['time'], self.__plotValues[connection][val]) x, y = self.__plotValues[connection]['time'], self.__plotValues[connection][val]
self.__plotLines[connection][val].set_data(x, y)
for port in self.options.filterPorts: self.__lastDrawTimestamp = time.perf_counter()
for val in VALUES_TO_PLOT:
if(self.__plotLineConfigs[port][val]):
if(len(self.__plotValues[port][val]) > 0):
maxYval = max(maxYval, max(self.__plotValues[port][val]))
minYval = min(minYval, min(self.__plotValues[port][val]))
if(maxYval > 0): # y-scaling
newYmin = minYval - 50 lines = self.__ax.get_lines()
newYmin = max(newYmin, 0) bot,top = np.inf, -np.inf
newYmax = maxYval * 2 for line in lines:
self.__ax.set_ylim(newYmin, newYmax) if(line.get_visible()):
new_bot, new_top = self.determineNewYvalues(line)
if(new_bot != new_top):
if(new_bot < bot):
bot = new_bot
if(new_top > top):
top = new_top
if(bot != np.inf and top != -np.inf):
self.__ax.set_ylim(bot, top)
else:
# intial y-scale
self.__ax.set_ylim(0, 500)
self.__lastDrawTimestamp = time.perf_counter()
return self.returnAllLines() return self.returnAllLines()
def determineNewYvalues(self, line, margin=0.25):
xLine = line.get_xdata()
yLine = line.get_ydata()
xData = np.array(xLine)
yData = np.array(yLine)
low,high = self.__ax.get_xlim()
yVisibleMask = yData[((xData>low) & (xData<high))]
if(len(yVisibleMask) > 0):
height = np.max(yVisibleMask) - np.min(yVisibleMask)
bot = np.min(yVisibleMask) - margin * height
top = np.max(yVisibleMask) + margin * height
return bot,top
else:
return 0,0
def plotInit(self): def plotInit(self):
"""Helper to initialize plot.""" """Helper to initialize plot."""
for port in self.options.filterPorts: for port in self.options.filterPorts:
for val in VALUES_TO_PLOT: for val in VALUES_TO_PLOT:
self.__plotLines[port][val].set_data([], []) self.__plotLines[port][val].set_data([], [])
# hide "invisible" lines
if(val not in self.options.linesToShow):
self.__plotLineConfigs[port][val] = False
self.__plotLines[port][val].set_visible(False)
newXmin = 0 newXmin = 0
newXmax = newXmin + self.options.xDelta newXmax = newXmin + self.options.xDelta
self.__ax.set_xlim(newXmin, newXmax) self.__ax.set_xlim(newXmin, newXmax)
if(self.options.debug): # if(self.options.debug):
print("Plot init done.") # print("Plot init done.")
return self.returnAllLines() return self.returnAllLines()
...@@ -341,7 +418,10 @@ class LiveGui(GuiBase): ...@@ -341,7 +418,10 @@ class LiveGui(GuiBase):
data = self.__connectionBuffer[port].popleft() data = self.__connectionBuffer[port].popleft()
except IndexError: except IndexError:
pass pass
except KeyError:
pass
else: else:
# print(data)
#re-add first sample (to head of dequeue) #re-add first sample (to head of dequeue)
self.__connectionBuffer[port].appendleft(data) self.__connectionBuffer[port].appendleft(data)
self.__initSampletimeTimestamp = float(data['time']) self.__initSampletimeTimestamp = float(data['time'])
......
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import os
import sys
import select import select
from .input_base import InputBase from .input_base import InputBase
from ...utils.utilty import Utility
THREAD_STOPFLAG_WAIT = 0.000001 # in seconds THREAD_STOPFLAG_WAIT = 0.000001 # in seconds
...@@ -10,39 +13,39 @@ class FileInput(InputBase): ...@@ -10,39 +13,39 @@ class FileInput(InputBase):
def __init__(self, options, infoRegistry): def __init__(self, options, infoRegistry):
self.options = options self.options = options
self.infoRegistry = infoRegistry self.infoRegistry = infoRegistry
self.__logFileHandler = None
def startupCheck(self): def startupCheck(self):
pass if not os.access(self.options.logFilePath, os.R_OK):
Utility.eprint("Error: Input file " + self.options.logFilePath + " not readable. Exiting...")
sys.exit(1)
def startUp(self): def startUp(self):
pass
def tearDown(self):
pass
def retrieveNewSamples(self):
pass
try: try:
self.__logFileHandler = open(self.options.logFilePath, 'r') self.__logFileHandler = open(self.options.logFilePath, 'r')
except IOError: except IOError:
print ("Error: while parsing " + self.options.logFilePath) Utility.eprint("Error: Input file " + self.options.logFilePath + " not readable. Exiting...")
SystemExit sys.exit(1)
return
else: else:
inputready,outputready,exceptready = select.select([self.__logFileHandler.fileno()],[],[]) if(self.options.debug):
while(not self.__stopped.wait(THREAD_STOPFLAG_WAIT)): print("Openend file '" + self.options.logFilePath + "'")
if self.__stopped.isSet():
return
else:
for s in inputready:
if(s == self.__logFileHandler.fileno()):
for line in self.__logFileHandler:
if self.__stopped.isSet():
return
#ignore comments
if not line.startswith("#"):
line = line.strip()
self.__tmpBuffer.append(line,)
def tearDown(self):
self.__logFileHandler.close()
if(self.options.debug):
print("Closed file '" + self.options.logFilePath + "'")
def retrieveNewSamples(self):
inputready,outputready,exceptready = select.select([self.__logFileHandler.fileno()],[],[])
for s in inputready:
if(s == self.__logFileHandler.fileno()):
tmpBuffer = []
for line in self.__logFileHandler: