Skip to content

GitLab

T
TCPlivePLOT
Commit f5f05487 authored by Michael König's avatar Michael König
Browse files
Options

refactoring++ & better auto-scaling

parent 34a7f0e9
Hide whitespace changes
Inline Side-by-side
Showing with 325 additions and 219 deletions
.gitignore
View file @ f5f05487
......@@ -5,3 +5,4 @@ __pycache__/
.old
*.egg-info
*.pyc
todo.txt
README.txt → README.md
View file @ f5f05487
TCPlivePLOT is based on Python3.
TCPlivePLOT
================================================================================
Realtime-visualization of TCP flows logged by TCPlog.
TCPlivePLOT is based on Python3 and tested on GNU/Linux 4.{1-7}.
For TCPlog see https://git.scc.kit.edu/CPUnetLOG/TCPlog/
Tested on GNU/Linux 4.{1-6}
Requirements TCPlivePLOT:
================================================================================
--------------------------------------------------------------------------------
* python3
* python-matplotlib
* matplotlib
* numpy
* matplotlib-interactive backend
recommended:
* recommended backends:
* Qt4Agg or
* Qt5Agg
Running TCPplot:
================================================================================
Running TCPlivePLOT:
--------------------------------------------------------------------------------
* ./tcpliveplot.py OR
* python3 -m TCPlivePLOT OR
* tcpliveplot (after installation)
Installation of TCPplot:
================================================================================
Installation of TCPlivePLOT:
--------------------------------------------------------------------------------
* via pip3:
* sudo pip3 install . # system-wide installation
* pip3 install --user . # local installation
* via setup.py:
* python3 setup.py install # system-wide installation
* python3 setup.py install --user # local installation
setup.py
View file @ f5f05487
......@@ -3,9 +3,7 @@
from setuptools import setup
with open("VERSION.txt", "rb") as f:
version = f.read().decode("utf-8")
version = 0.1.0
desc = "Realtime-visualization of TCP flows logged by TCPlog."
setup(
......@@ -41,7 +39,7 @@ setup(
#
],
extras_require = {
'Logging': ["tcplog>=0.2"]
'Logging': ["tcplog"]
},
keywords = ['tcp', 'flow', 'plot', 'visualize', 'graph', 'live', 'analyze', 'network', 'traffic'],
classifiers = [
......
tcpliveplot.py
View file @ f5f05487
......@@ -3,7 +3,7 @@
"""Convenience wrapper for running TCPlivePLOT directly from source tree."""
from TCPlivePLOT.main import main
from tcpliveplot.main import main
if __name__ == '__main__':
main()
tcpliveplot/backends/gui/live.py
View file @ f5f05487
# -*- coding: utf-8 -*-
# TODO:
# * dynamic flow detection
# * flowIdentifer i/o "port"
# * fix socket-input
# * dynamic value detection
import matplotlib
matplotlib.use('Qt5Agg')
matplotlib.use('Qt4Agg')
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from matplotlib.widgets import Button
from matplotlib.widgets import Button, RadioButtons
import numpy as np
import math
import sys
import time
import threading
from collections import deque
from .gui_base import GuiBase
VALUES_TO_PLOT = ['cwnd', 'sst', 'rtt', 'bw'] # (only values for Y-axis)
VALUES_TO_PLOT = ['cwnd', 'sst', 'rtt', 'smoothedThroughput'] # (only values for Y-axis)
VALUES_TO_PROCESS = ['time'] + VALUES_TO_PLOT #helper to init all data structures
# Strings for UI-elements
......@@ -25,14 +34,19 @@ QUIT = "Quit"
CLEAR_GAP = 0.2 # gap in s
INFINITY_THRESHOLD = 1e8
class LiveGui(GuiBase):
def __init__(self, options, infoRegistry):
self.options = options
self.infoRegistry = infoRegistry
self.__stopped = threading.Event()
self.timestampOfLastGuiRefresh = 0
def setConnectionBuffer(self, connectionBuffer):
self.__connectionBuffer = connectionBuffer
def tearDown(self):
plt.close()
sys.exit(0)
pass
def startUp(self):
......@@ -53,6 +67,23 @@ class LiveGui(GuiBase):
self.__plotLines[port][lineID].set_visible(self.__plotLineConfigs[port][lineID])
self.drawPlotLegend()
def updateValueVisibility(self, label):
for port in self.options.filterPorts:
for i in range(1, len(VALUES_TO_PLOT)+1):
self.__plotLineConfigs[port][VALUES_TO_PLOT[(i-1)]] = False
self.__plotLines[port][(VALUES_TO_PLOT[(i-1)])].set_visible(self.__plotLineConfigs[port][(VALUES_TO_PLOT[(i-1)])])
if label == 'cwnd':
self.toggleVisibility(VALUES_TO_PLOT[0])
elif label == 'sst':
self.toggleVisibility(VALUES_TO_PLOT[1])
elif label == 'rtt':
self.toggleVisibility(VALUES_TO_PLOT[2])
elif label == 'bw':
self.toggleVisibility(VALUES_TO_PLOT[3])
else:
pass
def drawPlotLegend(self):
"""(Re)draws legend with visible lines."""
labelObjs = []
......@@ -91,7 +122,6 @@ class LiveGui(GuiBase):
def stopPlotting(self, event):
"""Callback function to stop plotting and the programm."""
self.__tmpTimestamp = time.perf_counter()
plt.close()
self.tearDown()
def plotGraph(self):
......@@ -100,7 +130,6 @@ class LiveGui(GuiBase):
self.__minVal = 9999999999
self.__maxVal = 0
fig = plt.figure(FIGURE_TITLE)
fig.canvas.mpl_connect('key_press_event', self.plotKeyPressCallback)
self.__ax = plt.axes()
......@@ -110,13 +139,19 @@ class LiveGui(GuiBase):
self.__plotLines = {}
self.__plotValues = {}
self.__plotValuesMin = {}
self.__plotValuesMax = {}
self.__plotLineConfigs = {}
for port in self.options.filterPorts:
self.__plotLines[port] = {}
self.__plotValues[port] = {}
self.__plotValuesMin[port] = {}
self.__plotValuesMax[port] = {}
self.__plotLineConfigs[port] = {}
self.__plotLineConfigs[port]['lastTimestamp'] = 0
for val in VALUES_TO_PROCESS:
self.__plotValuesMin[port][val] = math.inf
self.__plotValuesMax[port][val] = -math.inf
self.__plotValues[port][val] = deque(maxlen=(int(self.options.xDelta / self.options.plotResolution * 10)))
index = 1
for val in VALUES_TO_PLOT:
......@@ -137,6 +172,15 @@ class LiveGui(GuiBase):
quitButton = Button(quitAx, QUIT)
quitButton.on_clicked(self.stopPlotting)
# valueCheckboxesAx = plt.axes([0.05, 0.4, 0.1, 0.15])
# valueCheckboxes = CheckButtons(valueCheckboxesAx, ('cwnd', 'sst', 'rtt', 'bw'), (True, False, False, False))
# valueCheckboxes.on_clicked(self.updateValueVisibility)
# valueRadiobuttonsAx = plt.axes([0.020, 0.025, 0.075, 0.15])
# valueRadiobuttons = RadioButtons(valueRadiobuttonsAx, ('cwnd', 'sst', 'rtt', 'bw'))
# valueRadiobuttons.on_clicked(self.updateValueVisibility)
if(self.options.preloadBuffer > 0):
self.__preloading = True
else:
......@@ -154,12 +198,17 @@ class LiveGui(GuiBase):
self.__apsFixFactor = 1
# call update-routine
animation.FuncAnimation(fig, self.plotGraphUpdate, init_func=self.plotInit,
frames=self.options.drawFps, interval=self.options.drawIntervall, blit=self.options.blitting, repeat=True)
if self.__stopped.isSet():
return
else:
plt.show()
print("foo")
# self.plotInit()
# self.__plotLine = self.plotGraphUpdate(0)
animation.FuncAnimation(fig, self.plotGraphUpdate, init_func=self.plotInit, frames=self.options.drawFps, interval=self.options.drawIntervall, blit=self.options.blitting, repeat=True)
# if self.__stopped.isSet():
# return
# else:
# plt.ioff()
# plt.draw()
plt.show()
# print("bar")
def returnAllLines(self):
"""Macro to return all lines as they are."""
......@@ -187,7 +236,7 @@ class LiveGui(GuiBase):
return self.returnAllLines()
# fill playback-buffer
if(self.__preloading):
if(False and self.__preloading):
bufferLength = -1
for port in self.options.filterPorts:
bufferLength = max(bufferLength, len(self.__connectionBuffer[port]))
......@@ -219,8 +268,8 @@ class LiveGui(GuiBase):
newXmin = newXmax - self.options.xDelta
self.__ax.set_xlim(newXmin, newXmax)
maxYval = 0
minYval = 9999999
maxYval = -math.inf
minYval = math.inf
connectionsData = {}
# skip this cycle, plot resolution not yet reached
......@@ -237,7 +286,7 @@ class LiveGui(GuiBase):
whileRun = False
pass
else:
lineTime = self.__initRealtimeTimestamp + (data['time'] - self.__initSampletimeTimestamp)
lineTime = self.__initRealtimeTimestamp + (float(data['time']) - self.__initSampletimeTimestamp)
# time in past
if(lineTime < newXmin):
continue
......@@ -254,8 +303,13 @@ class LiveGui(GuiBase):
connectionsData[port].append(nanSample)
infinityReached = False
for val in VALUES_TO_PLOT:
if(data[val] > INFINITY_THRESHOLD):
try:
convertedValue = float(data[val])
except ValueError:
data[val] = np.nan
else:
if(convertedValue > INFINITY_THRESHOLD):
data[val] = np.nan
# nanSample = self.returnNanSample(lineTime)
# connectionsData[port].append(nanSample)
# infinityReached = True
......@@ -279,58 +333,81 @@ class LiveGui(GuiBase):
# copy raw-value into corresponding lists
for connection in connectionsData:
while(len(connectionsData[connection]) > 0):
data = connectionsData[connection].popleft()
lineTime = self.__initRealtimeTimestamp + (data['time'] - self.__initSampletimeTimestamp)
self.__plotLineConfigs[connection]['lastTimestamp'] = data['time']
lineTime = self.__initRealtimeTimestamp + (float(data['time']) - self.__initSampletimeTimestamp)
self.__plotLineConfigs[connection]['lastTimestamp'] = float(data['time'])
for val in VALUES_TO_PROCESS:
if(val == 'time'):
self.__plotValues[connection][val].append(lineTime)
else:
self.__plotValues[connection][val].append(data[val])
try:
currentVal = float(data[val])
except ValueError:
pass
else:
self.__plotValues[connection][val].append(currentVal)
# update axis (xy-tuple) with data from lists
for val in VALUES_TO_PLOT:
self.__plotLines[connection][val].set_data(self.__plotValues[connection]['time'], self.__plotValues[connection][val])
x, y = self.__plotValues[connection]['time'], self.__plotValues[connection][val]
self.__plotLines[connection][val].set_data(x, y)
for port in self.options.filterPorts:
for val in VALUES_TO_PLOT:
if(self.__plotLineConfigs[port][val]):
if(len(self.__plotValues[port][val]) > 0):
maxYval = max(maxYval, max(self.__plotValues[port][val]))
minYval = min(minYval, min(self.__plotValues[port][val]))
self.__lastDrawTimestamp = time.perf_counter()
if(maxYval > 0):
newYmin = minYval - 50
newYmin = max(newYmin, 0)
newYmax = maxYval * 2
self.__ax.set_ylim(newYmin, newYmax)
# y-scaling
lines = self.__ax.get_lines()
bot,top = np.inf, -np.inf
for line in lines:
if(line.get_visible()):
new_bot, new_top = self.determineNewYvalues(line)
if(new_bot != new_top):
if(new_bot < bot):
bot = new_bot
if(new_top > top):
top = new_top
if(bot != np.inf and top != -np.inf):
self.__ax.set_ylim(bot, top)
else:
# intial y-scale
self.__ax.set_ylim(0, 500)
self.__lastDrawTimestamp = time.perf_counter()
return self.returnAllLines()
def determineNewYvalues(self, line, margin=0.25):
xLine = line.get_xdata()
yLine = line.get_ydata()
xData = np.array(xLine)
yData = np.array(yLine)
low,high = self.__ax.get_xlim()
yVisibleMask = yData[((xData>low) & (xData<high))]
if(len(yVisibleMask) > 0):
height = np.max(yVisibleMask) - np.min(yVisibleMask)
bot = np.min(yVisibleMask) - margin * height
top = np.max(yVisibleMask) + margin * height
return bot,top
else:
return 0,0
def plotInit(self):
"""Helper to initialize plot."""
for port in self.options.filterPorts:
for val in VALUES_TO_PLOT:
self.__plotLines[port][val].set_data([], [])
# hide "invisible" lines
if(val not in self.options.linesToShow):
self.__plotLineConfigs[port][val] = False
self.__plotLines[port][val].set_visible(False)
newXmin = 0
newXmax = newXmin + self.options.xDelta
self.__ax.set_xlim(newXmin, newXmax)
if(self.options.debug):
print("Plot init done.")
# if(self.options.debug):
# print("Plot init done.")
return self.returnAllLines()
......@@ -341,7 +418,10 @@ class LiveGui(GuiBase):
data = self.__connectionBuffer[port].popleft()
except IndexError:
pass
except KeyError:
pass
else:
# print(data)
#re-add first sample (to head of dequeue)
self.__connectionBuffer[port].appendleft(data)
self.__initSampletimeTimestamp = float(data['time'])
......
tcpliveplot/backends/input/file.py
View file @ f5f05487
# -*- coding: utf-8 -*-
import os
import sys
import select
from .input_base import InputBase
from ...utils.utilty import Utility
THREAD_STOPFLAG_WAIT = 0.000001 # in seconds
......@@ -10,39 +13,39 @@ class FileInput(InputBase):
def __init__(self, options, infoRegistry):
self.options = options
self.infoRegistry = infoRegistry
self.__logFileHandler = None
def startupCheck(self):
pass
if not os.access(self.options.logFilePath, os.R_OK):
Utility.eprint("Error: Input file " + self.options.logFilePath + " not readable. Exiting...")
sys.exit(1)
def startUp(self):
pass
def tearDown(self):
pass
def retrieveNewSamples(self):
pass
try:
self.__logFileHandler = open(self.options.logFilePath, 'r')
except IOError:
print ("Error: while parsing " + self.options.logFilePath)
SystemExit
return
Utility.eprint("Error: Input file " + self.options.logFilePath + " not readable. Exiting...")
sys.exit(1)
else:
inputready,outputready,exceptready = select.select([self.__logFileHandler.fileno()],[],[])
while(not self.__stopped.wait(THREAD_STOPFLAG_WAIT)):
if self.__stopped.isSet():
return
else:
for s in inputready:
if(s == self.__logFileHandler.fileno()):
for line in self.__logFileHandler:
if self.__stopped.isSet():
return
#ignore comments
if not line.startswith("#"):
line = line.strip()
self.__tmpBuffer.append(line,)
if(self.options.debug):
print("Openend file '" + self.options.logFilePath + "'")
def tearDown(self):
self.__logFileHandler.close()
if(self.options.debug):
print("Closed file '" + self.options.logFilePath + "'")
def retrieveNewSamples(self):
inputready,outputready,exceptready = select.select([self.__logFileHandler.fileno()],[],[])
for s in inputready:
if(s == self.__logFileHandler.fileno()):
tmpBuffer = []
for line in self.__logFileHandler:
#ignore comments
if not line.startswith("#"):
line = line.strip()
tmpBuffer.append(line,)
return tmpBuffer
tcpliveplot/backends/input/socket.py
View file @ f5f05487
......@@ -4,8 +4,6 @@ import time
import socket
from ipaddress import ip_address
import threading
from .input_base import InputBase
# to remove --> use optioms
......@@ -22,61 +20,64 @@ class SocketInput(InputBase):
self.options = options
self.infoRegistry = infoRegistry
ip, separator, port = self.options.logServer.rpartition(':')
if(':' not in self.options.logServer or port is ''):
self.logServerPort = DEFAULT_SOCKETSERVER_PORT
self.logServerIp = ip_address(socket.gethostbyname(self.options.logServer.strip("[]")))
else:
self.logServerPort = int(port)
self.logServerIp = ip_address(socket.gethostbyname(ip.strip("[]")))
self.dst = str(self.logServerIp) + ":" + str(self.logServerPort)
def startupCheck(self):
pass
def startUp(self):
pass
#TODO: eprint + debug..
try:
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
except socket.error:
print("Failed to create socket")
else:
print("Socket created.")
try:
self.socket.connect((str(self.logServerIp), self.logServerPort))
print("Successfully connected to " + self.dst + "")
except socket.error:
print("\
Error: Could not connect to " + self.dst + "\
Retrying in " + str(LOGSERVER_CONNECT_RETRY_TIME) + "s ...\
")
time.sleep(LOGSERVER_CONNECT_RETRY_TIME)
else:
pass
def tearDown(self):
self.socket.close()
pass
def retrieveNewSamples(self):
"""Reads data (in blocks) from a socket and adds the received data to an income buffer."""
self.__processSocketLogData = threading.Thread(target=self.processSocketLogData)
self.__processSocketLogData.start()
ip, separator, port = self.options.logServer.rpartition(':')
if(':' not in self.options.logServer or port is ''):
logServerPort = DEFAULT_SOCKETSERVER_PORT
logServerIp = ip_address(socket.gethostbyname(self.options.logServer.strip("[]")))
else:
logServerPort = int(port)
logServerIp = ip_address(socket.gethostbyname(ip.strip("[]")))
dst = str(logServerIp) + ":" + str(logServerPort)
"""
Reads data (in blocks) from a socket and adds the received data to an temporaray income buffer.
"""
try:
self.__socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
except socket.error:
print("Failed to create socket")
while(not self.__stopped.wait(0.00001)):
try:
self.__socket.connect((str(logServerIp), logServerPort))
print("Successfully connected to " + dst + "")
except socket.error:
print("Error: Could not connect to " + dst + " Retrying in " + str(LOGSERVER_CONNECT_RETRY_TIME) + "s ...")
time.sleep(LOGSERVER_CONNECT_RETRY_TIME)
else:
break
while(not self.__stopped.wait(0.00001)):
try:
data = self.__socket.recv(4096)
except socket.timeout:
print("Connection timeout.")
self.__socket.close()
return
except IOError:
print("Error: Could not retrieve data from " + dst)
self.__socket.close()
return
else:
if(len(data) == 0):
print("Connection closed by foreign host.")
self.__socket.close()
break;
self.__incomeBuffer.append(data)
data = self.socket.recv(4096)
except socket.timeout:
print("Connection timeout.")
self.socket.close()
return ""
except IOError:
print("Error: Could not retrieve data from " + self.dst)
self.socket.close()
return ""
else:
if(len(data) == 0):
print("Connection closed by foreign host.")
self.socket.close()
return data
self.tmpBuffer.append(data)
def processSocketLogData(self):
......@@ -94,11 +95,11 @@ class SocketInput(InputBase):
data = i.split(" ")
if(tmpBuffer != ""):
tmpBuffer += i
self.__tmpBuffer.append(tmpBuffer)
self.incomeBuffer.append(tmpBuffer)
tmpBuffer = ""
continue
if(len(data) < NUMBER_OF_VALUES):
tmpBuffer += i
else:
self.__tmpBuffer.append(i)
self.incomeBuffer.append(i)
tcpliveplot/main.py
View file @ f5f05487
......@@ -5,6 +5,7 @@ import argparse
import sys
import signal
from .utils.utilty import Utility
from .tcpliveplot import TcpLivePlot
from .info_registry import InfoRegistry
......@@ -13,6 +14,7 @@ from .info_registry import InfoRegistry
TCP_PLOT_VERSION = "0.2.0"
TCP_LOG_FORMAT_VERSION_MIN = "2"
DEFAULT_LOGFILE_PATH = "/tmp/tcplog.log"
DEFAULT_INPUT_BACKEND = "socket"
# default values
......@@ -72,20 +74,23 @@ def main():
guiBackend.startupCheck()
guiBackend.startUp()
try:
tcpLivePlot = TcpLivePlot(inputBackend, guiBackend, options, infoRegistry)
signal.signal(signal.SIGINT, tcpLivePlot.handleSignals)
signal.signal(signal.SIGTERM, tcpLivePlot.handleSignals)
tcpLivePlot.run()
# ^-- starts the main programme
except Exception as e:
# utility.eprint(str(e))
print(str(e))
raise e
finally: