Plot Top #1, #2 CPUs as area chart + ext. Legends

Note: The legend placing code is still a bit rough,
though, a major breakthrough.

cnl_library.py

@@ -214,7 +214,10 @@ class CNLParser:
         return self.header["General"]["SystemInfo"]
 
     def get_hostname(self):
+        try:
             return self.get_sysinfo()["hostname"]
+        except KeyError:
+            return "(unknown)"
 
     def get_environment(self):
         return self.header["General"]["Environment"]

plot_cpu.py

+# -*- coding:utf-8 -*-
+
+from collections import defaultdict
+from operator import itemgetter
+import numpy
+import copy
+
+from cnl_library import merge_lists
+
+
+CPU_COLORS = defaultdict(lambda : "grey")
+CPU_COLORS["usr"] = "green"
+CPU_COLORS["system"] = "yellow"
+CPU_COLORS["irq"] = "cyan"
+CPU_COLORS["softirq"] = "magenta"
+CPU_COLORS["util"] = "red"      ## XXX
+CPU_COLORS["idle"] = "blue"     ## XXX
+
+
+def _create_cpu_cols_by_util(cnl_file):
+    """
+    The actual creation of the "virtual top-cpus" is outsourced to this function:
+
+    For each sample,
+
+    first sort the CPUs by CPU utilization;
+
+    then, for the top-n CPU (n from 1 to num_cpus),
+    append the values of the cpu-utilization fields (usr, system, ...).
+
+    Also, stores which CPU was the top-n CPU in the respective sample.
+    """
+    cpus = cnl_file.get_cpus()
+    cpu_fields = cnl_file.get_json_header()["ClassDefinitions"]["CPU"]["Fields"]
+    num_samples = len(cnl_file.cols["begin"])
+
+    # init return list (of dicts of lists)
+    top_cpus = list()
+    for cpu in cpus:
+        elem = dict()
+        elem["name"] = list()
+        for field in cpu_fields:
+            elem[field] = list()
+
+        top_cpus.append(elem)
+
+
+    # Fill return list.
+    for i in range(num_samples):
+        cpu_utils = [ (cpu, cnl_file.cols[cpu + ".util"][i]) for cpu in cpus ]
+        cpus_sorted_by_util = sorted( cpu_utils, key=itemgetter(1), reverse=True)
+
+        for top_cpu, cur_cpu in zip(top_cpus, cpus_sorted_by_util):
+            cpu_name = cur_cpu[0]
+
+            top_cpu["name"].append(cpu_name)
+            for field in cpu_fields:
+                top_cpu[field].append( cnl_file.cols[cpu_name + "." + field][i] )
+
+    return top_cpus
+
+
+
+def plot_area_chart(ax, cnl_file, cols, legend_outside=True, legend_title=None):
+    """
+    Plots an area chart of the CPU utilization (usr, sys, ...).
+    """
+
+    cpu_fields = copy.copy( cnl_file.get_json_header()["ClassDefinitions"]["CPU"]["Fields"] )
+    cpu_fields.remove("idle")
+    cpu_fields.remove("util")
+
+    # Axes
+    ax.set_ylim(0,100)
+    #ax.set_ylabel('CPU ' + "/".join(cpu_fields) + ' (%)')
+    ax.set_ylabel('CPU util (%)')
+
+    # Plot
+    y_offsets = numpy.array([0.0] * len(cnl_file.cols["begin"]))
+    z = 0
+    for field in cpu_fields:
+        #values = cols[field]
+        v = cols[field] + y_offsets
+        values = merge_lists( v, v )
+
+        # as bar chart -- slooooow!!
+        #ax.bar(cnl_file.cols["begin"], values, cnl_file.cols["duration"], bottom=y_offsets,
+               #color=CPU_COLORS[field], label=field)
+
+        # draw with lines (okay, but not filled..)
+        #ax.plot(cnl_file.x_values, values,
+               #color=CPU_COLORS[field], label=field, zorder=z)
+
+
+        # fill (seems to be the best option)
+        ax.fill(cnl_file.x_values, values,
+               color=CPU_COLORS[field], label=field, zorder=z)
+
+
+        # NOTE: ax.stackplot could actually be the right choice here..
+
+
+        #y_offsets += values
+        y_offsets += cols[field]
+
+        z -= 1
+
+
+    # Legend
+    if ( legend_outside ):
+        l = ax.legend(bbox_to_anchor=(1.2, 1.02),fancybox=True, shadow=True, title=legend_title)
+    else:
+        l = ax.legend(loc=0)
+
+    l.draggable(True)
+
+
+def plot_top_cpus(cnl_file, axes, indices=[0]):
+    """
+    This function creates "virtual top-cpus" and plots the utilization fields (usr, system, ...)
+
+    The CPU utilization samples are rearranged as if always the first CPU had the highest load,
+    the seconds CPU the second highest load, etc.
+
+    This means, usually each "virtual top-cpus" contains samples from many/all real CPUs.
+    Thus, origin of the sample is stored as an extra field and can be plotted as well.
+      (Therefore each CPU needs to have an assigned color).
+
+    ***
+
+    Plots the "virtual top-cpus" according to |indices| in the specified |axes|.
+
+    """
+
+    top_cpus = _create_cpu_cols_by_util(cnl_file)
+
+    for ax, i in zip(axes, indices):
+        label = "Top #{} CPU".format(i+1)
+        cols = top_cpus[i]
+        plot_area_chart(ax, cnl_file, cols, True, label)
+

simple_plotter.py

@@ -5,6 +5,8 @@ import sys
 import matplotlib
 
 from cnl_library import CNLParser, calc_ema, merge_lists, pretty_json
+from plot_cpu import plot_top_cpus
+
 
 ## matplotlib.use('QT4Agg')  # override matplotlibrc
 import matplotlib.pyplot as plt
@@ -38,6 +40,7 @@ def parse_cnl_file(filename):
             net_cols.append( nic_name + nic_field )
 
     cpu_cols = [ cpu_name + ".util" for cpu_name in cnl_file.get_cpus() ]
+    #cpu_cols = [ cpu_name + ".irq" for cpu_name in cnl_file.get_cpus() ]   ## XXX
 
     cols = cnl_file.get_csv_columns()
     #x_values = cols["end"]
@@ -71,25 +74,44 @@ def plot(ax, x_values, cols, active_cols, **kwargs):
             #ax.plot(x_values , calc_ema(cols[col_name], 0.2), label=col_name+" (ema)")
 
 
-def plot_net(ax, cnl_file):
+def plot_net(ax, cnl_file, legend_outside=True):
     ax.set_ylim(0,10**10)
     ax.set_ylabel('Throughput (Bit/s)')
 
     plot(ax, cnl_file.x_values, cnl_file.cols, cnl_file.net_col_names)
 
-    ax.legend(loc=0)
-    #ax.legend(loc=8)
+    # Legend
+    if ( legend_outside ):
+        offset = matplotlib.transforms.ScaledTranslation(0, -20, matplotlib.transforms.IdentityTransform())
+        trans = ax.transAxes + offset
+
+        l = ax.legend( loc='upper left', bbox_to_anchor=(0, 0), ncol=int(len(cnl_file.net_col_names)/2),
+                      bbox_transform = trans,
+                      fancybox=False, shadow=False)
+    else:
+        l = ax.legend(loc=0)
 
 
-def plot_cpu(ax, cnl_file):
+def plot_cpu(ax, cnl_file, legend_outside=True):
     ax.set_ylim(0,100)
     ax.set_ylabel('CPU util (%)')
 
     plot(ax, cnl_file.x_values, cnl_file.cols, cnl_file.cpu_col_names)
 
-    ax.legend(loc=0)
-    #ax.legend(loc=1)
+    # Legend
+    if ( legend_outside ):
+        offset = matplotlib.transforms.ScaledTranslation(0, -20, matplotlib.transforms.IdentityTransform())
+        trans = ax.transAxes + offset
+
+        l = ax.legend( loc='upper left', bbox_to_anchor=(0, 0), ncol=int(len(cnl_file.cpu_col_names)/2),
+                      bbox_transform = trans,
+                      fancybox=False, shadow=False)
+    else:
+        l = ax.legend(loc=0)
+
+    ax.set_label("Testlabel")
 
+    l.draggable(True)
 
 
 
@@ -102,6 +124,8 @@ if __name__ == "__main__":
     fig = plt.figure()
     fig.canvas.set_window_title('CPUnetPlot')
 
+    num_cols = 2
+
     old_ax_net = None
     old_ax_cpu = None
     for i in range(1, num_files+1):
@@ -121,8 +145,9 @@ if __name__ == "__main__":
 
 
         ## Prepare subplots
-        ax_net = fig.add_subplot(2, num_files, i, sharex=old_ax_net, sharey=old_ax_net)
-        ax_cpu = fig.add_subplot(2, num_files, i+num_files, sharex=ax_net, sharey=old_ax_cpu)
+        fig.subplots_adjust(left=0.1, wspace=0.2, right=0.9, top=0.92, hspace=0.4, bottom=0.12)
+        ax_net = fig.add_subplot(2, num_cols, i, sharex=old_ax_net, sharey=old_ax_net)
+        ax_cpu = fig.add_subplot(2, num_cols, i+num_cols, sharex=ax_net, sharey=old_ax_cpu)
         #ax_net = fig.add_subplot(111)  ## twin axis
         #ax_cpu = ax_net.twinx()        ## twin axis
 
@@ -142,8 +167,17 @@ if __name__ == "__main__":
         old_ax_cpu = ax_cpu
 
 
+    ## If we have only one input file, plot CPU area charts.
+    if ( num_files == 1 ):
+        ax1 = fig.add_subplot(2, num_cols, 2, sharex=old_ax_net, sharey=old_ax_cpu)
+        ax2 = fig.add_subplot(2, num_cols, 4, sharex=ax_net, sharey=old_ax_cpu)
+
+        plot_top_cpus( cnl_file, (ax1, ax2), (0,1) )
+
+
+
     ## maximize window
-    if ( num_files > 1 ):
+    if ( num_files > 1 or True ):  ## XXX always maximize?
         try:
             figManager = plt.get_current_fig_manager()
             figManager.window.showMaximized()