cnl_library.py 6.14 KB
Newer Older
1
2
3
4
5
6
7
8
9
#!/usr/bin/env python3
# -*- coding:utf-8 -*-

from io import StringIO

import json
import csv


Mario Hock's avatar
Mario Hock committed
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

def merge_lists(first, second):
    """
    Merges two lists alternately.

    E.g.:

    first = [1, 2]
    second = ["A", "B"]

    result = [1, "A", 2, "B"]
    """

    return [item for pair in zip(first, second) for item in pair]


Mario Hock's avatar
Mario Hock committed
26
27
28
29
## Exponential moving average
def calc_ema(values, alpha=0.2):
    ret = list()
    beta = 1 - alpha
Mario Hock's avatar
Mario Hock committed
30

Mario Hock's avatar
Mario Hock committed
31
32
33
34
35
36
37
38
39
40
41
42
43
44
    ## init
    it = iter(values)
    ema_value = float(next(it))
    ret.append(ema_value)

    ## loop
    for v in it:
        ema_value = alpha * float(v) + beta * ema_value
        ret.append(ema_value)

    return ret



45
46
47
def pretty_json(data):
    return json.dumps(data, sort_keys=True, indent=4)

Mario Hock's avatar
Mario Hock committed
48
49

## Helper functions for CNLParser -- but they could also be handy in other contexts.
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

def cnl_slice(file, start_delimiter, end_delimiter):

    ## Find beginning
    for line in file:
        if ( line.startswith(start_delimiter) ):
            break

    ## Skip comments and quit on end
    for line in file:
        if ( line.startswith(end_delimiter) ):
            return

        # skip empty or commented lines
        if ( not line or line[0] == "%" or line[0] == "#" ):
            continue

        yield line



Mario Hock's avatar
Mario Hock committed
71
72
73
74
75
76
77
78
79
80
def create_csv_index(csv_header):
    ## Create an index that maps csv_header names to tuple indices.
    csv_field_index = dict()
    i = 0
    for field in csv_header:
        csv_field_index[field] = i
        i += 1

    return csv_field_index

81
82
83
84
85
86
87
88
89
90
91
92
93
94
95


def read_header(f):
    str_io = StringIO()

    for line in cnl_slice(f, "%% Begin_Header", "%% End_Header"):
        str_io.write(line)

    str_io.seek(0)
    header = json.load( str_io )

    return header



Mario Hock's avatar
Mario Hock committed
96
97
98
class CNLParser:
    def __init__(self, filename):
        self.filename = filename
99

100
        with open( self.filename ) as in_file:
Mario Hock's avatar
Mario Hock committed
101
102
            ## Check file format version.
            assert( in_file.readline() == "%% CPUnetLOGv1\n" )
103

Mario Hock's avatar
Mario Hock committed
104
105
            ## Read JSON header.
            self.header = read_header(in_file)
106

Mario Hock's avatar
Mario Hock committed
107
108
109
110
            ## Read CSV "header"
            csv_reader = csv.reader( cnl_slice(in_file, "%% Begin_Body", "%% End_Body"), skipinitialspace=True )
            self.csv_header = next(csv_reader)
            self.csv_index = create_csv_index(self.csv_header)
111
112


Mario Hock's avatar
Mario Hock committed
113
114
115
    def get_csv_iterator(self, fields=None):
        """
        Returns an iterator to get the csv-values line by line.
Mario Hock's avatar
Mario Hock committed
116

Mario Hock's avatar
Mario Hock committed
117
118
119
        @param fields [list] Only the "columns" specified in |fields| are included in the returned list (in that order).
                      [None] All "columns" are included (order defined by |self.csv_header|.
        """
Mario Hock's avatar
Mario Hock committed
120
121
122
123
124

        indices = None

        ## Only return selected columns (if the |fields| option is set).
        if ( fields ):
125
            indices = self.get_csv_indices_of(fields)
Mario Hock's avatar
Mario Hock committed
126
127
128


        ## Read from file.
129
        with open( self.filename ) as in_file:
Mario Hock's avatar
Mario Hock committed
130
131
132
133
134
            ## Find start of the CSV part.
            csv_reader = csv.reader( cnl_slice(in_file, "%% Begin_Body", "%% End_Body"), skipinitialspace=True )
            csv_header = next(csv_reader)
            assert( csv_header == self.csv_header )

Mario Hock's avatar
Mario Hock committed
135
136
            ## TODO convert every field to float..?

137

Mario Hock's avatar
Mario Hock committed
138
139
140
            ## Yield line by line.
            for line in csv_reader:
                if ( not indices ):
141
142
                    #yield line
                    yield [ float( v ) for v in line ]
Mario Hock's avatar
Mario Hock committed
143
                else:
144
145
                    #yield [ line[ind] for ind in indices ]
                    yield [ float( line[ind] ) for ind in indices ]
Mario Hock's avatar
Mario Hock committed
146
147
148
149
150
151
152
153


    def get_csv_columns(self, fields=None):
        """
        Returns a dictionary holding the CSV values grouped into columns.

        Dict-keys correspond to |self.csv_header|, if |fields| is set only the specified columns are included.
        """
Mario Hock's avatar
Mario Hock committed
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180

        ## TODO should we really use "get_..." for an I/O and computation intensive function..?

        if ( fields ):
            field_names = fields
        else:
            field_names = self.csv_header

        num_cols = len(field_names)


        ## Create a list for each column.
        cols = [ list() for i in range(num_cols) ]

        ## Read all csv lines and put the values in the corresponding columns,
        for line in self.get_csv_iterator(fields):
            for i in range(num_cols):
                cols[i].append( line[i] )


        ## Create output dictionary.
        ret = dict()
        for i in range(num_cols):
            ret[ field_names[i] ] = cols[i]

        return ret

Mario Hock's avatar
Mario Hock committed
181

Mario Hock's avatar
Mario Hock committed
182
    ## Convenience functions ##
183

Mario Hock's avatar
Mario Hock committed
184
185
186
    def get_json_header(self):
        return self.header

Mario Hock's avatar
Mario Hock committed
187
188
    def print_json_header(self):
        print( json.dumps(self.header, sort_keys=True, indent=4) )
189

Mario Hock's avatar
Mario Hock committed
190
191
192
193
194
195
196
197
    def get_csv_index_of(self, field_name):
        return self.csv_index[field_name]

    def get_csv_indices_of(self, field_names):
        return [ self.get_csv_index_of(name) for name in field_names ]

    # Specific getters:

198
199
200
    def get_general_header(self):
        return self.header["General"]

Mario Hock's avatar
Mario Hock committed
201
202
203
    def get_type(self):
        return self.header["General"]["Type"]

Mario Hock's avatar
Mario Hock committed
204
205
206
    def get_comment(self):
        return self.header["General"]["Comment"]

Mario Hock's avatar
Mario Hock committed
207
208
209
210
211
212
    def get_cpus(self):
        return self.header["ClassDefinitions"]["CPU"]["Siblings"]

    def get_nics(self):
        return self.header["ClassDefinitions"]["NIC"]["Siblings"]

Mario Hock's avatar
Mario Hock committed
213
214
215
    def get_sysinfo(self):
        return self.header["General"]["SystemInfo"]

216
    def get_hostname(self):
217
218
219
220
        try:
            return self.get_sysinfo()["hostname"]
        except KeyError:
            return "(unknown)"
221

Mario Hock's avatar
Mario Hock committed
222
223
224
    def get_environment(self):
        return self.header["General"]["Environment"]

Mario Hock's avatar
Mario Hock committed
225
226
227
228
229
230


## MAIN ##
if __name__ == "__main__":

    ### DEMO:
231
    import sys
Mario Hock's avatar
Mario Hock committed
232
233
234

    filename = sys.argv[1]
    print( filename )
Mario Hock's avatar
Mario Hock committed
235

Mario Hock's avatar
Mario Hock committed
236
237
    ## * Parse input file. *
    cnl_file = CNLParser(filename)
Mario Hock's avatar
Mario Hock committed
238

239

Mario Hock's avatar
Mario Hock committed
240
241
242
    ## Display header informations.
    print( cnl_file.get_type() )
    print( json.dumps(cnl_file.get_json_header(), sort_keys=True, indent=4) )
243

Mario Hock's avatar
Mario Hock committed
244
245
    print( "CPUs: " + str(cnl_file.get_cpus()) )
    print( "NICs: " + str(cnl_file.get_nics()) )
246

Mario Hock's avatar
Mario Hock committed
247
248
249
250
    ## Display some csv/data fields.
    names = None
    names = ["eth0.send", "eth0.receive"]
    print( names )
251

Mario Hock's avatar
Mario Hock committed
252
253
    for x in cnl_file.get_csv_iterator(names):
        print( ", ".join(x) )