csfile.py 11.6 KB
Newer Older
1
#!/usr/bin/env python
Daniel Armbruster's avatar
Daniel Armbruster committed
2
## @file csfile.py
Daniel Armbruster's avatar
Daniel Armbruster committed
3
4
# @brief  Provide a module to read, write and treat with a csback
# checksumfiles.
Daniel Armbruster's avatar
Daniel Armbruster committed
5
6
7
8
9
10
11
# 
# -----------------------------------------------------------------------------
# 
# $Id$
# @author Daniel Armbruster
# \date 15/09/2011
# 
Daniel Armbruster's avatar
Daniel Armbruster committed
12
13
# Purpose: Provide a module to read, write and treat with a csback
# checksumfiles.
Daniel Armbruster's avatar
Daniel Armbruster committed
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
#
# ----
# This file is part of csback.
#
# csback is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# csback is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with csback.  If not, see <http://www.gnu.org/licenses/>.
# ----
# 
# Copyright (c) 2011 by Daniel Armbruster
# 
# REVISIONS and CHANGES 
# 15/09/2011  V0.1    Daniel Armbruster
Daniel Armbruster's avatar
Daniel Armbruster committed
36
37
# 01/01/2012  V0.1.1  finished implementation
# 02/01/2012  V0.1.2  implemented debugMode
Daniel Armbruster's avatar
Daniel Armbruster committed
38
39
# 
# =============================================================================
40
""" CsFile module to handle checksum files. """
Daniel Armbruster's avatar
Daniel Armbruster committed
41

Daniel Armbruster's avatar
Daniel Armbruster committed
42
43
import os
import re
44
import sys
Daniel Armbruster's avatar
Daniel Armbruster committed
45
import hashlib
Daniel Armbruster's avatar
Daniel Armbruster committed
46
from datetime import datetime
Daniel Armbruster's avatar
Daniel Armbruster committed
47

48
__version__ = "V0.1"
Daniel Armbruster's avatar
Daniel Armbruster committed
49
__subversion__ = "$Id$"
50
51
__license__ = "GPLv2"
__author__ = "Daniel Armbruster"
Daniel Armbruster's avatar
Daniel Armbruster committed
52
53
54
55
56
__copyright__ = "Copyright (c) 2012 by Daniel Armbruster"

# -----------------------------------------------------------------------------
# global variables

57
chunkSize = 1024 * 128 # 128kB
Daniel Armbruster's avatar
Daniel Armbruster committed
58
debugMode = False
Daniel Armbruster's avatar
Daniel Armbruster committed
59

60
61
# -----------------------------------------------------------------------------
class CsFileError(Exception):
Daniel Armbruster's avatar
Daniel Armbruster committed
62
63
64
65
  """
  Exception class of csfile module.
  """
  def __init__(self, line, msg):
66
    self.msg = msg
67
    self.line = line
68
69

  def display(self):
Daniel Armbruster's avatar
Daniel Armbruster committed
70
71
72
73
74
75
76
77
78
    sys.stderr.write("csfile (ERROR): " + str(self.msg) + "\n")
    sys.stderr.write("triggered in line: " + str(self.line) + "\n")

class CsReport(CsFileError):
  """
  Auxiliary class for debugging.
  """
  def display(self):
    sys.stderr.write("csfile (REPORT): " + str(self.msg) + "\n")
79
    sys.stderr.write("triggered in line: " + str(self.line) + "\n")
80
81
82

# -----------------------------------------------------------------------------
class CsFile:
83
84
85
  """
  Provides an interface to handle a csback checksum file.
  """
Daniel Armbruster's avatar
Daniel Armbruster committed
86
87
88
  def __init__(self, filepath, hashfunc='sha256'):
    self.__filepath = filepath
    self.__filename = ".cs"
89
90
    self.__cslines = []
    self.__hashfunc = hashfunc
91
    self.createFile()
Daniel Armbruster's avatar
Daniel Armbruster committed
92
93
94
95
96
97

  def createFile(self):
    path = self.__filepath+os.sep+self.__filename
    if not os.path.isfile(path):
      try:
        csfile = open(path, 'w')
98
      except IOError as err:
99
        raise CsFileError(99, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
Daniel Armbruster's avatar
Daniel Armbruster committed
100
          +err.filename)
101
      else:
Daniel Armbruster's avatar
Daniel Armbruster committed
102
        csfile.close()
103
104

  def read(self):
105
106
107
    """
    Read a checksum file.
    """
Daniel Armbruster's avatar
Daniel Armbruster committed
108
    path = self.__filepath+os.sep+self.__filename
109
    try:
Daniel Armbruster's avatar
Daniel Armbruster committed
110
      if debugMode:
111
        CsReport(111, "Start reading checksumfile: "+path).display()
Daniel Armbruster's avatar
Daniel Armbruster committed
112
      csfile = open(path)
113
114
      self.__cslines = [CsLine(line.split()) for line in csfile \
      if len(line.rstrip()) and line[0] != '#'] 
115
116
117
    except IOError as err:
      # Maybe better to create the file here and keep on going.
      # Will be managed later during further dev.
118
      raise CsFileError(118, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
Daniel Armbruster's avatar
Daniel Armbruster committed
119
        +err.filename)
120
121
    else:
      csfile.close()
Daniel Armbruster's avatar
Daniel Armbruster committed
122
      if debugMode:
123
        CsReport(123, "Finished reading checksumfile: "+path).display()
124
125
126
127
128
129
130

  def write(self):
    """
    Write the entire checksumfile.
    """
    path = self.__filepath+os.sep+self.__filename
    try:
Daniel Armbruster's avatar
Daniel Armbruster committed
131
      if debugMode:
132
        CsReport(132, "Start writing checksumfile: "+path).display()
133
134
      csfile = open(path, 'w')
      for csline in self.__cslines:
Daniel Armbruster's avatar
Daniel Armbruster committed
135
        if debugMode:
136
          CsReport(136, "Writing line: "+ str(csline)).display()
137
138
139
        if isinstance(csline, CsLine):
          csfile.write(str(csline) + '\n')
        else:
140
          raise CsFileError(140, "Argument must be of type CsLine.")
141
    except IOError as err:
142
      raise CsFileError(142, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
Daniel Armbruster's avatar
Daniel Armbruster committed
143
        +err.filename)
144
    else:
145
      csfile.close()
Daniel Armbruster's avatar
Daniel Armbruster committed
146
      if debugMode:
147
        CsReport(147, "Finished writing checksumfile: "+path)
Daniel Armbruster's avatar
Daniel Armbruster committed
148

Daniel Armbruster's avatar
Daniel Armbruster committed
149
  def append(self, cslines):
150
151
152
    """
    Append checksum lines to the checksum file.
    """
Daniel Armbruster's avatar
Daniel Armbruster committed
153
    path = self.__filepath+os.sep+self.__filename
Daniel Armbruster's avatar
Daniel Armbruster committed
154
    if 0 == len(cslines) and debugMode:
155
      CsReport(155, "Empty list passed. Nothing to append.").display()
156
    else:
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
      try:
        if debugMode:
          CsReport(159, "Start appending to checksumfile: "+path).display()
        csfile = open(path, 'a')
        for csline in cslines:
          if debugMode:
            CsReport(163, "Writing line: "+str(csline)).display()
          if isinstance(csline, CsLine):
            csfile.write(str(csline) + '\n')
          else:
            raise CsFileError(167, "Argument must be of type CsLine.")
      except IOError as err:
        raise CsFileError(169, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
          +err.filename)
      else:
        if debugMode:
          CsReport(173, "Finished appending to checksumfile: "+path).display()
        csfile.close()
Daniel Armbruster's avatar
Daniel Armbruster committed
175
176
      
  def update(self, regexes=[]):
177
178
179
180
181
    """
    Update a checksum file. Also includes appending of not registered files to
    checksum file in current directory.
    """
    if not isinstance(regexes, list):
182
      raise CsFileError(182, "Pass regular expressions in a list.")
Daniel Armbruster's avatar
Daniel Armbruster committed
183
    if debugMode:
184
      CsReport(184, "Updating checksumfile ...").display()
Daniel Armbruster's avatar
Daniel Armbruster committed
185
    # fetch cslines in current csfile
186
    self.read()
Daniel Armbruster's avatar
Daniel Armbruster committed
187
    if debugMode:
188
189
      CsReport(188, "Fetching files not registered yet.").display()
    registeredFiles = set(csline.path for csline in self.__cslines)
Daniel Armbruster's avatar
Daniel Armbruster committed
190
191
192
193
194
195
196
197
198
199
200
    # fetch files (pathes)
    newFiles = os.listdir(self.__filepath)
    newFiles = set(os.path.join(self.__filepath, file) for file in newFiles \
      if os.path.isfile(os.path.join(self.__filepath, file)))
    # exclude files matching regexes
    regexes.append(os.path.join(self.__filepath,self.__filename))
    for regex in regexes:
      matching = set(file for file in newFiles \
        if None != re.match(regex, file))
      newFiles -= matching
    regexes.remove(os.path.join(self.__filepath,self.__filename))
Daniel Armbruster's avatar
Daniel Armbruster committed
201
    # exclude registered files
202
    newFiles -= registeredFiles
Daniel Armbruster's avatar
Daniel Armbruster committed
203
    # generate cslines of newFiles
204
205
206
207
208
209
210
    cslines = []
    for file in newFiles:
      csline = CsLine(file, self.__hashfunc)
      csline.generate(chunkSize)
      cslines.append(csline)
      print(csline)
    self.append(cslines)
Daniel Armbruster's avatar
Daniel Armbruster committed
211

212
213
214
215
216
217
218
  def check(self, srcDir=''):
    """
    Check a checksum file which means:
    1. read checksum file
    2, calculate checksum of file which is located in srcDir and check results
    3. write the result to the checksum file
    """
Daniel Armbruster's avatar
Daniel Armbruster committed
219
    if debugMode:
Daniel Armbruster's avatar
Daniel Armbruster committed
220
      CsReport(220, "Start checking checksums.").display()
221
    self.read()
Daniel Armbruster's avatar
Daniel Armbruster committed
222
    if 0 == len(self.__cslines) and debugMode:
Daniel Armbruster's avatar
Daniel Armbruster committed
223
      CsReport(223, "CSFILE does not contain any lines.").display()
224
225
    for csline in self.__cslines:
      if not len(srcDir):
Daniel Armbruster's avatar
Daniel Armbruster committed
226
        if debugMode:
Daniel Armbruster's avatar
Daniel Armbruster committed
227
          CsReport(227, "Performing check of file with file itself.").display()
228
229
230
        csline.check()
      else:
        filename = csline.path.split(os.sep)[-1]
Daniel Armbruster's avatar
Daniel Armbruster committed
231
        if debugMode:
Daniel Armbruster's avatar
Daniel Armbruster committed
232
          CsReport(232, "Performing check of file with source: "+ \
Daniel Armbruster's avatar
Daniel Armbruster committed
233
            srcDir+os.sep+filename).display()
234
235
        csline.check(srcDir+os.sep+filename)
    self.write()
236
237

  def displayLines(self):
238
239
240
241
242
    """
    Display the content of the checksum file at stdout.
    """
    if not len(self.__cslines):
      raise CsFileError("CSFILE does not contain any lines.", 193)
243
244
245
246
247
    for line in self.__cslines:
      sys.stdout.write(line)

# -----------------------------------------------------------------------------
class CsLine:
248
249
250
  """
  Class to handle a checksum and further data for a registered file.
  """
Daniel Armbruster's avatar
Daniel Armbruster committed
251

252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
  def __init__(self, *args):
    if isinstance(args[0], list):
      argList = args[0]
      self.checksum = argList[0]
      self.path = argList[1]
      self.hashfunc = argList[2]
      self.creationDateFile = argList[3]
      self.creationLocationChecksum = argList[4]
      self.creationDateChecksum = argList[5]
      self.dateLastCheck = argList[6]
      self.statusLastCheck = argList[7]
    elif isinstance(args[0], str) and isinstance(args[1], str):
      self.checksum = ''
      self.path = args[0]
      self.hashfunc = args[1]
      self.creationDateFile = ''
      self.creationLocationChecksum = ''
      self.creationDateChecksum = ''
      self.dateLastCheck = ''
      self.statusLastCheck = ''
    else:
Daniel Armbruster's avatar
Daniel Armbruster committed
273
      CsFileError(273, "Invalid argument")
Daniel Armbruster's avatar
Daniel Armbruster committed
274
275

  def generate(self, chunkSize):
276
277
278
279
    """
    Generate the checksum and establish corresponding data for a file. The
    result is a fully configured checksum line.
    """
280
    if debugMode:
Daniel Armbruster's avatar
Daniel Armbruster committed
281
      CsReport(281, "Calculating checksum for file: "+self.path).display()
Daniel Armbruster's avatar
Daniel Armbruster committed
282
283
284
285
    # generate checksum
    try:
      hashfunc = hashlib.new(self.hashfunc)
      blockSize = chunkSize * hashfunc.block_size
286
287
288
289
290
      file = open(self.path, 'rb')
      data = file.read(blockSize)
      while data:
        hashfunc.update(data)
        data = file.read(blockSize)
Daniel Armbruster's avatar
Daniel Armbruster committed
291
292
      self.checksum = hashfunc.hexdigest()
    except IOError as err:
Daniel Armbruster's avatar
Daniel Armbruster committed
293
      raise CsFileError(293, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
Daniel Armbruster's avatar
Daniel Armbruster committed
294
        +err.filename)
295
296
297
    else:
      file.close()
      
Daniel Armbruster's avatar
Daniel Armbruster committed
298
299
300
301
302
303
304
305
306
307
    # set remaining data
    self.creationDateFile = \
      datetime.fromtimestamp(os.path.getctime(self.path)).strftime( \
      "%Y/%m/%d-%H:%M:%S")
    self.creationLocationChecksum = os.uname()[1]
    self.creationDateChecksum = datetime.now().strftime("%Y/%m/%d-%H:%M:%S")
    self.dateLastCheck = self.creationDateChecksum 
    self.statusLastCheck = 'ok'

  def check(self, src=''):
308
309
310
311
312
313
314
315
316
317
    """
    Check a checksum line. Either compare the checksum of the line with the
    checksum of the file located at src or compare the checksum with the checksum
    of the file located at the path of its one checksum line.
    """
    # calculate checksum
    try:
      hashfunc = hashlib.new(self.hashfunc)
      blockSize = chunkSize * hashfunc.block_size
      if 0 == len(src):
318
        file = open(self.path, 'rb')
319
      else:
320
321
322
323
324
325
        file = open(src, 'rb')
      data = file.read(blockSize)
      while data:
        hashfunc.update(data)
        data = file.read(blockSize)
      checksum = hashfunc.hexdigest()
326
327
    except IOError as err:
      # Maybe better to avoid Exception here and just put the status to notice
Daniel Armbruster's avatar
Daniel Armbruster committed
328
      raise CsFileError(328, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
Daniel Armbruster's avatar
Daniel Armbruster committed
329
        +err.filename)
330
331
332
333
    else:
      file.close()
      # checks
      if not len(self.checksum):
Daniel Armbruster's avatar
Daniel Armbruster committed
334
        raise CsFileError(334, "Caclulation of checksum was not successful.")
335
336
337
338
      if checksum == self.checksum:
        self.statusLastCheck = 'ok'
      else:
        self.statusLastCheck = 'error'
339

Daniel Armbruster's avatar
Daniel Armbruster committed
340
  def __str__(self):
341
342
343
    """
    String representation of a checksum line.
    """
Daniel Armbruster's avatar
Daniel Armbruster committed
344
345
346
347
    return '{0} {1} {2} {3} {4} {5} {6} {7}'.format(self.checksum, self.path, \
    self.hashfunc, self.creationDateFile, self.creationLocationChecksum, \
    self.creationDateChecksum, self.dateLastCheck, self.statusLastCheck)
    
348
# -----------------------------------------------------------------------------
349
# Tests
350
if __name__ == '__main__':
351
  try:
Daniel Armbruster's avatar
Daniel Armbruster committed
352
    debugMode = True
353
354
355
356
357
    file = CsFile('/home/daniel/')
    file.read()
  except CsFileError as err:
    err.display()
    sys.exit()
Daniel Armbruster's avatar
Daniel Armbruster committed
358
359
  
  
Daniel Armbruster's avatar
Daniel Armbruster committed
360
# ----- END OF csfile.py -----