csfile.py 11.3 KB
Newer Older
1
#!/usr/bin/env python
Daniel Armbruster's avatar
Daniel Armbruster committed
2
## @file csfile.py
Daniel Armbruster's avatar
Daniel Armbruster committed
3
4
# @brief  Provide a module to read, write and treat with a csback
# checksumfiles.
Daniel Armbruster's avatar
Daniel Armbruster committed
5
6
7
8
9
10
11
# 
# -----------------------------------------------------------------------------
# 
# $Id$
# @author Daniel Armbruster
# \date 15/09/2011
# 
Daniel Armbruster's avatar
Daniel Armbruster committed
12
13
# Purpose: Provide a module to read, write and treat with a csback
# checksumfiles.
Daniel Armbruster's avatar
Daniel Armbruster committed
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
#
# ----
# This file is part of csback.
#
# csback is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# csback is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with csback.  If not, see <http://www.gnu.org/licenses/>.
# ----
# 
# Copyright (c) 2011 by Daniel Armbruster
# 
# REVISIONS and CHANGES 
# 15/09/2011  V0.1    Daniel Armbruster
Daniel Armbruster's avatar
Daniel Armbruster committed
36
37
# 01/01/2012  V0.1.1  finished implementation
# 02/01/2012  V0.1.2  implemented debugMode
Daniel Armbruster's avatar
Daniel Armbruster committed
38
39
# 
# =============================================================================
40
""" CsFile module to handle checksum files. """
Daniel Armbruster's avatar
Daniel Armbruster committed
41

Daniel Armbruster's avatar
Daniel Armbruster committed
42
43
import os
import re
44
import sys
Daniel Armbruster's avatar
Daniel Armbruster committed
45
import hashlib
Daniel Armbruster's avatar
Daniel Armbruster committed
46
from datetime import datetime
Daniel Armbruster's avatar
Daniel Armbruster committed
47

48
__version__ = "V0.1"
Daniel Armbruster's avatar
Daniel Armbruster committed
49
__subversion__ = "$Id$"
50
51
__license__ = "GPLv2"
__author__ = "Daniel Armbruster"
Daniel Armbruster's avatar
Daniel Armbruster committed
52
53
54
55
56
57
__copyright__ = "Copyright (c) 2012 by Daniel Armbruster"

# -----------------------------------------------------------------------------
# global variables

chunkSize = 131072 # 128kB
Daniel Armbruster's avatar
Daniel Armbruster committed
58
debugMode = False
Daniel Armbruster's avatar
Daniel Armbruster committed
59

60
61
# -----------------------------------------------------------------------------
class CsFileError(Exception):
Daniel Armbruster's avatar
Daniel Armbruster committed
62
63
64
65
  """
  Exception class of csfile module.
  """
  def __init__(self, line, msg):
66
    self.msg = msg
67
    self.line = line
68
69

  def display(self):
Daniel Armbruster's avatar
Daniel Armbruster committed
70
71
72
73
74
75
76
77
78
    sys.stderr.write("csfile (ERROR): " + str(self.msg) + "\n")
    sys.stderr.write("triggered in line: " + str(self.line) + "\n")

class CsReport(CsFileError):
  """
  Auxiliary class for debugging.
  """
  def display(self):
    sys.stderr.write("csfile (REPORT): " + str(self.msg) + "\n")
79
    sys.stderr.write("triggered in line: " + str(self.line) + "\n")
80
81
82

# -----------------------------------------------------------------------------
class CsFile:
83
84
85
  """
  Provides an interface to handle a csback checksum file.
  """
Daniel Armbruster's avatar
Daniel Armbruster committed
86
87
88
  def __init__(self, filepath, hashfunc='sha256'):
    self.__filepath = filepath
    self.__filename = ".cs"
89
90
    self.__cslines = []
    self.__hashfunc = hashfunc
91
    self.createFile()
Daniel Armbruster's avatar
Daniel Armbruster committed
92
93
94
95
96
97

  def createFile(self):
    path = self.__filepath+os.sep+self.__filename
    if not os.path.isfile(path):
      try:
        csfile = open(path, 'w')
98
      except IOError as err:
Daniel Armbruster's avatar
Daniel Armbruster committed
99
100
        raise CsFileError(97, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
          +err.filename)
101
      else:
Daniel Armbruster's avatar
Daniel Armbruster committed
102
        csfile.close()
103
104

  def read(self):
105
106
107
    """
    Read a checksum file.
    """
Daniel Armbruster's avatar
Daniel Armbruster committed
108
    path = self.__filepath+os.sep+self.__filename
109
    try:
Daniel Armbruster's avatar
Daniel Armbruster committed
110
111
      if debugMode:
        CsReport(109, "Start reading checksumfile: "+path).display()
Daniel Armbruster's avatar
Daniel Armbruster committed
112
      csfile = open(path)
113
114
      self.__cslines = [CsLine(line.split()) for line in csfile \
      if len(line.rstrip()) and line[0] != '#'] 
115
116
117
    except IOError as err:
      # Maybe better to create the file here and keep on going.
      # Will be managed later during further dev.
Daniel Armbruster's avatar
Daniel Armbruster committed
118
119
      raise CsFileError(116, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
        +err.filename)
120
121
    else:
      csfile.close()
Daniel Armbruster's avatar
Daniel Armbruster committed
122
123
      if debugMode:
        CsReport(121, "Finished reading checksumfile: "+path).display()
124
125
126
127
128
129
130

  def write(self):
    """
    Write the entire checksumfile.
    """
    path = self.__filepath+os.sep+self.__filename
    try:
Daniel Armbruster's avatar
Daniel Armbruster committed
131
132
      if debugMode:
        CsReport(130, "Start writing checksumfile: "+path).display()
133
134
      csfile = open(path, 'w')
      for csline in self.__cslines:
Daniel Armbruster's avatar
Daniel Armbruster committed
135
136
        if debugMode:
          CsReport(134, "Writing line: "+ str(csline)).display()
137
138
139
        if isinstance(csline, CsLine):
          csfile.write(str(csline) + '\n')
        else:
Daniel Armbruster's avatar
Daniel Armbruster committed
140
          raise CsFileError(136, "Argument must be of type CsLine.")
141
    except IOError as err:
Daniel Armbruster's avatar
Daniel Armbruster committed
142
143
      raise CsFileError(138, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
        +err.filename)
144
    else:
145
      csfile.close()
Daniel Armbruster's avatar
Daniel Armbruster committed
146
147
      if debugMode:
        CsReport(145, "Finished writing checksumfile: "+path)
Daniel Armbruster's avatar
Daniel Armbruster committed
148

Daniel Armbruster's avatar
Daniel Armbruster committed
149
  def append(self, cslines):
150
151
152
    """
    Append checksum lines to the checksum file.
    """
Daniel Armbruster's avatar
Daniel Armbruster committed
153
    path = self.__filepath+os.sep+self.__filename
Daniel Armbruster's avatar
Daniel Armbruster committed
154
155
    if 0 == len(cslines) and debugMode:
      CsReport(153, "Invalid argument. Empty list.").display()
Daniel Armbruster's avatar
Daniel Armbruster committed
156
    try:
Daniel Armbruster's avatar
Daniel Armbruster committed
157
158
      if debugMode:
        CsReport(156, "Start appending to checksumfile: "+path).display()
Daniel Armbruster's avatar
Daniel Armbruster committed
159
160
      csfile = open(path, 'a')
      for csline in cslines:
Daniel Armbruster's avatar
Daniel Armbruster committed
161
162
        if debugMode:
          CsReport(160, "Writing line: "+str(csline)).display()
Daniel Armbruster's avatar
Daniel Armbruster committed
163
164
165
        if isinstance(csline, CsLine):
          csfile.write(str(csline) + '\n')
        else:
Daniel Armbruster's avatar
Daniel Armbruster committed
166
          raise CsFileError(164, "Argument must be of type CsLine.")
Daniel Armbruster's avatar
Daniel Armbruster committed
167
    except IOError as err:
Daniel Armbruster's avatar
Daniel Armbruster committed
168
169
      raise CsFileError(166, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
        +err.filename)
170
    else:
Daniel Armbruster's avatar
Daniel Armbruster committed
171
172
      if debugMode:
        CsReport(170, "Finished appending to checksumfile: "+path).display()
Daniel Armbruster's avatar
Daniel Armbruster committed
173
174
175
      csfile.close()
      
  def update(self, regexes=[]):
176
177
178
179
180
    """
    Update a checksum file. Also includes appending of not registered files to
    checksum file in current directory.
    """
    if not isinstance(regexes, list):
Daniel Armbruster's avatar
Daniel Armbruster committed
181
182
183
      raise CsFileError(179, "Pass regular expressions in a list.")
    if debugMode:
      CsReport(181, "Updating checksumfile ...").display()
Daniel Armbruster's avatar
Daniel Armbruster committed
184
    # fetch cslines in current csfile
185
    self.read()
Daniel Armbruster's avatar
Daniel Armbruster committed
186
187
    if debugMode:
      CsReport(185, "Fetching files not registered yet.").display()
188
    registeredFiles = [csline.path.split(os.sep)[-1] \
Daniel Armbruster's avatar
Daniel Armbruster committed
189
190
191
192
      for csline in self.__cslines]
    # fetch files excluding regex matches
    newFiles = list(set([file for file in os.listdir(self.__filepath) \
      if os.path.isfile(file) for regex in regexes \
Daniel Armbruster's avatar
Daniel Armbruster committed
193
      if None == re.search(str(regex),file)]))
Daniel Armbruster's avatar
Daniel Armbruster committed
194
195
196
197
198
199
    # exclude registered files
    newFiles = list(set([file for file in newFiles \
      for registeredFile in registeredFiles if registeredFile != file]))
    # generate cslines of newFiles
    csLines = [CsLine(self.__filepath+os.sep+file, self.__hashfunc) \
      for file in newFiles]
200
    for line in csLines:
Daniel Armbruster's avatar
Daniel Armbruster committed
201
202
      if debugMode:
        CsReport(200, "Calculating checksum of file: "+file).display()
203
204
      line.generate(chunkSize)
    self.append(csLines)
Daniel Armbruster's avatar
Daniel Armbruster committed
205

206
207
208
209
210
211
212
  def check(self, srcDir=''):
    """
    Check a checksum file which means:
    1. read checksum file
    2, calculate checksum of file which is located in srcDir and check results
    3. write the result to the checksum file
    """
Daniel Armbruster's avatar
Daniel Armbruster committed
213
214
    if debugMode:
      CsReport(212, "Start checking checksums.").display()
215
    self.read()
Daniel Armbruster's avatar
Daniel Armbruster committed
216
217
    if 0 == len(self.__cslines) and debugMode:
      CsReport(215, "CSFILE does not contain any lines.").display()
218
219
    for csline in self.__cslines:
      if not len(srcDir):
Daniel Armbruster's avatar
Daniel Armbruster committed
220
221
        if debugMode:
          CsReport(219, "Performing check of file with file itself.").display()
222
223
224
        csline.check()
      else:
        filename = csline.path.split(os.sep)[-1]
Daniel Armbruster's avatar
Daniel Armbruster committed
225
226
227
        if debugMode:
          CsReport(219, "Performing check of file with source: "+ \
            srcDir+os.sep+filename).display()
228
229
        csline.check(srcDir+os.sep+filename)
    self.write()
230
231

  def displayLines(self):
232
233
234
235
236
    """
    Display the content of the checksum file at stdout.
    """
    if not len(self.__cslines):
      raise CsFileError("CSFILE does not contain any lines.", 193)
237
238
239
240
241
    for line in self.__cslines:
      sys.stdout.write(line)

# -----------------------------------------------------------------------------
class CsLine:
242
243
244
  """
  Class to handle a checksum and further data for a registered file.
  """
Daniel Armbruster's avatar
Daniel Armbruster committed
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268

  def __init__(self, path, hashfunc='sha256'):
    self.checksum = ''
    self.path = path
    self.hashfunc = hashfunc
    self.creationDateFile = ''
    self.creationLocationChecksum = ''
    self.creationDateChecksum = ''
    self.dateLastCheck = ''
    self.statusLastCheck = ''

  def __init__(self, argList):
    if not isinstance(argList, list):
      raise CsFileError("Invalid argument.")
    self.checksum = argList[0]
    self.path = argList[1]
    self.hashfunc = argList[2]
    self.creationDateFile = argList[3]
    self.creationLocationChecksum = argList[4]
    self.creationDateChecksum = argList[5]
    self.dateLastCheck = argList[6]
    self.statusLastCheck = argList[7]

  def generate(self, chunkSize):
269
270
271
272
    """
    Generate the checksum and establish corresponding data for a file. The
    result is a fully configured checksum line.
    """
Daniel Armbruster's avatar
Daniel Armbruster committed
273
274
275
276
277
    # generate checksum
    try:
      hashfunc = hashlib.new(self.hashfunc)
      blockSize = chunkSize * hashfunc.block_size
      with open(self.path, "rb") as file:
278
        for chunk in iter(lambda: file.read(blockSize), ''):
Daniel Armbruster's avatar
Daniel Armbruster committed
279
280
281
          hasfunc.update(chunk)
      self.checksum = hashfunc.hexdigest()
    except IOError as err:
Daniel Armbruster's avatar
Daniel Armbruster committed
282
283
      raise CsFileError(280, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
        +err.filename)
284
285
286
    else:
      file.close()
      
Daniel Armbruster's avatar
Daniel Armbruster committed
287
288
289
290
291
292
293
294
295
296
    # set remaining data
    self.creationDateFile = \
      datetime.fromtimestamp(os.path.getctime(self.path)).strftime( \
      "%Y/%m/%d-%H:%M:%S")
    self.creationLocationChecksum = os.uname()[1]
    self.creationDateChecksum = datetime.now().strftime("%Y/%m/%d-%H:%M:%S")
    self.dateLastCheck = self.creationDateChecksum 
    self.statusLastCheck = 'ok'

  def check(self, src=''):
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
    """
    Check a checksum line. Either compare the checksum of the line with the
    checksum of the file located at src or compare the checksum with the checksum
    of the file located at the path of its one checksum line.
    """
    # calculate checksum
    try:
      hashfunc = hashlib.new(self.hashfunc)
      blockSize = chunkSize * hashfunc.block_size
      if 0 == len(src):
        with open(self.path, "rb") as file:
          for chunk in iter(lambda: file.read(blockSize), ''):
            hasfunc.update(chunk)
      else:
        with open(src, "rb") as file:
          for chunk in iter(lambda: file.read(blockSize), ''):
            hasfunc.update(chunk)
        checksum = hashfunc.hexdigest()
    except IOError as err:
      # Maybe better to avoid Exception here and just put the status to notice
Daniel Armbruster's avatar
Daniel Armbruster committed
317
318
      raise CsFileError(275, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
        +err.filename)
319
320
321
322
    else:
      file.close()
      # checks
      if not len(self.checksum):
Daniel Armbruster's avatar
Daniel Armbruster committed
323
        raise CsFileError(280, "Caclulation of checksum was not successful.")
324
325
326
327
      if checksum == self.checksum:
        self.statusLastCheck = 'ok'
      else:
        self.statusLastCheck = 'error'
328

Daniel Armbruster's avatar
Daniel Armbruster committed
329
  def __str__(self):
330
331
332
    """
    String representation of a checksum line.
    """
Daniel Armbruster's avatar
Daniel Armbruster committed
333
334
335
336
    return '{0} {1} {2} {3} {4} {5} {6} {7}'.format(self.checksum, self.path, \
    self.hashfunc, self.creationDateFile, self.creationLocationChecksum, \
    self.creationDateChecksum, self.dateLastCheck, self.statusLastCheck)
    
337
# -----------------------------------------------------------------------------
338
# Tests
339
if __name__ == '__main__':
340
  try:
Daniel Armbruster's avatar
Daniel Armbruster committed
341
    debugMode = True
342
343
344
345
346
    file = CsFile('/home/daniel/')
    file.read()
  except CsFileError as err:
    err.display()
    sys.exit()
Daniel Armbruster's avatar
Daniel Armbruster committed
347
348
  
  
Daniel Armbruster's avatar
Daniel Armbruster committed
349
# ----- END OF csfile.py -----