csfile.py 11.4 KB
Newer Older
1
#!/usr/bin/env python
Daniel Armbruster's avatar
Daniel Armbruster committed
2
## @file csfile.py
Daniel Armbruster's avatar
Daniel Armbruster committed
3
4
# @brief  Provide a module to read, write and treat with a csback
# checksumfiles.
Daniel Armbruster's avatar
Daniel Armbruster committed
5
6
7
8
9
10
11
# 
# -----------------------------------------------------------------------------
# 
# $Id$
# @author Daniel Armbruster
# \date 15/09/2011
# 
Daniel Armbruster's avatar
Daniel Armbruster committed
12
13
# Purpose: Provide a module to read, write and treat with a csback
# checksumfiles.
Daniel Armbruster's avatar
Daniel Armbruster committed
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
#
# ----
# This file is part of csback.
#
# csback is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# csback is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with csback.  If not, see <http://www.gnu.org/licenses/>.
# ----
# 
# Copyright (c) 2011 by Daniel Armbruster
# 
# REVISIONS and CHANGES 
# 15/09/2011  V0.1    Daniel Armbruster
Daniel Armbruster's avatar
Daniel Armbruster committed
36
37
# 01/01/2012  V0.1.1  finished implementation
# 02/01/2012  V0.1.2  implemented debugMode
Daniel Armbruster's avatar
Daniel Armbruster committed
38
39
# 
# =============================================================================
40
""" CsFile module to handle checksum files. """
Daniel Armbruster's avatar
Daniel Armbruster committed
41

Daniel Armbruster's avatar
Daniel Armbruster committed
42
43
import os
import re
44
import sys
Daniel Armbruster's avatar
Daniel Armbruster committed
45
import hashlib
Daniel Armbruster's avatar
Daniel Armbruster committed
46
from datetime import datetime
Daniel Armbruster's avatar
Daniel Armbruster committed
47

48
__version__ = "V0.1"
Daniel Armbruster's avatar
Daniel Armbruster committed
49
__subversion__ = "$Id$"
50
51
__license__ = "GPLv2"
__author__ = "Daniel Armbruster"
Daniel Armbruster's avatar
Daniel Armbruster committed
52
53
54
55
56
__copyright__ = "Copyright (c) 2012 by Daniel Armbruster"

# -----------------------------------------------------------------------------
# global variables

57
chunkSize = 1024 * 128 # 128kB
Daniel Armbruster's avatar
Daniel Armbruster committed
58
debugMode = False
Daniel Armbruster's avatar
Daniel Armbruster committed
59

60
61
# -----------------------------------------------------------------------------
class CsFileError(Exception):
Daniel Armbruster's avatar
Daniel Armbruster committed
62
63
64
65
  """
  Exception class of csfile module.
  """
  def __init__(self, line, msg):
66
    self.msg = msg
67
    self.line = line
68
69

  def display(self):
Daniel Armbruster's avatar
Daniel Armbruster committed
70
71
72
73
74
75
76
77
78
    sys.stderr.write("csfile (ERROR): " + str(self.msg) + "\n")
    sys.stderr.write("triggered in line: " + str(self.line) + "\n")

class CsReport(CsFileError):
  """
  Auxiliary class for debugging.
  """
  def display(self):
    sys.stderr.write("csfile (REPORT): " + str(self.msg) + "\n")
79
    sys.stderr.write("triggered in line: " + str(self.line) + "\n")
80
81
82

# -----------------------------------------------------------------------------
class CsFile:
83
84
85
  """
  Provides an interface to handle a csback checksum file.
  """
Daniel Armbruster's avatar
Daniel Armbruster committed
86
87
88
  def __init__(self, filepath, hashfunc='sha256'):
    self.__filepath = filepath
    self.__filename = ".cs"
89
90
    self.__cslines = []
    self.__hashfunc = hashfunc
91
    self.createFile()
Daniel Armbruster's avatar
Daniel Armbruster committed
92
93
94
95
96
97

  def createFile(self):
    path = self.__filepath+os.sep+self.__filename
    if not os.path.isfile(path):
      try:
        csfile = open(path, 'w')
98
      except IOError as err:
99
        raise CsFileError(99, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
Daniel Armbruster's avatar
Daniel Armbruster committed
100
          +err.filename)
101
      else:
Daniel Armbruster's avatar
Daniel Armbruster committed
102
        csfile.close()
103
104

  def read(self):
105
106
107
    """
    Read a checksum file.
    """
Daniel Armbruster's avatar
Daniel Armbruster committed
108
    path = self.__filepath+os.sep+self.__filename
109
    try:
Daniel Armbruster's avatar
Daniel Armbruster committed
110
      if debugMode:
111
        CsReport(111, "Start reading checksumfile: "+path).display()
Daniel Armbruster's avatar
Daniel Armbruster committed
112
      csfile = open(path)
113
114
      self.__cslines = [CsLine(line.split()) for line in csfile \
      if len(line.rstrip()) and line[0] != '#'] 
115
116
117
    except IOError as err:
      # Maybe better to create the file here and keep on going.
      # Will be managed later during further dev.
118
      raise CsFileError(118, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
Daniel Armbruster's avatar
Daniel Armbruster committed
119
        +err.filename)
120
121
    else:
      csfile.close()
Daniel Armbruster's avatar
Daniel Armbruster committed
122
      if debugMode:
123
        CsReport(123, "Finished reading checksumfile: "+path).display()
124
125
126
127
128
129
130

  def write(self):
    """
    Write the entire checksumfile.
    """
    path = self.__filepath+os.sep+self.__filename
    try:
Daniel Armbruster's avatar
Daniel Armbruster committed
131
      if debugMode:
132
        CsReport(132, "Start writing checksumfile: "+path).display()
133
134
      csfile = open(path, 'w')
      for csline in self.__cslines:
Daniel Armbruster's avatar
Daniel Armbruster committed
135
        if debugMode:
136
          CsReport(136, "Writing line: "+ str(csline)).display()
137
138
139
        if isinstance(csline, CsLine):
          csfile.write(str(csline) + '\n')
        else:
140
          raise CsFileError(140, "Argument must be of type CsLine.")
141
    except IOError as err:
142
      raise CsFileError(142, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
Daniel Armbruster's avatar
Daniel Armbruster committed
143
        +err.filename)
144
    else:
145
      csfile.close()
Daniel Armbruster's avatar
Daniel Armbruster committed
146
      if debugMode:
147
        CsReport(147, "Finished writing checksumfile: "+path)
Daniel Armbruster's avatar
Daniel Armbruster committed
148

Daniel Armbruster's avatar
Daniel Armbruster committed
149
  def append(self, cslines):
150
151
152
    """
    Append checksum lines to the checksum file.
    """
Daniel Armbruster's avatar
Daniel Armbruster committed
153
    path = self.__filepath+os.sep+self.__filename
Daniel Armbruster's avatar
Daniel Armbruster committed
154
    if 0 == len(cslines) and debugMode:
155
      CsReport(155, "Empty list passed. Nothing to append.").display()
156
    else:
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
      try:
        if debugMode:
          CsReport(159, "Start appending to checksumfile: "+path).display()
        csfile = open(path, 'a')
        for csline in cslines:
          if debugMode:
            CsReport(163, "Writing line: "+str(csline)).display()
          if isinstance(csline, CsLine):
            csfile.write(str(csline) + '\n')
          else:
            raise CsFileError(167, "Argument must be of type CsLine.")
      except IOError as err:
        raise CsFileError(169, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
          +err.filename)
      else:
        if debugMode:
          CsReport(173, "Finished appending to checksumfile: "+path).display()
        csfile.close()
Daniel Armbruster's avatar
Daniel Armbruster committed
175
176
      
  def update(self, regexes=[]):
177
178
179
180
181
    """
    Update a checksum file. Also includes appending of not registered files to
    checksum file in current directory.
    """
    if not isinstance(regexes, list):
182
      raise CsFileError(182, "Pass regular expressions in a list.")
Daniel Armbruster's avatar
Daniel Armbruster committed
183
    if debugMode:
184
      CsReport(184, "Updating checksumfile ...").display()
Daniel Armbruster's avatar
Daniel Armbruster committed
185
    # fetch cslines in current csfile
186
    self.read()
Daniel Armbruster's avatar
Daniel Armbruster committed
187
    if debugMode:
188
189
190
191
192
193
194
195
196
      CsReport(188, "Fetching files not registered yet.").display()
    registeredFiles = set(csline.path for csline in self.__cslines)
    # fetch files (pathes) excluding those matching regexes
    regexes.append(self.__filename)
    newFiles = set([os.path.join(self.__filepath, file) \
      for file in os.listdir(self.__filepath) \
      if os.path.isfile(os.path.join(self.__filepath,file)) \
      for regex in regexes if None == re.match(regex, file)])
    regexes.remove(self.__filename)
Daniel Armbruster's avatar
Daniel Armbruster committed
197
    # exclude registered files
198
    newFiles -= registeredFiles
Daniel Armbruster's avatar
Daniel Armbruster committed
199
    # generate cslines of newFiles
200
201
202
203
204
205
206
    cslines = []
    for file in newFiles:
      csline = CsLine(file, self.__hashfunc)
      csline.generate(chunkSize)
      cslines.append(csline)
      print(csline)
    self.append(cslines)
Daniel Armbruster's avatar
Daniel Armbruster committed
207

208
209
210
211
212
213
214
  def check(self, srcDir=''):
    """
    Check a checksum file which means:
    1. read checksum file
    2, calculate checksum of file which is located in srcDir and check results
    3. write the result to the checksum file
    """
Daniel Armbruster's avatar
Daniel Armbruster committed
215
    if debugMode:
216
      CsReport(216, "Start checking checksums.").display()
217
    self.read()
Daniel Armbruster's avatar
Daniel Armbruster committed
218
    if 0 == len(self.__cslines) and debugMode:
219
      CsReport(219, "CSFILE does not contain any lines.").display()
220
221
    for csline in self.__cslines:
      if not len(srcDir):
Daniel Armbruster's avatar
Daniel Armbruster committed
222
        if debugMode:
223
          CsReport(223, "Performing check of file with file itself.").display()
224
225
226
        csline.check()
      else:
        filename = csline.path.split(os.sep)[-1]
Daniel Armbruster's avatar
Daniel Armbruster committed
227
        if debugMode:
228
          CsReport(228, "Performing check of file with source: "+ \
Daniel Armbruster's avatar
Daniel Armbruster committed
229
            srcDir+os.sep+filename).display()
230
231
        csline.check(srcDir+os.sep+filename)
    self.write()
232
233

  def displayLines(self):
234
235
236
237
238
    """
    Display the content of the checksum file at stdout.
    """
    if not len(self.__cslines):
      raise CsFileError("CSFILE does not contain any lines.", 193)
239
240
241
242
243
    for line in self.__cslines:
      sys.stdout.write(line)

# -----------------------------------------------------------------------------
class CsLine:
244
245
246
  """
  Class to handle a checksum and further data for a registered file.
  """
Daniel Armbruster's avatar
Daniel Armbruster committed
247

248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
  def __init__(self, *args):
    if isinstance(args[0], list):
      argList = args[0]
      self.checksum = argList[0]
      self.path = argList[1]
      self.hashfunc = argList[2]
      self.creationDateFile = argList[3]
      self.creationLocationChecksum = argList[4]
      self.creationDateChecksum = argList[5]
      self.dateLastCheck = argList[6]
      self.statusLastCheck = argList[7]
    elif isinstance(args[0], str) and isinstance(args[1], str):
      self.checksum = ''
      self.path = args[0]
      self.hashfunc = args[1]
      self.creationDateFile = ''
      self.creationLocationChecksum = ''
      self.creationDateChecksum = ''
      self.dateLastCheck = ''
      self.statusLastCheck = ''
    else:
      CsFileError(269, "Invalid argument")
Daniel Armbruster's avatar
Daniel Armbruster committed
270
271

  def generate(self, chunkSize):
272
273
274
275
    """
    Generate the checksum and establish corresponding data for a file. The
    result is a fully configured checksum line.
    """
276
277
    if debugMode:
      CsReport(277, "Calculating checksum for file: "+self.path).display()
Daniel Armbruster's avatar
Daniel Armbruster committed
278
279
280
281
    # generate checksum
    try:
      hashfunc = hashlib.new(self.hashfunc)
      blockSize = chunkSize * hashfunc.block_size
282
283
284
285
286
      file = open(self.path, 'rb')
      data = file.read(blockSize)
      while data:
        hashfunc.update(data)
        data = file.read(blockSize)
Daniel Armbruster's avatar
Daniel Armbruster committed
287
288
      self.checksum = hashfunc.hexdigest()
    except IOError as err:
289
      raise CsFileError(289, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
Daniel Armbruster's avatar
Daniel Armbruster committed
290
        +err.filename)
291
292
293
    else:
      file.close()
      
Daniel Armbruster's avatar
Daniel Armbruster committed
294
295
296
297
298
299
300
301
302
303
    # set remaining data
    self.creationDateFile = \
      datetime.fromtimestamp(os.path.getctime(self.path)).strftime( \
      "%Y/%m/%d-%H:%M:%S")
    self.creationLocationChecksum = os.uname()[1]
    self.creationDateChecksum = datetime.now().strftime("%Y/%m/%d-%H:%M:%S")
    self.dateLastCheck = self.creationDateChecksum 
    self.statusLastCheck = 'ok'

  def check(self, src=''):
304
305
306
307
308
309
310
311
312
313
    """
    Check a checksum line. Either compare the checksum of the line with the
    checksum of the file located at src or compare the checksum with the checksum
    of the file located at the path of its one checksum line.
    """
    # calculate checksum
    try:
      hashfunc = hashlib.new(self.hashfunc)
      blockSize = chunkSize * hashfunc.block_size
      if 0 == len(src):
314
        file = open(self.path, 'rb')
315
      else:
316
317
318
319
320
321
        file = open(src, 'rb')
      data = file.read(blockSize)
      while data:
        hashfunc.update(data)
        data = file.read(blockSize)
      checksum = hashfunc.hexdigest()
322
323
    except IOError as err:
      # Maybe better to avoid Exception here and just put the status to notice
324
      raise CsFileError(324, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
Daniel Armbruster's avatar
Daniel Armbruster committed
325
        +err.filename)
326
327
328
329
    else:
      file.close()
      # checks
      if not len(self.checksum):
330
        raise CsFileError(330, "Caclulation of checksum was not successful.")
331
332
333
334
      if checksum == self.checksum:
        self.statusLastCheck = 'ok'
      else:
        self.statusLastCheck = 'error'
335

Daniel Armbruster's avatar
Daniel Armbruster committed
336
  def __str__(self):
337
338
339
    """
    String representation of a checksum line.
    """
Daniel Armbruster's avatar
Daniel Armbruster committed
340
341
342
343
    return '{0} {1} {2} {3} {4} {5} {6} {7}'.format(self.checksum, self.path, \
    self.hashfunc, self.creationDateFile, self.creationLocationChecksum, \
    self.creationDateChecksum, self.dateLastCheck, self.statusLastCheck)
    
344
# -----------------------------------------------------------------------------
345
# Tests
346
if __name__ == '__main__':
347
  try:
Daniel Armbruster's avatar
Daniel Armbruster committed
348
    debugMode = True
349
350
351
352
353
    file = CsFile('/home/daniel/')
    file.read()
  except CsFileError as err:
    err.display()
    sys.exit()
Daniel Armbruster's avatar
Daniel Armbruster committed
354
355
  
  
Daniel Armbruster's avatar
Daniel Armbruster committed
356
# ----- END OF csfile.py -----