csfile.py 28.8 KB
Newer Older
1
#!/usr/bin/env python
Daniel Armbruster's avatar
Daniel Armbruster committed
2
## @file csfile.py
Daniel Armbruster's avatar
Daniel Armbruster committed
3 4
# @brief  Provide a module to read, write and treat with a csback
# checksumfiles.
Daniel Armbruster's avatar
Daniel Armbruster committed
5 6 7 8 9 10 11
# 
# -----------------------------------------------------------------------------
# 
# $Id$
# @author Daniel Armbruster
# \date 15/09/2011
# 
Daniel Armbruster's avatar
Daniel Armbruster committed
12 13
# Purpose: Provide a module to read, write and treat with a csback
# checksumfiles.
Daniel Armbruster's avatar
Daniel Armbruster committed
14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
#
# ----
# This file is part of csback.
#
# csback is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# csback is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with csback.  If not, see <http://www.gnu.org/licenses/>.
# ----
# 
32
# Copyright (c) 2011-2012 by Daniel Armbruster
Daniel Armbruster's avatar
Daniel Armbruster committed
33 34 35
# 
# REVISIONS and CHANGES 
# 15/09/2011  V0.1    Daniel Armbruster
Daniel Armbruster's avatar
Daniel Armbruster committed
36
# 01/01/2012  V0.1.1  finished implementation
37
# 04/01/2012  V0.1.3  implemented a debug mode using the python logging module
38
# 17/01/2012  V0.3    provide compatibility with unix sha???sum tools
39
#                     CsTimeProcessor class
40
# 29/01/2012  V0.4    improved result file logging mechanism
41
#                     now time flag syntax exactly equal to unix find command
42
# 21/05/2012  V0.5    improved documentation and correction of time exclusion
43 44 45 46
# 25/02/2013  V0.5.1  add csline access function of class CsFile
# 26/02/2013  V0.6    introduce csbackErrorCodes; provide a simplified
#                     CsLine.check() function (provide no logging anymore)
# 12/03/2013  V0.6.1  BUG in CsFile.check() function fixed
47
# 14/03/2013  V0.6.2  make use with statement
48
# 26/10/2013  V0.6.3  increase verbosity in debug mode (CsFile.update())
49
# 08/11/2013  V0.6.4  Time processing bugs fixed.
50 51 52 53
# 05/12/2013  V0.6.5  (thof) bug fix ticket:248 (race condition when excluding
#                     files based on time stamp values) for open time windows
#                     (like all younger than 3 days), we use datetime.max as
#                     the end time
54 55
# 01/08/2019  V0.6.6  (thof) let getSubDirectories produce a reasonable error
#                     message, if application of a regex fails
56 57
#             V0.6.7  (thof) capture problem with whitespace in file names
#                     which results in not accepted hash types
58
#
Daniel Armbruster's avatar
Daniel Armbruster committed
59
# =============================================================================
60
""" CsFile module to handle checksumfiles. """
Daniel Armbruster's avatar
Daniel Armbruster committed
61

Daniel Armbruster's avatar
Daniel Armbruster committed
62 63
import os
import re
64
import sys
65
import pwd
Daniel Armbruster's avatar
Daniel Armbruster committed
66
import hashlib
Daniel Armbruster's avatar
Daniel Armbruster committed
67
import logging
Daniel Armbruster's avatar
Daniel Armbruster committed
68
from datetime import datetime
69 70 71
from datetime import timedelta
from datetime import time
from datetime import date
72
import csbacklog
73
import csbackErrorCodes as eCodes
Daniel Armbruster's avatar
Daniel Armbruster committed
74

75
__version__ = "V0.6.7"
Daniel Armbruster's avatar
Daniel Armbruster committed
76
__subversion__ = "$Id$"
77
__license__ = "GPLv2+"
78
__author__ = "Daniel Armbruster"
Daniel Armbruster's avatar
Daniel Armbruster committed
79 80 81
__copyright__ = "Copyright (c) 2012 by Daniel Armbruster"

# -----------------------------------------------------------------------------
82
# variables
Daniel Armbruster's avatar
Daniel Armbruster committed
83

84
chunkSize = 1024 * 128 # 128kB
85

86
csfileLoggerName = ''
Daniel Armbruster's avatar
Daniel Armbruster committed
87

88 89 90
BASENAME = "checksumfile"
CSSUFFIX = ".cs"
RESULTSUFFIX = ".result"
91
LOCKFILENAME = '.lock'
92

Daniel Armbruster's avatar
Daniel Armbruster committed
93 94 95 96 97 98 99 100 101 102 103
# -----------------------------------------------------------------------------
# functions

def getSubDirectories(path, regexes, followLinks=False):
  """
  To generate a list of subdirectories of path using os.walk(). Note that path
  itself was not appended to the list.
  """
  subDirs = set()
  try:
    # collect subdirectories
104
    for root, dirs, files in os.walk(path, followlinks=followLinks):
Daniel Armbruster's avatar
Daniel Armbruster committed
105 106 107 108
      for dir in dirs:
        subDirs.add(os.path.join(root, dir))
    # exclude directories matching regexes
    for regex in regexes:
109 110 111 112
      try:
	matching = set(dir for dir in subDirs if None != re.match(regex, dir))
      except:
        raise CsFileError("application of exclude regex %s failed!" % regex, eCodes.CSFILE_CollectingSubDirs)
Daniel Armbruster's avatar
Daniel Armbruster committed
113 114
      subDirs -= matching
  except OSError as err:
115 116
    raise CsFileError("[Errno "+str(err.errno)+"] "+err.strerror+": " \
      +err.filename, eCodes.CSFILE_CollectingSubDirs)
Daniel Armbruster's avatar
Daniel Armbruster committed
117 118 119
  else:
    return subDirs

120 121 122 123 124
def hasCsFile(path):
  """
  Checks if path contains a checksumfile. Returns True if path contains a file
  named with an CsFile filename.
  """
125
  return os.path.isfile(os.path.join(path, BASENAME+CSSUFFIX))
126

127 128
# -----------------------------------------------------------------------------
class CsFileError(Exception):
Daniel Armbruster's avatar
Daniel Armbruster committed
129 130 131
  """
  Exception class of csfile module.
  """
132
  def __init__(self, msg, errorCode):
133
    self.msg = msg
134
    self.errorCode = errorCode
135

136 137
  def __str__(self):
    return "csfile (ERROR): {0} ({1}).\n".format(self.msg, self.errorCode)
Daniel Armbruster's avatar
Daniel Armbruster committed
138

139 140
# -----------------------------------------------------------------------------
class CsFile:
141
  """
142
  Provides an interface to handle a csback checksum file. A checksum file
Daniel Armbruster's avatar
Daniel Armbruster committed
143
  possesses the ability to take the files for generating the checksums from a
144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161
  different source directory which can be configured with the srcdir variable.

  Keyword arguments:
  filedir     -- directory path of checksum file
  srcdir      -- directory path to the data files which will be used for
                 comparison and/or update mechanisms of member routines.
  hashfunc    -- type of the hash function used
  timeDict    -- additional dictionary of special time expressions which will be
                 used to exclude data files matching the expressions. The
                 content will be processed by inherited classes from ABC
                 TimeProcessor.
  followlinks -- Boolean flag. If set to True member functions also will
                 consider symbolic links

  Additional notes:
  A checksum file usually contains checksum lines (type CsLine) of refering to
  data files. Every (sub)directory contains its own checksum file containing
  entries of data files located in the directory.
162
  """
163 164
  def __init__(self, filedir, srcdir, hashfunc='sha256', timeDict={},
    followlinks=False):
165
    self.filedir = filedir
166
    self.srcdir = srcdir
167 168 169 170
    self.followlinks = followlinks
    self.__hashfunc = hashfunc
    self.__timeDict = timeDict
    self.__cslines = []
171
    self.logger = logging.getLogger(csfileLoggerName+".CsFile")
Daniel Armbruster's avatar
Daniel Armbruster committed
172

173 174
  def read(self):
    """
Daniel Armbruster's avatar
Daniel Armbruster committed
175
    Read a checksumfile.
176 177
    """
    if not os.access(self.filedir, os.F_OK):
178
      raise CsFileError("Invalid directory path.", eCodes.CSFILE_DirNotExistent)
179
    path = os.path.join(self.filedir, BASENAME+CSSUFFIX)
180 181
    # no checksumfile available -> create new file
    if os.access(self.filedir, os.F_OK) and not os.path.isfile(path):
182
      self.logger.info("Creating checksumfile in '%s'", self.filedir)
Daniel Armbruster's avatar
Daniel Armbruster committed
183 184
      try:
        csfile = open(path, 'w')
185
      except IOError as err:
186 187
        raise CsFileError("[Errno "+str(err.errno)+"] "+err.strerror+": " \
          +err.filename, eCodes.CSFILE_UnableToCreateChecksumFile)
188
      else:
Daniel Armbruster's avatar
Daniel Armbruster committed
189
        csfile.close()
190
    # checksumfile available -> read file
191
    else:
192
      try:
193
        self.logger.debug("Start reading checksumfile '%s'",path)
194 195 196
        with open(path, 'r') as csfile: 
          self.__cslines = [CsLine(line.split(), self.srcdir) \
          for line in csfile if len(line.rstrip()) and line[0] != '#'] 
197
      except IOError as err:
198 199
        raise CsFileError("[Errno "+str(err.errno)+"] "+err.strerror+": " \
          +err.filename, eCodes.CSFILE_UnableToReadChecksumFile)
200
      else:
201
        self.logger.debug("Finished reading checksumfile '%s'", path)
202

Daniel Armbruster's avatar
Daniel Armbruster committed
203
  def append(self, cslines):
204
    """
205
    Append checksum lines to the checksumfile.
206
    """
207
    path = os.path.join(self.filedir, BASENAME+CSSUFFIX)
208 209
    if 0 == len(cslines):
      self.logger.debug("Empty list passed. Nothing to append.")
210
    else:
211 212
      try:
        self.logger.debug("Start appending to checksumfile '%s'", path)
213 214 215 216 217 218 219 220
        with open(path, 'a') as csfile: 
          for csline in cslines:
            self.logger.debug("Writing line '%s'", str(csline))
            if isinstance(csline, CsLine):
              csfile.write(str(csline) + '\n')
            else:
              raise CsFileError("Argument must be of type CsLine.", \
                  eCodes.CSFILE_InvalidDataType)
221
      except IOError as err:
222
        raise CsFileError("[Errno "+str(err.errno)+"] "+err.strerror+": " \
223
            +err.filename, eCodes.CSFILE_UnableToAppend)
224 225
      else:
        self.logger.debug("Finished appending to checksumfile '%s'", path)
226

227
  def update(self, regexes=[], dryRunMode=False):
228 229 230
    """
    Update a checksum file. Also includes appending of not registered files to
    checksum file in current directory.
231 232 233 234

    Key arguments:
    regexes -- list of regular expressions to exclude files and directories 
               from the registration
235
    """
236
    self.logger.debug("Updating checksumfile ...")
Daniel Armbruster's avatar
Daniel Armbruster committed
237
    # fetch cslines in current csfile
238
    self.read()
239
    self.logger.debug("Fetching files not registered yet.")
240 241
    registeredFiles = set(csline.filename for csline in self.__cslines)
    # fetch files
242
    newFiles = set()
243 244
    newFiles = set(file for file in os.listdir(self.srcdir) \
      if os.path.isfile(os.path.join(self.srcdir, file)))
245 246 247 248 249
    # exclude links if demanded
    if not self.followlinks:
      newfiles = set(file for file in newFiles \
        if not os.path.islink(os.path.join(self.srcdir, file)))
    # exclude files matching time selection
250
    excludedFiles = set()
251 252 253 254 255 256 257 258 259 260 261 262
    try:
      self.logger.debug("Excluding files matching time selection.")
      if self.__timeDict['hasaTimes']:
        aExcludes = CsTimeProcessor(self.__timeDict['amin'], \
          self.__timeDict['atime'], self.__timeDict['daystart']).getResult()
        for pair in aExcludes:
          matching = set(file for file in newFiles \
            if datetime.fromtimestamp(os.stat(os.path.join(self.srcdir, \
              file)).st_atime) > pair[0] and \
              datetime.fromtimestamp(os.stat(os.path.join(self.srcdir, \
              file)).st_atime) < pair[1])
          newFiles -= matching
263
          excludedFiles.update(matching)
264 265 266 267 268 269 270 271 272 273
      if self.__timeDict['hascTimes']:
        cExcludes = CsTimeProcessor(self.__timeDict['cmin'], \
          self.__timeDict['ctime'], self.__timeDict['daystart']).getResult() 
        for pair in cExcludes:
          matching = set(file for file in newFiles \
            if datetime.fromtimestamp(os.stat(os.path.join(self.srcdir, \
              file)).st_ctime) > pair[0] and \
              datetime.fromtimestamp(os.stat(os.path.join(self.srcdir, \
              file)).st_ctime) < pair[1])
          newFiles -= matching
274
          excludedFiles.update(matching)
275 276 277 278 279 280 281 282 283 284
      if self.__timeDict['hasmTimes']:
        mExcludes = CsTimeProcessor(self.__timeDict['mmin'], \
          self.__timeDict['mtime'], self.__timeDict['daystart']).getResult() 
        for pair in mExcludes:
          matching = set(file for file in newFiles \
            if datetime.fromtimestamp(os.stat(os.path.join(self.srcdir, \
              file)).st_mtime) > pair[0] and \
              datetime.fromtimestamp(os.stat(os.path.join(self.srcdir, \
              file)).st_mtime) < pair[1])
          newFiles -= matching
285
          excludedFiles.update(matching)
286 287 288 289
    except KeyError:
      self.logger.debug( \
        "KeyError while excluding files matching times selection.")
      pass
Daniel Armbruster's avatar
Daniel Armbruster committed
290
    # exclude files matching regexes
291 292 293 294 295 296 297 298 299
    try:
      regexes.append(BASENAME+CSSUFFIX)
      regexes.append(BASENAME+RESULTSUFFIX)
      regexes.append(BASENAME+RESULTSUFFIX+r'\.[1-2]')
      regexes.append(LOCKFILENAME)
    except TypeError:
      raise CsFileError("Pass regular expressions in a list.", \
          eCodes.CSFILE_InvalidType)

300
    self.logger.debug("Excluding files matching regular expressions.")
Daniel Armbruster's avatar
Daniel Armbruster committed
301 302 303 304
    for regex in regexes:
      matching = set(file for file in newFiles \
        if None != re.match(regex, file))
      newFiles -= matching
305
      excludedFiles.update(matching)
306
    regexes.remove(BASENAME+CSSUFFIX)
Daniel Armbruster's avatar
Daniel Armbruster committed
307 308
    regexes.append(BASENAME+RESULTSUFFIX)
    regexes.remove(BASENAME+RESULTSUFFIX+r'\.[1-2]')
309
    regexes.append(LOCKFILENAME)
Daniel Armbruster's avatar
Daniel Armbruster committed
310
    # exclude registered files
311
    self.logger.debug("Excluding already registered files.")
312
    newFiles -= registeredFiles
313 314 315 316 317 318 319 320
    excludedFiles.update(registeredFiles)
    # increase verbosity in debug mode
    if len(excludedFiles):
      self.logger.debug("List of excluded files:")
    for file in excludedFiles:
      self.logger.debug("{0}".format(file))
    if len(newFiles):
      self.logger.debug("A checksum will be generated for the following files:")
321
    for file in newFiles:
322 323 324 325 326 327 328 329 330 331 332 333
      self.logger.debug("{0}".format(file))

    if not dryRunMode:
      # generate cslines of newFiles
      cslines = []
      for file in newFiles:
        csline = CsLine(file, self.srcdir, self.__hashfunc)
        csline.generate(chunkSize)
        cslines.append(csline)
      self.append(cslines)
      path = os.path.join(self.filedir, BASENAME+CSSUFFIX)
      self.logger.debug("Update of checksumfile '%s' finished.", path)
Daniel Armbruster's avatar
Daniel Armbruster committed
334

335
  def check(self, regexes, beTolerant=False):
336
    """
337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358
    Check a checksum file which means and log the results. Concretely this
    means that a checksum for a data file (located in self.srcdir) is computed
    and the result is compared to the checksum available in the checksum file.

    Keyword arguments:
    regexes    -- list of regular expressions to exclude files and directories 
                  from the verification process
    beTolerant -- Flag (type: Boolean). In case no file had been found in src
                  and the beTolerant flag had been set to True the status of the
                  log message will be set to 'WARNING' else to 'ERROR'.  
  
    Additional notes:
    Besides of logging the results to the general csback log file (usually
    located in /var/log/) more detailed logs will be safed to so called
    checksumfile.result files.
    INFO log messages will be safed to the log files if the data files have
    integrity. In case a corrupt data file has been detected a log message
    with a CRITICAL log level will be issued.

    Keep in mind that this function does not check if there are unregistered
    data files in the directory. Adding checksum lines to the checksumfile has
    to be done by using the update member function.
359
    """
360
    if 0 == len(self.__cslines):
Daniel Armbruster's avatar
Daniel Armbruster committed
361
      self.logger.info( \
362
        "CSFILE does not contain any lines or had not been read yet.")
363 364
    # initialize handler for logger 
    resultLogger = csbacklog.ResultLogger(self.filedir, BASENAME+RESULTSUFFIX)
365
 
366 367 368 369 370
    # exclude those files matching regex in regexes
    self.logger.debug("Exclude files matching regexes")
    cslinesSet = set(self.__cslines)
    for regex in regexes:
      matching = set(csline for csline in self.__cslines \
Daniel Armbruster's avatar
Daniel Armbruster committed
371
        if None != re.match(regex, csline.filename))
372
      cslinesSet -= matching
373 374 375 376
    # exclude links if demanded
    if not self.followlinks:
      cslinesSet = set(csline for csline in cslinesSet \
        if not os.path.islink(os.path.join(csline.srcdir, csline.filename)))
377
    # perform check with increased log verbosity
378
    self.logger.debug("Start checking checksums ...")
379
    for csline in self.__cslines:
380
      if csline in cslinesSet: 
381
        path=os.path.join(csline.srcdir, csline.filename) 
382
        self.logger.debug( \
383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405
          "Performing check of file '{0}' with csline in checksumfile '{1}'.". \
          format(path, self.filedir))
        try:
          if csline.check():
            logoutput = "Check of file '%s' was successful." 
            resultLogger.getLogger().info(logoutput, csline.filename)
            self.logger.debug(logoutput, path)
          else:
            logoutput = "File '%s' has no integrity anymore." 
            resultLogger.getLogger().critical(logoutput, csline.filename)
            self.logger.critical(logoutput, path)
        except CsFileError:
          if beTolerant:
            logoutput = "While checking: file '%s' does not exist." 
            resultLogger.getLogger().warning(logoutput, csline.filename)
            self.logger.warning(logoutput, path)
          else:
            logoutput = "While checking: file '%s' does not exist." 
            resultLogger.getLogger().error(logoutput, csline.filename)
            self.logger.error(logoutput, path)
        except:
          raise CsFileError("Unhandled exception.", \
              eCodes.GLOBAL_UnhandledException)
406
    self.logger.debug("Finished checking checksums.")
407
    resultLogger.reset()
408

409
  def display(self):
410
    """
411
    Display the content of the checksum file to stdout.
412
    """
413 414
    if 0 == len(self.__cslines):
      self.logger.info("CSFILE does not contain any lines.")
415 416 417
    for line in self.__cslines:
      sys.stdout.write(line)

418 419 420 421 422 423 424 425 426 427 428 429
  def __getitem__(self, key):
    try:
      return self.__cslines[key]
    except TypeError:
      for line in self.__cslines:
        if line.filename == key:
          return line
      raise CsFileError( \
          "Checksum file does not contain an entry for file '{0}'.".format( \
          key), eCodes.CSFILE_NoEntryAvail)


430 431
# -----------------------------------------------------------------------------
class CsLine:
432 433 434
  """
  Class to handle a checksum and further data for a registered file.
  """
435
  def __init__(self, *args):
436
    self.logger = logging.getLogger(csfileLoggerName+".CsLine")
437
    if isinstance(args[0], list) and isinstance(args[1], str):
438
      self.srcdir = args[1]
439 440 441 442 443 444 445 446 447 448 449 450
      self.checksum = args[0][0]
      self.filename = args[0][1]
      if len(args[0]) > 2:
        self.hashfunc = args[0][2]
        self.creationDateFile = args[0][3]
        self.creationLocationChecksum = args[0][4]
        self.creationDateChecksum = args[0][5]
      else:
        self.hashfunc = ''
        self.creationDateFile = ''
        self.creationLocationChecksum = ''
        self.creationDateChecksum = ''
451 452
    elif isinstance(args[0], str) and isinstance(args[1], str) and \
      isinstance(args[2], str):
453
      self.checksum = ''
454
      self.filename = args[0]
455
      self.srcdir = args[1]
456
      self.hashfunc = args[2]
457 458 459 460
      self.creationDateFile = ''
      self.creationLocationChecksum = ''
      self.creationDateChecksum = ''
    else:
461
      raise CsFileError("Invalid argument(s).", eCodes.CSFILE_InvalidType)
Daniel Armbruster's avatar
Daniel Armbruster committed
462 463

  def generate(self, chunkSize):
464 465 466 467
    """
    Generate the checksum and establish corresponding data for a file. The
    result is a fully configured checksum line.
    """
Daniel Armbruster's avatar
Daniel Armbruster committed
468
    # generate checksum
469
    path = os.path.join(self.srcdir, self.filename) 
470
    self.logger.debug("Calculating checksum for '%s'", path)
Daniel Armbruster's avatar
Daniel Armbruster committed
471 472 473
    try:
      hashfunc = hashlib.new(self.hashfunc)
      blockSize = chunkSize * hashfunc.block_size
474
      with open(path, 'rb') as file:
475
        data = file.read(blockSize)
476 477 478 479
        while data:
          hashfunc.update(data)
          data = file.read(blockSize)
        self.checksum = hashfunc.hexdigest()
Daniel Armbruster's avatar
Daniel Armbruster committed
480
    except IOError as err:
481 482
      raise CsFileError("[Errno "+str(err.errno)+"] "+err.strerror+": " \
        +err.filename, eCodes.CSFILE_UnableToReadDataFile)
483
      
Daniel Armbruster's avatar
Daniel Armbruster committed
484 485
    # set remaining data
    self.creationDateFile = \
486
      datetime.fromtimestamp(os.path.getctime(path)).strftime( \
Daniel Armbruster's avatar
Daniel Armbruster committed
487 488 489 490
      "%Y/%m/%d-%H:%M:%S")
    self.creationLocationChecksum = os.uname()[1]
    self.creationDateChecksum = datetime.now().strftime("%Y/%m/%d-%H:%M:%S")

491
  def check(self):
492
    """
493 494 495 496 497 498
    Check a checksum line and return True or False depending on the integrity of
    the data file.

    The checksum of the line will be compared with the checksum of the file
    located in src. In case no data file was found in src an CsFileError
    exception will be thrown.
499
    """
500 501 502 503 504 505 506 507
    try:
      hashfunc = hashlib.new(self.hashfunc)
    except:
      raise CsFileError(("When evaluating checksum file entry\n  in directory %s\n"
       +"  hash type '%s' as specified for file '%s' is not accepted\n"
       +"  consider a problem with whitespace in file names\n"
       +"  the respective checksum file entry is\n"
       +"  %s") % (self.srcdir, self.hashfunc, self.filename, self), eCodes.GLOBAL_UnhandledException)
508
    blockSize = chunkSize * hashfunc.block_size
509
    path = os.path.join(self.srcdir, self.filename)
510
    try:
511 512 513 514 515 516 517
      with open(path, 'rb') as file:
        # calculate checksum
        data = file.read(blockSize)
        while data:
          hashfunc.update(data)
          data = file.read(blockSize)
        checksum = hashfunc.hexdigest()
518
    except IOError:
519 520
      raise CsFileError("While checking: file '{0}' does not exist.".format( \
          path), eCodes.CSFILE_DataFileNotExistent)
521
    else:
522
      return checksum == self.checksum
523

Daniel Armbruster's avatar
Daniel Armbruster committed
524
  def __str__(self):
525 526 527
    """
    String representation of a checksum line.
    """
528 529 530
    return '{0} {1} {2} {3} {4} {5}'.format(self.checksum, self.filename, \
    self.hashfunc, self.creationDateFile, self.creationLocationChecksum, \
    self.creationDateChecksum)
531

532

533
# -----------------------------------------------------------------------------
534
class CsTimeProcessor:
535 536
  """
  Generate datetimes tuples for file exclusion.
537

538 539 540 541 542 543
  The generated tuples contain two elements (index 0 and 1):
  index 0: beginning of time window (start)
  index 1: end of time window (end)
  The time window does not contain the start and end time themselves (this is
  the way time windows are evaluated in CsFile.update()

544 545
  Minute processing is done within this class though time processing was passed
  to remote abstract TimeProcessor class.
546 547
  """
  def __init__(self, min, time, daystart=False):
548 549 550 551 552 553
    """
    Parameters:
    min:      arguments to -[acm]min parameter
    time:     arguments to [-acm]time parameter
    daystart: daystart flag (value is true, if option is selected)
    """
554 555 556 557 558 559 560
    self.result = []
    self.min = min
    self.time = time
    self.daystart = daystart

  def process(self):
    """
561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582
    Processing function.
    """
    self.process_minutes()
    if self.daystart:
      daystartProcessor = DaystartTimeProcessor(self.time) 
      daystartProcessor.process()
      self.result.extend(daystartProcessor.getResult())
    else:
      standardProcessor = StandardTimeProcessor(self.time) 
      standardProcessor.process()
      self.result.extend(standardProcessor.getResult())

  def getResult(self):
    """
    Return result. List of tuples (lower value, upper value).
    """
    self.process()
    return self.result

  def process_minutes(self):
    """
    Process minute information to concrete tuples
583 584 585
    """
    try:
      start = datetime.now()
586
      #start = now-timedelta(microseconds=now.microsecond)
587 588 589 590 591 592
      if self.daystart and len(self.min) != 0:
        raise CsFileError("Using option '--daystart' additionally to \
'--[acm]min #' is not supported.", eCodes.GLOBAL_NotImplemented)
        # FIXME: Implementation of *nix find in this case seems quite confusing.
        #t = time(0)
        #start = datetime.combine(date.today(),t)
593 594 595 596 597 598

      minPlus = 0
      minMinus = 0
      minFixed = []
      # sort minutes
      for min in self.min:
Daniel Armbruster's avatar
Daniel Armbruster committed
599
        if '+' == min[0]:
600
          minPlus = int(min)
Daniel Armbruster's avatar
Daniel Armbruster committed
601
        elif '-' == min[0]:
602 603 604 605 606 607 608 609
          minMinus = int(min)
        else:
          minFixed.append(int(min))
      # process minutes
      if abs(minPlus) > abs(minMinus):
        if 0 != minPlus:
          self.result.append((datetime.min,start-timedelta(minutes=minPlus)))
        if 0 != minMinus:
610
          self.result.append((start+timedelta(minutes=minMinus), datetime.max))
611 612 613
      elif abs(minPlus) == abs(minMinus): 
        pass
      else:
614
        if 0 == minPlus:
615
          self.result.append((start+timedelta(minutes=minMinus), datetime.max))
616 617 618
        else:
          self.result.append((start+timedelta(minutes=minMinus), \
            start-timedelta(minutes=minPlus+1)))
619
      for min in minFixed: 
620 621
        self.result.append((start-timedelta(minutes=min), \
          start-timedelta(minutes=min-1)))
622 623 624 625 626 627 628 629 630 631
    except TypeError as err:
      raise CsFileError(514,str(err))
    except OverflowError as err:
      raise CsFileError(516, str(err))

# -----------------------------------------------------------------------------
# time processing classes
 
class TimeProcessor():
  """
632
  Abstract converter base class for time processors.
633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648
  """
  def __init__(self, time):
    self.time = time
    self.result = []

  def process(self):
    """
    Abstract function to process time information
    """
    self.processTime()

  def processTime(self):
    raise NotImplementedError("function must be defined!")

  def getResult(self):
    return self.result
649

650 651 652 653 654 655 656

class StandardTimeProcessor(TimeProcessor):
  """
  Standard time processor class
  """
  def processTime(self):
    try:
Daniel Armbruster's avatar
Daniel Armbruster committed
657
      start = datetime.now()
658 659 660 661
      timePlus = 0
      timeMinus = 0
      timeFixed = []
      # sort times
662
      #FIXME: +0 and -0 problem
663 664 665 666 667
      for t in self.time:
          if '+' == t[0]:
            timePlus = int(t)
          elif '-' == t[0]:
            timeMinus = int(t)
668
          else:
669
            timeFixed.append(int(t))
670 671
      if abs(timePlus) > abs(timeMinus):
        if 0 != timePlus:
672
          self.result.append((datetime.min, start-timedelta(days=timePlus+1)))
673
        if 0 != timeMinus:
674
          self.result.append((start+timedelta(days=timeMinus), datetime.max))
675 676 677
      elif abs(timePlus) == abs(timeMinus):
        pass
      else:
678
        if 0 == timePlus:
679
          self.result.append((start+timedelta(days=timeMinus), datetime.max))
680 681 682
        else:
          self.result.append((start+timedelta(days=timeMinus), \
            start-timedelta(days=timePlus+1)))
683 684 685
      for t in timeFixed:
        self.result.append((start-timedelta(days=t+1), \
          start-timedelta(days=t)))
686
    except TypeError as err:
687
      raise CsFileError(str(err), eCodes.CSFILE_TimeProcessingTypeError)
688
    except OverflowError as err:
689
      raise CsFileError(str(err), eCodes.CSFILE_TimeProcessingOverflowError)
Daniel Armbruster's avatar
Daniel Armbruster committed
690
    
691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712

class DaystartTimeProcessor(TimeProcessor):
  """
  Time processor class for set daystart flag
  """
  def processTime(self):
    try:
      t = time(0)
      start = datetime.combine(date.today(),t)
      timePlus = []
      timeMinus = []
      timeFixed = []
      # sort times
      for t in self.time:
          if '+' == t[0]:
            timePlus.append(int(t))
          elif '-' == t[0]:
            timeMinus.append(int(t))
          else:
            timeFixed.append(int(t))
      # process daystart times
      if len(timePlus) and len(timeMinus):
713
        # case 1:
714
        # ============|---------------------|==============|=========>
715 716 717
        #             ^                     ^              ^         t
        # exclude     | include             | exclude      | 
        #     daystart-timePlus     daystart+timeMinus    now
718 719 720
        if abs(timePlus[-1]) > abs(timeMinus[-1]):
          self.result.append((datetime.min, \
              start-timedelta(days=timePlus[-1])))
721
          self.result.append((start+timedelta(days=timeMinus[-1]), \
722
              datetime.max))
723
        # case 2: same times -> do nothing
724 725
        elif abs(timePlus[-1]) == abs(timeMinus[-1]):
          pass
726 727 728
        # case 3:
        # ------------|======================|--------------|--------->
        #             ^                      ^              ^         t
729 730
        # include     |        exclude       |    include   | include
        #     daystart+timeMinus+1   daystart-timePlus     now
731
        else:
732
          self.result.append((start+timedelta(days=timeMinus[-1]+1), \
733
            start-timedelta(days=timePlus[-1])))
734 735 736 737 738
        # case 4:
        # ============|------------|---------->
        #             ^            ^          t
        # exclude     | include    | include 
        #     daystart-timePlus   now
739 740
      elif len(timePlus) and 0 == len(timeMinus):
        self.result.append((datetime.min, start-timedelta(days=timePlus[-1])))
741
        
742
        # case 5:
743
        # ------------|============|==========>
744 745
        #             ^            ^          t
        # include     | exclude    | include 
746
        #    daystart+timeMinus+1 now
747
      elif 0 == len(timePlus) and len(timeMinus):
748
        self.result.append((start+timedelta(days=timeMinus[-1]+1), \
749
            datetime.max))
750 751
      for t in timeFixed:
        if 0 == t:
752
          # case 6:
753
          # ----------------|===========|=============>
754
          #                 ^           ^             t
755 756
          #                 |  exclude  |
          #              daystart      now
757
          self.result.append((start, datetime.max))
758
        else:
759 760
          # case 7:
          # ----------------|==================|---------|--------->
761 762 763 764 765
          #                 ^                  ^         ^         t
          #                 |     exclude      |         |
          #          daystart-days     daystart-days+1  now
          self.result.append((start-timedelta(days=t), \
            start-timedelta(days=t-1)))
766
    except TypeError as err:
767
      raise CsFileError(str(err), eCodes.CSFILE_TimeProcessingTypeError)
768
    except OverflowError as err:
769
      raise CsFileError(str(err), eCodes.CSFILE_TimeProcessingOverflowError)
770 771


772
# -----------------------------------------------------------------------------
773
# Tests
774
if __name__ == '__main__':
775 776 777 778 779 780
  try:
    file = CsFile('/home/daniel/')
    file.read()
  except CsFileError as err:
    err.display()
    sys.exit()
Daniel Armbruster's avatar
Daniel Armbruster committed
781
  
Daniel Armbruster's avatar
Daniel Armbruster committed
782
# ----- END OF csfile.py -----