csbackgen.py

#!/usr/bin/env python
## @file csbackgen.py
# @brief Generate checksum files.
# 
# -----------------------------------------------------------------------------
# 
# $Id$
# @author Daniel Armbruster
# \date 11/09/2011
# 
# Purpose: Generate checksum files.
#
# ----
# This file is part of csback.
#
# csback is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# 
# csback is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with csback.  If not, see <http://www.gnu.org/licenses/>.
# ----
# 
# Copyright (c) 2012 by Daniel Armbruster
# 
# REVISIONS and CHANGES 
# 02/01/2012  V0.1  Daniel Armbruster
# 05/01/2012  V0.2  added logging flag to enable a file logging mechanism
# 10/01/2012  V0.3  introduced --hash flag
# 11/01/2012  V0.4  pid lockfile handler ability added
# 17/01/2012  V0.5  implemented file selection regarding time
# 
# =============================================================================
 
import getopt
import os
import sys
import re
import pwd
import logging
import csfile
import csbacklog
import pidlock

__version__ = "V0.5"
__subversion__ = "$Id$"
__license__ = "GPLv2"
__author__ = "Daniel Armbruster"
__copyright__ = "Copyright (c) 2012 by Daniel Armbruster"

# -----------------------------------------------------------------------------
class Error(Exception):

  def __init__(self, msg):
    self.msg = msg

  def display(self):
    sys.stderr.write("csbackgen (ERROR): " + self.msg + "\n")


class Usage(Error):

  def display(self):
    usage_text = "Version: "+__version__+"\nLicense: "+__license__+ \
      "\n"+__subversion__+"\nAuthor: "+__author__+ """
 Usage: csbackgen [-v|--verbose] [-e REGEX [-e REGEX [...]]]
                  [-R|--notrecursive] [-d|--debug] [-f|--followlinks]
                  [-t|--target DIR] [-l|--logging] [-H|--hash ARG]
                  [-L|--lock] [-D|--daystart] [--atime N] [--mtime N] 
                  [--ctime N] [--amin N] [--mmin N] [--cmin N] PATH
    or: csbackgen -h|--help\n"""
    sys.stderr.write("csbackgen: " + self.msg + "\n")
    sys.stderr.write(usage_text)

# -----------------------------------------------------------------------------
def help():
  """
  Printing helptext to stdout.
  """
  help_text = "Version: "+__version__+"\nLicense: "+__license__+"\n"+ \
    __subversion__+"\nAuthor: "+__author__+"""
 Usage: csbackgen [-v|--verbose] [-e REGEX [-e REGEX [...]]]
                  [-R|--notrecursive] [-d|--debug] [-f|--followlinks] 
                  [-t|--target ROOTDIR] [-l|--logging] [-H|--hash ARG]
                  [-L|--lock] [-D|--daystart] [--atime N] [--mtime N] 
                  [--ctime N] [--amin N] [--mmin N] [--cmin N] PATH
    or: csbackgen -h|--help
-------------------------------------------------------------------------------
 -v|--verbose         Be verbose.
 -h|--help            Display this help.
 -e REGEX             While generating a checksumfile exclude files matching
                      REGEX(s).
 -R|--notrecursive    Do not generate checksumfiles for subdirectories of PATH.
 -d|--debug           Debug mode. Be really verbose.
 -f|--followlinks     Follow symbolic links. Only available if option -R is not
                      set. Note that this option can lead to infinite
                      recursion.
 -t|--target ROOTDIR  Root target directory for checksumfile. The checksumfile
                      will be put to the appropiated location as the files had
                      in PATH or rather Paths' subdirectories. So target must
                      have the same subdirectory structure as PATH.
 -l|--logging         Switch on logging to files. Logfiles will be located in
                      /var/log/ .
 -H|--hash ARG        Set the hash function algorithm. Valid values are:
                      sha224, sha256, sha384, sha512. (default: sha256)
 -L|--lock            Lock the directories working at. This flag is useful in
                      case csbackgen was run simultaneously with other csback
                      processes working in the same directory. Setting this
                      option avoids checksumfile access problems which might
                      occur.
 -D|--daystart        Measure times (for --amin, --atime, --cmin, --ctime,
                      --mmin, and --mtime) from the beginning of today rather
                      than from 24 hours ago.
 --amin N             Exclude files which were last accessed N minutes ago.
 --atime N            Exclude files which were last accessed N*24 hours ago.
                      To match --atime +1, a file has to have been accessed at
                      least two days ago.
 --cmin N             Exclude files which status was last changed N minutes ago.
 --ctime N            Exclude files which status was last changed N*24 hours
                      ago.
 --mmin N             Exclude files which data was last modified N minutes ago.            
 --mtime N            Exclude files which data was last modified N*24 hours
                      ago.
 PATH                 Path to generate checksumfile(s) for including its
                      subdirectories if option '-R' is not set.
-------------------------------------------------------------------------------

1. General
==========
csbackgen.py will either generate a checksumfile if still no checksumfile is
available or in case there is an existing checksumfile csbackgen.py will append
the not yet registered files to the current checksumfile. In the latter case
csbackgen.py is working in its update mode.
Notice that in case PATH contains subdirectories and either option '-R' is set
or the subdirectory is excluded by a matching regular expression or time
specification every subdirectory will contain its own checksumfile.

2. Time specification
=====================
For file selection regarding time numeric arguments can be passed as follows:

    +N     for greater/older than N,

    -N     for less/younger than N,

    N      for exactly N.

a) Notice if several +N (for greater than N) or -N (for less than N) values are
passed with one parameter the last one will overwrite the previous values.
b) --cmin and --ctime arguments are evaluated platform dependent i.e. time of
most recent metadata change on Unix, or the time of creation on Windows. 
c) If times for selection file are specified here the files matching were
excluded in contrast to the Unix find command where files were included.\n"""
  sys.stdout.write(help_text)

# -----------------------------------------------------------------------------
# global variables
VALIDHASHES = ['sha224', 'sha256', 'sha384', 'sha512']

# -----------------------------------------------------------------------------
def main(argv=None):
  # configure logger
  logger = csbacklog.CsbackLog('csbackgen')
  
  console = logging.StreamHandler()
  console.setFormatter(logging.Formatter( \
    '%(name)-8s [%(lineno)d]: %(levelname)-8s %(message)s'))

  if argv is None:
    argv = sys.argv
  try:
    try:
      opts, args = getopt.getopt(argv[1:], "vhe:Rdft:lLH:D", ["help", 
        "verbose", "notrecursive", "debug", "followlinks", "target=", \
        "logging", "hash=", "lock", "daystart", "amin=", "atime=", \
        "cmin=", "ctime=", "mmin=", "mtime="])
    except getopt.GetoptError as err:
      raise Usage(err.msg)
    verbose = False
    debugMode = False
    notRecursive = False
    followlinks = False
    regexes = []
    targetSet = False
    enableLogging = False
    hashfunc = 'sha256'
    pidLock = False
    timeDict = {'daystart': False, 'hasaTimes': False, 'hascTimes': False, \
      'hasmTimes': False, 'amin': [], 'atime': [], 'cmin': [], 'ctime': [], \
      'mmin': [], 'mtime': []}
    for opt, arg in opts:
      if opt in ("-v", "--verbose"):
        verbose = True
        console.setLevel(logging.INFO)
      elif opt in ("-h", "--help"):
        sys.exit(help())
      elif opt in ("-R", "--notrecursive"):
        notRecursive = True
      elif opt in ("-e"):
        regexes.append(arg)
      elif opt in ("-d", "--debug"):
        debugMode = True
        console.setLevel(logging.DEBUG)
      elif opt in ("-f", "--followlinks"):
        followlinks = True
      elif opt in ("-t", "--target"):
        targetSet = True
        targetDirectory = arg.rstrip(os.sep)+os.sep
      elif opt in ("-l", "--logging"):
        logger.configure()
      elif opt in ("-H", "--hash"):
        if arg in VALIDHASHES:
          hashfunc = arg
        else:
          raise Usage("Invalid argument (hash).")
      elif opt in ("-L", "--lock"):
        pidLock = True
      elif opt in ("-D", "--daystart"):
        timeDict['daystart'] = True
      elif opt in ("--amin"):
        timeDict['hasaTimes'] = True
        timeDict['amin'].append(arg)
      elif opt in ("--cmin"):
        timeDict['hascTimes'] = True
        timeDict['cmin'].append(arg)
      elif opt in ("--mmin"):
        timeDict['hasmTimes'] = True
        timeDict['mmin'].append(arg)
      elif opt in ("--atime"):
        timeDict['hasaTimes'] = True
        timeDict['atime'].append(arg)
      elif opt in ("--ctime"):
        timeDict['hascTimes'] = True
        timeDict['ctime'].append(arg)
      elif opt in ("--mtime"):
        timeDict['hasmTimes'] = True
        timeDict['mtime'].append(arg)
      else:
        raise Usage("Unhandled option chosen.")

    if verbose or debugMode:
      logger.addHandler(console)

    # fetch remaining commandline arguments   
    if 1 == len(args):
      srcpath = str(args[0]).rstrip(os.sep)+os.sep
    else:
      raise Usage("Invalid argument.")
    if not os.path.isdir(srcpath):
      raise Usage("PATH not a valid directory.")
     
    # major part
    logger.getLogger().debug("Start collecting subdirectories ...")
    dirs = [srcpath]
    # if no target path for checksumfiles had been passed generate the and
    # create the checksum files in the srcpath
    if not targetSet:
      targetDirectory = srcpath
      targetdirs = dirs
    # otherwise use the target directory path passed
    else:
      targetdirs = [targetDirectory]
      if not os.path.isdir(targetDirectory):
        raise Usage("Target directory not a valid directory.")
    # generate checksumfiles for subdirectories (recursive mode)
    # so collect all available subdirectories here (excluding subdirectories)
    if not notRecursive:
      dirs.extend(csfile.getSubDirectories(srcpath, regexes, followlinks))
      # if a target directory had been passed extend the list of
      # target directories replacing the trunk of the source directory with the
      # target directory. Remember that this is only valid for recursive mode
      if targetSet:
        targetdirs.extend([dir.replace(dirs[0],targetDirectory,1) \
          for dir in dirs[1:]])
    
    # create a list of tuples now so that each target directory has its
    # corresponding source directory
    pathes = list(zip(targetdirs, dirs))
    logger.getLogger().debug("Directories arranged.")

    logger.getLogger().info("Start updating checksumfile(s) ...")
    # for all available pathes generate or rather update the checksumfile (after
    # performing pid lockfile checkings)
    for path in pathes:
      logger.getLogger().info( \
        "Updating checksumfile in '{0}' with files from '{1}' ...".format( \
        path[0], path[1]))
      lock = pidlock.PidLocker(path[0])
      checksumfile = csfile.CsFile(path[0], path[1], hashfunc=hashfunc, \
        timeDict=timeDict, followlinks=followlinks)

      # lock directory currently working in (use pid locking module) if possible
      # - else terminate raising an exception
      if lock.lockValid():
        raise pidlock.PidLockError(-1, \
          "Directory '{0}' locked.".format(path[0]))
      elif pidLock:
        lock.announce(os.getpid())
        checksumfile.update(regexes)
        lock.cancel(os.getpid())
      else:
        checksumfile.update(regexes)
      logger.getLogger().info( \
        "Finished updating checksumfile in '{0}'.".format(path[0]))
    
  except Usage as err:
    err.display()
    return 2
  except pidlock.PidLockError as err:
    logger.getLogger().critical("{0}".format(err.msg))
    err.display()
    return 2
  except csfile.CsFileError as err:
    logger.getLogger().error("{0}".format(err.msg))
    err.display()
    return 2
  else:
    logger.getLogger().info("Checksumfile(s) updated.")
    return 0
    

# -----------------------------------------------------------------------------
if __name__ == "__main__":
  sys.exit(main())

# ----- END OF csbackgen.py -----