Commit 8efe7ff7 authored by Daniel Armbruster's avatar Daniel Armbruster Committed by thomas.forbriger
Browse files

Both checking script and logging mechanism nearly implemented (proceeding).

This is a legacy commit from before 2015-05-18.
It may be incomplete as well as inconsistent.
See COPYING.legacy and README.history for details.

Not tested yet.


SVN Path:     http://gpitrsvn.gpi.uni-karlsruhe.de/repos/TFSoftware/trunk
SVN Revision: 4334
SVN UUID:     67feda4a-a26e-11df-9d6e-31afc202ad0c
parent 195ed18d
#!/usr/bin/env python
## @file csbackchk.py
# @brief
# @brief Check checksums or rather integrity of corresponding files.
#
# -----------------------------------------------------------------------------
#
# $Id$
# @author Daniel Armbruster
# \date 11/09/2011
# \date 03/01/2012
#
# Purpose:
# Purpose: Check checksums or rather integrity of corresponding files.
#
# ----
# This file is part of csback.
......@@ -27,13 +27,190 @@
# along with csback. If not, see <http://www.gnu.org/licenses/>.
# ----
#
# Copyright (c) 2011 by Daniel Armbruster
# Copyright (c) 2012 by Daniel Armbruster
#
# REVISIONS and CHANGES
# 11/09/2011 V0.1 Daniel Armbruster
# 03/01/2012 V0.1 Daniel Armbruster
#
# =============================================================================
import getopt
import os
import sys
import pwd
import logging
import logging.config
import csfile
__version__ = "V0.1"
__subversion__ = "$Id$"
__license__ = "GPLv2"
__author__ = "Daniel Armbruster"
__copyright__ = "Copyright (c) 2012 by Daniel Armbruster"
# ----- END OF csbackchk.py -----
\ No newline at end of file
# -----------------------------------------------------------------------------
class Error(Exception):
def __init__(self, msg):
self.msg = msg
def display(self):
sys.stderr.write("csbackchk (ERROR): " + self.msg + "\n")
class Usage(Error):
def display(self):
usage_text = "Version: "+__version__+"\nLicense: "+__license__+ \
"\n"+__subversion__+"\nAuthor: "+__author__+ """
Usage: csbackchk [-v|--verbose] [-e REGEX [-e REGEX [...]]]
[-R|--notrecursive] [-d|--debug] [-f|--followlinks]
[-t|--tolerant] [SOURCEPATH] PATH
or: csbackchk -h|--help\n"""
sys.stderr.write("csbackchk: " + self.msg + "\n")
sys.stderr.write(usage_text)
# -----------------------------------------------------------------------------
def help():
"""
Printing helptext to stdout.
"""
help_text = "Version: "+__version__+"\nLicense: "+__license__+"\n"+ \
__subversion__+"\nAuthor: "+__author__+"""
Usage: csbackchk [-v|--verbose] [-e REGEX [-e REGEX [...]]]
[-R|--notrecursive] [-d|--debug] [-f|--followlinks]
[-t|--tolerant] [SOURCEPATH] PATH
or: csbackchk -h|--help
-------------------------------------------------------------------------------
-v|--verbose Be verbose.
-h|--help Display this help.
-e REGEX While checking a checksumfile exclude files matching
REGEX.
-R|--notrecursive Do not search in subdirectories of PATH.
-d|--debug Debug mode. Be really verbose.
-f|--followlinks Follow symbolic links. Only available if option -R is not
set. Note that this option can lead to infinite recursion.
-t|--tolerant Be tolerant.
While checking don't report anything if a file listed in
the checksumfile is missing. This flag is useful if a test
of a directory is executed but this directory is realized
e.g. as a ring buffer.
SOURCEPATH Optional sourcepath for comparison with files backed up in
PATH. PATH and its subdirectories (if option '-R' is not
set) must contain the csback checksumfile(s).
Note that the directory structure (if option '-R' is not
set) bellow SOURCEPATH must be equal to those in PATH.
If SOURCEPATH is not passed a check of files located in
PATH with its checksumfiles will be performed.
PATH Path to perform check for.
"""
sys.stdout.write(help_text)
def main(argv=None):
# configure logger
logInfo = {'hostname': os.uname()[1], \
'user': str(pwd.getpwuid(os.getuid()).pw_name) \
'pid': os.getpid()}
csfile.__logInfo = logInfo
csfile.moduleLoggerName = 'csbackchk.csfile'
logger = logging.LoggerAdapter(logging.getLogger('csbackchk'), logInfo)
logging.basicConfig(level=logging.DEBUG, \
format='%(asctime)-15s %(hostname)-10s %(name)-15s[%(pid)-8s]: \
(%(user)-8s) %(levelname)-8s %(message)s')
console = logging.StreamHandler()
formatter = logging.Formatter( \
'%(name)-15s [%(lineno)d]: %(levelname)-15s %(message)s')
console.setFormatter(formatter)
if argv is None:
argv = sys.argv
try:
try:
opts, args = getopt.getopt(argv[1:], "vhe:Rdftl", ["help", "verbose", \
"notrecursive", "debug", "followlinks", "tolerant", "log"])
except getopt.GetoptError as err:
raise Usage(err.msg)
verbose = False
debugMode = False
notRecursive = False
followLinks = False
beTolerant = False
regexes = []
# collect arguments
for opt, arg in opts:
if opt in ("-v", "--verbose"):
verbose = True
console.setLevel(logging.INFO)
elif opt in ("-h", "--help"):
sys.exit(help())
return 0
elif opt in ("-R", "--notrecursive"):
notRecursive = True
elif opt in ("-e"):
regexes.append(arg)
elif opt in ("-d", "--debug"):
debugMode = True
console.setLevel(logging.DEBUG)
elif opt in ("-f", "--followlinks"):
followLinks = True
elif opt in ("-t", "--tolerant"):
beTolerant = True
else:
raise Usage("Unhandled option chosen.")
if verbose or debugMode:
logger.addHandler(console)
# fetch arguments
if 1 == len(args):
sourcepath = str(args[0]).rstrip(os.sep)+os.sep
inputpath = sourcepath
inputDirectories = [inputpath]
sourceDirectories = inputDirectories
elif 2 == len(args):
sourcepath = str(args[0]).rstrip(os.sep)+os.sep
inputpath = str(args[1]).rstrip(os.sep)+os.sep
inputDirectories = [inputpath]
sourceDirectories = [sourcepath]
else:
raise Usage("Invalid argument(s).")
if not notRecursive:
logger.info("Collecting subdirectories for inputpath: %s",inputpath)
inputDirectories.extend(csfile.getSubDirectories(inputpath, regexes, \
followLinks))
# case: different source directory chosen
if inputDirectories[0] != sourceDirectories[0]:
logger.info("Collecting subdirectories for sourcepath: %s",sourcepath)
sourceDirectories.extend(csfile.getSubDirectories(sourcepath, \
regexes, followLinks))
else:
logger.info("No recursion to subdirectories.")
# check directory structure of both sourcepath and inputpath
if inputDirectories[0] != sourceDirectories[0]:
logger.info("Checking directory structure.")
inTmp = [dir.rstrip(inputDirectories[0]) for dir in inputDirectories[1:]]
srcTmp = [dir.rstrip(sourceDirectories[0]) \
for dir in sourceDirectories[1:]]
if inTmp != srcTmp:
logger.error("Source directory does not contain the same directory \
structure like the input directory.")
directories = list(zip(inputDirectories, sourceDirectories))
for dirpair in directories:
csfile.CsFile(dirpair[0]).check(regexes, dirpair[1], beTolerant)
except Usage as err:
err.display()
return 2
except csfile.CsFileError as err:
err.display()
return 2
else:
logger.info("Checks performed.")
return 0
# -----------------------------------------------------------------------------
if __name__ == "__main__":
sys.exit(main())
# ----- END OF csbackchk.py -----
......@@ -63,8 +63,7 @@ class Usage(Error):
usage_text = "Version: "+__version__+"\nLicense: "+__license__+ \
"\n"+__subversion__+"\nAuthor: "+__author__+ """
Usage: csbackgen [-v|--verbose] [-e REGEX [-e REGEX [...]]]
[-R|--notrecursive] [-d|--debug] [-f|--followlinks]
[-l|--log] PATH
[-R|--notrecursive] [-d|--debug] [-f|--followlinks] PATH
or: csbackgen -h|--help\n"""
sys.stderr.write("csbackgen: " + self.msg + "\n")
sys.stderr.write(usage_text)
......@@ -77,8 +76,7 @@ def help():
help_text = "Version: "+__version__+"\nLicense: "+__license__+"\n"+ \
__subversion__+"\nAuthor: "+__author__+"""
Usage: csbackgen [-v|--verbose] [-e REGEX [-e REGEX [...]]]
[-R|--notrecursive] [-d|--debug] [-f|--followlinks]
[-l|--log] PATH
[-R|--notrecursive] [-d|--debug] [-f|--followlinks] PATH
or: csbackgen -h|--help
-------------------------------------------------------------------------------
-v|--verbose Be verbose.
......@@ -89,7 +87,6 @@ def help():
-d|--debug Debug mode. Be really verbose.
-f|--followlinks Follow symbolic links. Only available if option -R is not
set. Note that this option can lead to infinite recursion.
-l|--log Log status to ~/.csback/log
PATH Path to generate checksumfile for.
-------------------------------------------------------------------------------
csbackgen.py will either generate a checksumfile if still no checksumfile is
......@@ -100,6 +97,21 @@ csbackgen.py is working in its update mode.\n"""
# -----------------------------------------------------------------------------
def main(argv=None):
# configure logger
logInfo = {'hostname': os.uname()[1], \
'user': str(pwd.getpwuid(os.getuid()).pw_name) \
'pid': os.getpid()}
csfile.__logInfo = logInfo
csfile.moduleLoggerName = 'csbackgen.csfile'
logger = logging.LoggerAdapter(logging.getLogger('csbackgen'), logInfo)
logging.basicConfig(level=logging.DEBUG, \
format='%(asctime)-15s %(hostname)-10s %(name)-15s[%(pid)-8s]: \
(%(user)-8s) %(levelname)-8s %(message)s')
console = logging.StreamHandler()
formatter = logging.Formatter( \
'%(name)-15s [%(lineno)d]: %(levelname)-15s %(message)s')
console.setFormatter(formatter)
if argv is None:
argv = sys.argv
try:
......@@ -113,10 +125,10 @@ def main(argv=None):
notRecursive = False
followLinks = False
regexes = []
# collect arguments
for opt, arg in opts:
if opt in ("-v", "--verbose"):
verbose = True
console.setLevel(logging.INFO)
elif opt in ("-h", "--help"):
sys.exit(help())
elif opt in ("-R", "--notrecursive"):
......@@ -124,33 +136,31 @@ def main(argv=None):
elif opt in ("-e"):
regexes.append(arg)
elif opt in ("-d", "--debug"):
verbose = True
debugMode = True
csfile.debugMode = True
console.setLevel(logging.DEBUG)
elif opt in ("-f", "--followlinks"):
followLinks = True
else:
raise Usage("Unhandled option chosen.")
if verbose or debugMode:
logger.addHandler(console)
# fetch arguments
if 1 == len(args):
inputpath = str(args[0]).rstrip(os.sep)+os.sep
else:
raise Usage("Invalid argument.")
if notRecursive:
if verbose or debugMode:
sys.stdout.write("csbackgen.py: Generating checksumfile in: '"+ \
inputpath+"'\n")
logger.info("Generating checksumfile in: %s", inputpath)
csfile.CsFile(inputpath).update(regexes)
else:
if verbose or debugMode:
sys.stdout.write("csbackgen.py: Collecting subdirectories.\n")
logger.info("Collecting subdirectories.")
directories = [inputpath]
directories.extend(csfile.getSubDirectories(inputpath, regexes, \
followLinks))
print(directories)
if verbose or debugMode:
sys.stdout.write( \
"csbackgen.py: Generating checksumfiles in subdirectories.\n")
logger.info("Generating checksumfiles in subdirectories.")
for dir in directories:
csfile.CsFile(dir).update(regexes)
......@@ -161,6 +171,7 @@ def main(argv=None):
err.display()
return 2
else:
logger.info("Checksumfile(s) updated.")
return 0
......
......@@ -56,7 +56,9 @@ __copyright__ = "Copyright (c) 2012 by Daniel Armbruster"
# global variables
chunkSize = 1024 * 128 # 128kB
debugMode = False
__moduleLoggerName = ''
__logInfo = {}
# -----------------------------------------------------------------------------
# functions
......@@ -95,14 +97,6 @@ class CsFileError(Exception):
sys.stderr.write("csfile (ERROR): " + str(self.msg) + "\n")
sys.stderr.write("triggered in line: " + str(self.line) + "\n")
class CsReport(CsFileError):
"""
Auxiliary class for debugging.
"""
def display(self):
sys.stderr.write("csfile (REPORT): " + str(self.msg) + "\n")
sys.stderr.write("triggered in line: " + str(self.line) + "\n")
# -----------------------------------------------------------------------------
class CsFile:
"""
......@@ -113,6 +107,8 @@ class CsFile:
self.__filename = ".cs"
self.__cslines = []
self.__hashfunc = hashfunc
self.logger = logging.LoggerAdapter(logging.getLogger( \
__moduleLoggerName+".CsFile"), __logInfo)
self.createFile()
def createFile(self):
......@@ -132,8 +128,7 @@ class CsFile:
"""
path = self.__filepath+os.sep+self.__filename
try:
if debugMode:
CsReport(111, "Start reading checksumfile: "+path).display()
self.logger.debug("Start reading checksumfile %s",path)
csfile = open(path)
self.__cslines = [CsLine(line.split()) for line in csfile \
if len(line.rstrip()) and line[0] != '#']
......@@ -144,8 +139,7 @@ class CsFile:
+err.filename)
else:
csfile.close()
if debugMode:
CsReport(123, "Finished reading checksumfile: "+path).display()
self.logger.debug("Finished reading checksumfile: %s", path)
def write(self):
"""
......@@ -153,12 +147,10 @@ class CsFile:
"""
path = self.__filepath+os.sep+self.__filename
try:
if debugMode:
CsReport(132, "Start writing checksumfile: "+path).display()
self.logger.debug("Start writing checksumfile: %s",path)
csfile = open(path, 'w')
for csline in self.__cslines:
if debugMode:
CsReport(136, "Writing line: "+ str(csline)).display()
self.logger.debug("Writing line: %s",str(csline))
if isinstance(csline, CsLine):
csfile.write(str(csline) + '\n')
else:
......@@ -168,24 +160,20 @@ class CsFile:
+err.filename)
else:
csfile.close()
if debugMode:
CsReport(147, "Finished writing checksumfile: "+path)
self.logger.debug("Finished writing checksumfile: %s", path)
def append(self, cslines):
"""
Append checksum lines to the checksum file.
"""
path = self.__filepath+os.sep+self.__filename
if 0 == len(cslines) and debugMode:
CsReport(155, "Empty list passed. Nothing to append.").display()
else:
if 0 == len(cslines):
self.logger.debug("Empty list passed. Nothing to append.")
try:
if debugMode:
CsReport(159, "Start appending to checksumfile: "+path).display()
self.logger.debug("Start appending to checksumfile: %s", path)
csfile = open(path, 'a')
for csline in cslines:
if debugMode:
CsReport(163, "Writing line: "+str(csline)).display()
self.logger.debug("Writing line: %s", str(csline))
if isinstance(csline, CsLine):
csfile.write(str(csline) + '\n')
else:
......@@ -194,8 +182,7 @@ class CsFile:
raise CsFileError(169, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
+err.filename)
else:
if debugMode:
CsReport(173, "Finished appending to checksumfile: "+path).display()
self.logger.debug("Finished appending to checksumfile: %s", path)
csfile.close()
def update(self, regexes=[]):
......@@ -205,12 +192,10 @@ class CsFile:
"""
if not isinstance(regexes, list):
raise CsFileError(182, "Pass regular expressions in a list.")
if debugMode:
CsReport(184, "Updating checksumfile ...").display()
self.logger.debug("Updating checksumfile ...")
# fetch cslines in current csfile
self.read()
if debugMode:
CsReport(188, "Fetching files not registered yet.").display()
self.logger.debug("Fetching files not registered yet.")
registeredFiles = set(csline.path for csline in self.__cslines)
# fetch files (pathes)
newFiles = os.listdir(self.__filepath)
......@@ -233,33 +218,30 @@ class CsFile:
cslines.append(csline)
self.append(cslines)
def check(self, srcDir='', beTolerant=False):
def check(self, srcDir, beTolerant=False):
"""
Check a checksum file which means:
1. read checksum file
2, calculate checksum of file which is located in srcDir and check results
3. write the result to the checksum file
If the third argument (beTolerant) is set True the checking process will be
successful, too, if a file listed in a checksumfile is not available
anymore.
If the third argument (beTolerant) is set to True the checking process will
be successful, too, if a file listed in a checksumfile is not available
anymore. Then the status of the checksum line will be set to 'warning'.
Note that this function does not perform a check if there unregistered files
in the directory. Adding checksum lines to the checksumfile is done by the
update function.
"""
if debugMode:
CsReport(220, "Start checking checksums.").display()
self.logger.debug("Start checking checksums.")
self.read()
if 0 == len(self.__cslines) and debugMode:
CsReport(223, "CSFILE does not contain any lines.").display()
if 0 == len(self.__cslines):
self.logger.debug("CSFILE does not contain any lines.")
for csline in self.__cslines:
if not len(srcDir):
if debugMode:
CsReport(227, "Performing check of file with file itself.").display()
csline.check()
else:
filename = csline.path.split(os.sep)[-1]
if debugMode:
CsReport(232, "Performing check of file with source: "+ \
srcDir+os.sep+filename).display()
csline.check(srcDir+os.sep+filename)
filename = csline.path.split(os.sep)[-1]
self.logger.debug("Performing check of file with source: %s", \
os.path.join(srcDir,filename)).display()
csline.check(os.path.join(srcDir,filename), beTolerant)
self.write()
def displayLines(self):
......@@ -276,8 +258,9 @@ class CsLine:
"""
Class to handle a checksum and further data for a registered file.
"""
def __init__(self, *args):
self.logger = logging.LoggerAdapter(logging.getLogger( \
__moduleLoggerName+".CsLine"), __logInfo)
if isinstance(args[0], list):
argList = args[0]
self.checksum = argList[0]
......@@ -305,8 +288,7 @@ class CsLine:
Generate the checksum and establish corresponding data for a file. The
result is a fully configured checksum line.
"""
if debugMode:
CsReport(281, "Calculating checksum for file: "+self.path).display()
self.logger.debug("Calculating checksum for file: %", self.path)
# generate checksum
try:
hashfunc = hashlib.new(self.hashfunc)
......@@ -332,38 +314,38 @@ class CsLine:
self.dateLastCheck = self.creationDateChecksum
self.statusLastCheck = 'ok'
def check(self, src=''):
def check(self, src, beTolerant=False):
"""
Check a checksum line. Either compare the checksum of the line with the
checksum of the file located at src or compare the checksum with the checksum
of the file located at the path of its one checksum line.
Check a checksum line. The checksum of the line will be compared with the
checksum of the file located in src. In case no file had been found in src
and the beTolerant flag had been set to True the status of the line will be
set to 'warning' else to 'error'. If the checksums match the status was set
to 'ok' else to 'error', too.
"""
# calculate checksum
hashfunc = hashlib.new(self.hashfunc)
blockSize = chunkSize * hashfunc.block_size
try:
hashfunc = hashlib.new(self.hashfunc)
blockSize = chunkSize * hashfunc.block_size
if 0 == len(src):
file = open(self.path, 'rb')
file = open(src, 'rb')
except IOError:
if beTolerant:
self.statusLastCheck = 'warning'
self.logger.warning("While checking file %s does not exist", src)
else:
file = open(src, 'rb')
self.statusLastCheck = 'error'
self.logger.error("While checking file %s does not exist", src)
else:
# calculate checksum
data = file.read(blockSize)
while data:
hashfunc.update(data)
data = file.read(blockSize)
checksum = hashfunc.hexdigest()
except IOError as err:
# Maybe better to avoid Exception here and just put the status to notice
raise CsFileError(328, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
+err.filename)
else:
file.close()
# checks
if not len(self.checksum):
raise CsFileError(334, "Caclulation of checksum was not successful.")
if checksum == self.checksum:
self.statusLastCheck = 'ok'
else:
self.statusLastCheck = 'error'
self.logger.critical("File %s has no integrity anymore.", src)
def __str__(self):
"""
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment