Commit aec1d4b8 authored by Daniel Armbruster's avatar Daniel Armbruster Committed by thomas.forbriger
Browse files

[source:tfsoftware/trunk/src/python/csback/csbackchk.py] first time running.

This is a legacy commit from before 2015-05-18.
It may be incomplete as well as inconsistent.
See COPYING.legacy and README.history for details.

Not tested all cases yet.


SVN Path:     http://gpitrsvn.gpi.uni-karlsruhe.de/repos/TFSoftware/trunk
SVN Revision: 4340
SVN UUID:     67feda4a-a26e-11df-9d6e-31afc202ad0c
parent 4c69b099
......@@ -51,11 +51,13 @@ __copyright__ = "Copyright (c) 2012 by Daniel Armbruster"
# -----------------------------------------------------------------------------
class Error(Exception):
def __init__(self, msg):
def __init__(self, line, msg):
self.line = str(line)
self.msg = msg
def display(self):
sys.stderr.write("csbackchk (ERROR): " + self.msg + "\n")
sys.stderr.write("triggered in line: " + self.line + "\n")
class Usage(Error):
......@@ -67,7 +69,7 @@ class Usage(Error):
[-R|--notrecursive] [-d|--debug] [-f|--followlinks]
[-t|--tolerant] [SOURCEPATH] PATH
or: csbackchk -h|--help\n"""
sys.stderr.write("csbackchk: " + self.msg + "\n")
sys.stderr.write("csbackchk [line "+self.line+"]: "+self.msg+"\n")
sys.stderr.write(usage_text)
# -----------------------------------------------------------------------------
......@@ -105,14 +107,16 @@ def help():
PATH Path to perform check for.
"""
sys.stdout.write(help_text)
# -----------------------------------------------------------------------------
def main(argv=None):
# configure logger
logInfo = {'hostname': os.uname()[1], \
'user': pwd.getpwuid(os.getuid()).pw_name, \
'pid': os.getpid()}
csfile.csfileLogInfo = logInfo
csfile.csfileLoggerName = 'csbackgen.csfile'
log = logging.getLogger('csbackgen')
csfile.csfileLoggerName = 'csbackchk.csfile'
log = logging.getLogger('csbackchk')
logger = logging.LoggerAdapter(log, logInfo)
logging.basicConfig(level=logging.DEBUG, \
format='%(asctime)-15s %(hostname)-6s %(name)-8s[%(pid)s]: \
......@@ -122,6 +126,7 @@ def main(argv=None):
'%(name)-8s [%(lineno)d]: %(levelname)-8s %(message)s')
console.setFormatter(formatter)
# fetch arguments
if argv is None:
argv = sys.argv
try:
......@@ -129,7 +134,7 @@ def main(argv=None):
opts, args = getopt.getopt(argv[1:], "vhe:Rdft", ["help", "verbose", \
"notrecursive", "debug", "followlinks", "tolerant"])
except getopt.GetoptError as err:
raise Usage(err.msg)
raise Usage(137,err.msg)
verbose = False
debugMode = False
notRecursive = False
......@@ -156,60 +161,51 @@ def main(argv=None):
elif opt in ("-t", "--tolerant"):
beTolerant = True
else:
raise Usage("Unhandled option chosen.")
raise Usage(164,msg="Unhandled option chosen.")
if verbose or debugMode:
logger.addHandler(console)
log.addHandler(console)
# fetch arguments
if 1 == len(args):
sourcepath = str(args[0]).rstrip(os.sep)+os.sep
inputpath = sourcepath
inputDirectories = [inputpath]
sourceDirectories = inputDirectories
elif 2 == len(args):
sourcepath = str(args[0]).rstrip(os.sep)+os.sep
inputpath = str(args[1]).rstrip(os.sep)+os.sep
inputDirectories = [inputpath]
sourceDirectories = [sourcepath]
else:
raise Usage("Invalid argument(s).")
if not notRecursive:
logger.info("Collecting subdirectories for inputpath: %s",inputpath)
inputDirectories.extend(csfile.getSubDirectories(inputpath, regexes, \
followLinks))
# case: different source directory chosen
if inputDirectories[0] != sourceDirectories[0]:
logger.info("Collecting subdirectories for sourcepath: %s",sourcepath)
sourceDirectories.extend(csfile.getSubDirectories(sourcepath, \
regexes, followLinks))
else:
logger.info("No recursion to subdirectories.")
# check directory structure of both sourcepath and inputpath
if inputDirectories[0] != sourceDirectories[0]:
logger.info("Checking directory structure.")
inTmp = [dir.rstrip(inputDirectories[0]) for dir in inputDirectories[1:]]
srcTmp = [dir.rstrip(sourceDirectories[0]) \
for dir in sourceDirectories[1:]]
if inTmp != srcTmp:
logger.error("Source directory does not contain the same directory \
structure like the input directory.")
directories = list(zip(inputDirectories, sourceDirectories))
for dirpair in directories:
csfile.CsFile(dirpair[0]).check(regexes, dirpair[1], beTolerant)
raise Usage(176,"Invalid argument(s).")
# major part
if not csfile.hasCsFile(inputpath):
raise Error(180,"PATH does not contain a checksumfile.")
if notRecursive:
logger.debug("Fetching subdirectories for regexes in %s", inputpath)
subdirs = csfile.getSubDirectories(inputpath, regexes, followLinks)
subdirs = [subdir.strip(inputpath)+"*" for subdir in subdirs]
regexes.extend(subdirs)
print(regexes)
checksumfile = csfile.CsFile(inputpath, sourcepath)
checksumfile.read()
checksumfile.check(regexes, beTolerant)
checksumfile.write()
except Usage as err:
err.display()
return 2
except Error as err:
logger.error("csbackchk.Error occured while checking files.")
err.display()
return 2
except csfile.CsFileError as err:
logger.error("csfile.CsFileError occured while checking files.")
err.display()
return 2
else:
logger.info("Checks performed.")
return 0
# -----------------------------------------------------------------------------
if __name__ == "__main__":
sys.exit(main())
......
......@@ -173,8 +173,8 @@ def main(argv=None):
directories.extend(csfile.getSubDirectories(inputpath, regexes, \
followLinks))
for dir in directories:
logger.info("Update files in directory: %s to checksumfile: %s.cs", dir, \
targetDirectory)
logger.info("Update files in directory: %s to checksumfile: %s%s", dir, \
targetDirectory, csfile.CsFile.filename)
csfile.CsFile(targetDirectory, srcpath=dir).update(regexes)
except Usage as err:
......
......@@ -72,7 +72,7 @@ def getSubDirectories(path, regexes, followLinks=False):
subDirs = set()
try:
# collect subdirectories
for root, dirs, files in os.walk(path, True, None, followLinks):
for root, dirs, files in os.walk(path, followlinks=followLinks):
for dir in dirs:
subDirs.add(os.path.join(root, dir))
# exclude directories matching regexes
......@@ -85,6 +85,13 @@ def getSubDirectories(path, regexes, followLinks=False):
else:
return subDirs
def hasCsFile(path):
"""
Checks if path contains a checksumfile. Returns True if path contains a file
named with an CsFile filename.
"""
return os.path.isfile(os.path.join(path, CsFile.filename))
# -----------------------------------------------------------------------------
class CsFileError(Exception):
"""
......@@ -103,11 +110,13 @@ class CsFile:
"""
Provides an interface to handle a csback checksumfile. A checksumfile
possesses the ability to take the files for generating the checksums from a
different sourcdirectory which can be configured with the srcpath variable.
different sourcedirectory which can be configured with the srcpath variable.
A checksumfiles usually contains checksumlines (type CsLine) of files.
Generally this includes the files of the subdirectories in srcpath, too.
"""
def __init__(self, filedir, srcpath, hashfunc='sha256'):
self.filedir = filedir
self.filename = ".cs"
self.__cslines = []
self.__hashfunc = hashfunc
self.srcpath = srcpath
......@@ -120,10 +129,10 @@ class CsFile:
"""
if not os.access(self.filedir, os.F_OK):
raise CsFileError(122, "Invalid directory path.")
path = os.path.join(self.filedir, self.filename)
path = os.path.join(self.filedir, CsFile.filename)
# no checksumfile available -> create new file
if os.access(self.filedir, os.F_OK) and not os.path.isfile(path):
self.logger.info("Creating checksumfile in %s", self.filedir)
self.logger.info("Creating checksumfile in '%s'", self.filedir)
try:
csfile = open(path, 'w')
except IOError as err:
......@@ -134,7 +143,7 @@ class CsFile:
# checksumfile available -> read file
else:
try:
self.logger.debug("Start reading checksumfile %s",path)
self.logger.debug("Start reading checksumfile '%s'",path)
csfile = open(path)
self.__cslines = [CsLine(line.split()) for line in csfile \
if len(line.rstrip()) and line[0] != '#']
......@@ -143,18 +152,18 @@ class CsFile:
+err.filename)
else:
csfile.close()
self.logger.debug("Finished reading checksumfile: %s", path)
self.logger.debug("Finished reading checksumfile '%s'", path)
def write(self):
"""
Write the entire checksumfile.
"""
path = os.path.join(self.filedir, self.filename)
path = os.path.join(self.filedir, CsFile.filename)
try:
self.logger.debug("Start writing checksumfile: %s",path)
self.logger.debug("Start writing checksumfile '%s'",path)
csfile = open(path, 'w')
for csline in self.__cslines:
self.logger.debug("Writing line: %s",str(csline))
self.logger.debug("Writing line: '%s'",str(csline))
if isinstance(csline, CsLine):
csfile.write(str(csline) + '\n')
else:
......@@ -164,20 +173,20 @@ class CsFile:
+err.filename)
else:
csfile.close()
self.logger.debug("Finished writing checksumfile: %s", path)
self.logger.debug("Finished writing checksumfile '%s'", path)
def append(self, cslines):
"""
Append checksum lines to the checksumfile.
"""
path = os.path.join(self.filedir, self.filename)
path = os.path.join(self.filedir, CsFile.filename)
if 0 == len(cslines):
self.logger.debug("Empty list passed. Nothing to append.")
try:
self.logger.debug("Start appending to checksumfile: %s", path)
self.logger.debug("Start appending to checksumfile '%s'", path)
csfile = open(path, 'a')
for csline in cslines:
self.logger.debug("Writing line: %s", str(csline))
self.logger.debug("Writing line '%s'", str(csline))
if isinstance(csline, CsLine):
csfile.write(str(csline) + '\n')
else:
......@@ -186,7 +195,7 @@ class CsFile:
raise CsFileError(186, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
+err.filename)
else:
self.logger.debug("Finished appending to checksumfile: %s", path)
self.logger.debug("Finished appending to checksumfile '%s'", path)
csfile.close()
def update(self, regexes=[]):
......@@ -206,12 +215,12 @@ class CsFile:
newFiles = set(os.path.join(self.srcpath, file) for file in newFiles \
if os.path.isfile(os.path.join(self.srcpath, file)))
# exclude files matching regexes
regexes.append(os.path.join(self.srcpath,self.filename))
regexes.append(os.path.join(self.srcpath,CsFile.filename))
for regex in regexes:
matching = set(file for file in newFiles \
if None != re.match(regex, file))
newFiles -= matching
regexes.remove(os.path.join(self.srcpath,self.filename))
regexes.remove(os.path.join(self.srcpath,CsFile.filename))
# exclude registered files
newFiles -= registeredFiles
# generate cslines of newFiles
......@@ -221,44 +230,57 @@ class CsFile:
csline.generate(chunkSize)
cslines.append(csline)
self.append(cslines)
path = os.path.join(self.filedir, self.filename)
self.logger.debug("Update of checksumfile: %s finished.", path)
path = os.path.join(self.filedir, CsFile.filename)
self.logger.debug("Update of checksumfile '%s' finished.", path)
def check(self, srcDir, beTolerant=False):
def check(self, regexes, beTolerant=False):
"""
Check a checksum file which means:
1. read checksum file
2, calculate checksum of file which is located in srcDir and check results
3. write the result to the checksum file
Calculate checksum of a file which is located in self.srcpath and check
results. Files in the checksumfile matching one of the regexes are excluded.
If the third argument (beTolerant) is set to True the checking process will
be successful, too, if a file listed in a checksumfile is not available
anymore. Then the status of the checksum line will be set to 'warning'.
Note that this function does not perform a check if there are unregistered
files in the directory. Adding checksum lines to the checksumfile has to be
done by the update function.
Note that this function does not check if there are unregistered files in
the directory. Adding checksum lines to the checksumfile has to be done by
using the update function.
"""
self.logger.debug("Start checking checksums.")
self.read()
if 0 == len(self.__cslines):
self.logger.debug("CSFILE does not contain any lines.")
self.logger.debug( \
"CSFILE does not contain any lines or had not been read yet.")
# exclude those files matching regex in regexes
self.logger.debug("Exclude files matching regexes")
cslinesSet = set(self.__cslines)
for regex in regexes:
matching = set(csline for csline in self.__cslines \
if None != re.match(regex, csline.path))
cslinesSet -= matching
# perform check
self.logger.debug("Start checking checksums ...")
for csline in self.__cslines:
filename = csline.path.split(os.sep)[-1]
self.logger.debug("Performing check of file with source: %s", \
os.path.join(srcDir,filename)).display()
csline.check(os.path.join(srcDir,filename), beTolerant)
self.write()
if csline in cslinesSet:
path = csline.path.replace(self.filedir, self.srcpath)
self.logger.debug( \
"Performing check of csline-file '%s' with file '%s'", csline.path, \
path)
csline.check(path, beTolerant)
self.logger.debug("Finished checking checksums.")
def displayLines(self):
def display(self):
"""
Display the content of the checksum file at stdout.
Display the content of the checksum file to stdout.
"""
if not len(self.__cslines):
raise CsFileError(258, "CSFILE does not contain any lines.")
if 0 == len(self.__cslines):
self.logger.info("CSFILE does not contain any lines.")
for line in self.__cslines:
sys.stdout.write(line)
filename = ".cs"
# -----------------------------------------------------------------------------
class CsLine:
"""
......@@ -294,7 +316,7 @@ class CsLine:
Generate the checksum and establish corresponding data for a file. The
result is a fully configured checksum line.
"""
self.logger.debug("Calculating checksum for file: %s", self.path)
self.logger.debug("Calculating checksum for '%s'", self.path)
# generate checksum
try:
hashfunc = hashlib.new(self.hashfunc)
......@@ -335,10 +357,10 @@ class CsLine:
except IOError:
if beTolerant:
self.statusLastCheck = 'warning'
self.logger.warning("While checking file %s does not exist", src)
self.logger.warning("While checking file '%s' does not exist", src)
else:
self.statusLastCheck = 'error'
self.logger.error("While checking file %s does not exist", src)
self.logger.error("While checking file '%s' does not exist", src)
else:
# calculate checksum
data = file.read(blockSize)
......@@ -350,9 +372,10 @@ class CsLine:
# checks
if checksum == self.checksum:
self.statusLastCheck = 'ok'
self.logger.debug("Check of file: %s was successful.", src)
self.logger.debug("Check of file '%s' was successful.", src)
else:
self.logger.critical("File %s has no integrity anymore.", src)
self.statusLastCheck = 'error'
self.logger.critical("File '%s' has no integrity anymore.", src)
def __str__(self):
"""
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment