Commit 7c36b144 authored by Daniel Armbruster's avatar Daniel Armbruster Committed by thomas.forbriger
Browse files

Implementation and testing of

This is a legacy commit from before 2015-05-18.
It may be incomplete as well as inconsistent.
See COPYING.legacy and README.history for details.
source:tfsoftware/trunk/src/python/csback/csbackgen.py successful.

While testing csbackgen.py the csfile module had been debugged, too.


SVN Path:     http://gpitrsvn.gpi.uni-karlsruhe.de/repos/TFSoftware/trunk
SVN Revision: 4331
SVN UUID:     67feda4a-a26e-11df-9d6e-31afc202ad0c
parent 2b198722
#!/usr/bin/env python
## @file csbackgen.py
# @brief
# @brief Generate checksum files.
#
# -----------------------------------------------------------------------------
#
......@@ -8,7 +8,7 @@
# @author Daniel Armbruster
# \date 11/09/2011
#
# Purpose:
# Purpose: Generate checksum files.
#
# ----
# This file is part of csback.
......@@ -27,13 +27,159 @@
# along with csback. If not, see <http://www.gnu.org/licenses/>.
# ----
#
# Copyright (c) 2011 by Daniel Armbruster
# Copyright (c) 2012 by Daniel Armbruster
#
# REVISIONS and CHANGES
# 11/09/2011 V0.1 Daniel Armbruster
# 02/01/2012 V0.1 Daniel Armbruster
#
# =============================================================================
import getopt
import os
import sys
import re
import csfile
__version__ = "V0.1"
__subversion__ = "$Id$"
__license__ = "GPLv2"
__author__ = "Daniel Armbruster"
__copyright__ = "Copyright (c) 2012 by Daniel Armbruster"
# ----- END OF csbackgen.py -----
\ No newline at end of file
# -----------------------------------------------------------------------------
class Error(Exception):
def __init__(self, msg):
self.msg = msg
def display(self):
sys.stderr.write("csbackgen (ERROR): " + self.msg + "\n")
class Usage(Error):
def display(self):
usage_text = "Version: "+__version__+"\nLicense: "+__license__+ \
"\n"+__subversion__+"\nAuthor: "+__author__+ """
Usage: csbackgen [-v|--verbose] [-e REGEX [-e REGEX [...]]]
[-R|--notrecursive] [-d|--debug] [-f|--followlinks] PATH
or: csbackgen -h|--help\n"""
sys.stderr.write("csbackgen: " + self.msg + "\n")
sys.stderr.write(usage_text)
# -----------------------------------------------------------------------------
def help():
"""
Printing helptext to stdout.
"""
help_text = "Version: "+__version__+"\nLicense: "+__license__+"\n"+ \
__subversion__+"\nAuthor: "+__author__+"""
Usage: csbackgen [-v|--verbose] [-e REGEX [-e REGEX [...]]]
[-R|--notrecursive] [-d|--debug] [-f|--followlinks] PATH
or: csbackgen -h|--help
-------------------------------------------------------------------------------
-v|--verbose Be verbose.
-h|--help Display this help.
-e REGEX While generating a checksumfile exclude files matching
REGEX.
-R|--notrecursive Do not generate checksumfiles for subdirectories.
-d|--debug Debug mode. Be really verbose.
-f|--followlinks Follow symbolic links. Only available if option -R is not
set. Note that this option can lead to infinite recursion.
PATH Path to generate checksumfile for.
-------------------------------------------------------------------------------
csbackgen.py will either generate a checksumfile if still no checksumfile is
available or in case there is an existing checksumfile csbackgen.py will append
the not yet registered files to the current checksumfile. In the latter case
csbackgen.py is working in its update mode.\n"""
sys.stdout.write(help_text)
# -----------------------------------------------------------------------------
def getSubDirectories(path, regexes, followLinks=False):
"""
To generate a list of subdirectories of path using os.walk(). Note that path
itself was not appended to the list.
"""
subDirs = []
try:
# collect subdirectories
for root, dirs, files in os.walk(path, True, None, followLinks):
for dir in dirs:
subDirs.append(os.path.join(root, dir))
# exclude directories matching regexes
if [] != regexes:
subDirs = [dir for dir in subDirs for regex in regexes \
if None == re.match(str(regex),dir)]
except OSError as err:
raise Error("[Errno "+str(err.errno)+"] "+err.strerror+": "+err.filename)
else:
return subDirs
# -----------------------------------------------------------------------------
def main(argv=None):
if argv is None:
argv = sys.argv
try:
try:
opts, args = getopt.getopt(argv[1:], "vhe:Rdf", ["help", "verbose",\
"notrecursive", "debug", "followlinks"])
except getopt.GetoptError as err:
raise Usage(err.msg)
verbose = False
debugMode = False
notRecursive = False
followLinks = False
regexes = []
# collect arguments
for opt, arg in opts:
if opt in ("-v", "--verbose"):
verbose = True
elif opt in ("-h", "--help"):
sys.exit(help())
elif opt in ("-R", "--notrecursive"):
notRecursive = True
elif opt in ("-e"):
regexes.append(arg)
elif opt in ("-d", "--debug"):
verbose = True
debugMode = True
csfile.debugMode = True
elif opt in ("-f", "--followlinks"):
followLinks = True
else:
raise Usage("Unhandled option chosen.")
if 1 == len(args):
inputpath = str(args[0])
else:
raise Usage("Invalid argument.")
if notRecursive:
if verbose or debugMode:
sys.stdout.write("csbackgen.py: Generating checksumfile in: '"+ \
path+"'\n")
csfile.CsFile(inputpath).update(regexes)
else:
if verbose or debugMode:
sys.stdout.write("csbackgen.py: Collecting subdirectories.\n")
directories = [inputpath]
directories.extend(getSubDirectories(inputpath, regexes, \
followLinks))
if verbose or debugMode:
sys.stdout.write( \
"csbackgen.py: Generating checksumfiles in subdirectories.\n")
for dir in directories:
csfile.CsFile(dir).update(regexes)
except Usage as err:
err.display()
return 2
except csfile.CsFileError as err:
err.display()
return 2
else:
return 0
# -----------------------------------------------------------------------------
if __name__ == "__main__":
sys.exit(main())
# ----- END OF csbackgen.py -----
......@@ -54,7 +54,7 @@ __copyright__ = "Copyright (c) 2012 by Daniel Armbruster"
# -----------------------------------------------------------------------------
# global variables
chunkSize = 131072 # 128kB
chunkSize = 1024 * 128 # 128kB
debugMode = False
# -----------------------------------------------------------------------------
......@@ -96,7 +96,7 @@ class CsFile:
try:
csfile = open(path, 'w')
except IOError as err:
raise CsFileError(97, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
raise CsFileError(99, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
+err.filename)
else:
csfile.close()
......@@ -108,19 +108,19 @@ class CsFile:
path = self.__filepath+os.sep+self.__filename
try:
if debugMode:
CsReport(109, "Start reading checksumfile: "+path).display()
CsReport(111, "Start reading checksumfile: "+path).display()
csfile = open(path)
self.__cslines = [CsLine(line.split()) for line in csfile \
if len(line.rstrip()) and line[0] != '#']
except IOError as err:
# Maybe better to create the file here and keep on going.
# Will be managed later during further dev.
raise CsFileError(116, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
raise CsFileError(118, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
+err.filename)
else:
csfile.close()
if debugMode:
CsReport(121, "Finished reading checksumfile: "+path).display()
CsReport(123, "Finished reading checksumfile: "+path).display()
def write(self):
"""
......@@ -129,22 +129,22 @@ class CsFile:
path = self.__filepath+os.sep+self.__filename
try:
if debugMode:
CsReport(130, "Start writing checksumfile: "+path).display()
CsReport(132, "Start writing checksumfile: "+path).display()
csfile = open(path, 'w')
for csline in self.__cslines:
if debugMode:
CsReport(134, "Writing line: "+ str(csline)).display()
CsReport(136, "Writing line: "+ str(csline)).display()
if isinstance(csline, CsLine):
csfile.write(str(csline) + '\n')
else:
raise CsFileError(136, "Argument must be of type CsLine.")
raise CsFileError(140, "Argument must be of type CsLine.")
except IOError as err:
raise CsFileError(138, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
raise CsFileError(142, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
+err.filename)
else:
csfile.close()
if debugMode:
CsReport(145, "Finished writing checksumfile: "+path)
CsReport(147, "Finished writing checksumfile: "+path)
def append(self, cslines):
"""
......@@ -152,25 +152,26 @@ class CsFile:
"""
path = self.__filepath+os.sep+self.__filename
if 0 == len(cslines) and debugMode:
CsReport(153, "Invalid argument. Empty list.").display()
try:
if debugMode:
CsReport(156, "Start appending to checksumfile: "+path).display()
csfile = open(path, 'a')
for csline in cslines:
if debugMode:
CsReport(160, "Writing line: "+str(csline)).display()
if isinstance(csline, CsLine):
csfile.write(str(csline) + '\n')
else:
raise CsFileError(164, "Argument must be of type CsLine.")
except IOError as err:
raise CsFileError(166, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
+err.filename)
CsReport(155, "Empty list passed. Nothing to append.").display()
else:
if debugMode:
CsReport(170, "Finished appending to checksumfile: "+path).display()
csfile.close()
try:
if debugMode:
CsReport(159, "Start appending to checksumfile: "+path).display()
csfile = open(path, 'a')
for csline in cslines:
if debugMode:
CsReport(163, "Writing line: "+str(csline)).display()
if isinstance(csline, CsLine):
csfile.write(str(csline) + '\n')
else:
raise CsFileError(167, "Argument must be of type CsLine.")
except IOError as err:
raise CsFileError(169, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
+err.filename)
else:
if debugMode:
CsReport(173, "Finished appending to checksumfile: "+path).display()
csfile.close()
def update(self, regexes=[]):
"""
......@@ -178,30 +179,31 @@ class CsFile:
checksum file in current directory.
"""
if not isinstance(regexes, list):
raise CsFileError(179, "Pass regular expressions in a list.")
raise CsFileError(182, "Pass regular expressions in a list.")
if debugMode:
CsReport(181, "Updating checksumfile ...").display()
CsReport(184, "Updating checksumfile ...").display()
# fetch cslines in current csfile
self.read()
if debugMode:
CsReport(185, "Fetching files not registered yet.").display()
registeredFiles = [csline.path.split(os.sep)[-1] \
for csline in self.__cslines]
# fetch files excluding regex matches
newFiles = list(set([file for file in os.listdir(self.__filepath) \
if os.path.isfile(file) for regex in regexes \
if None == re.search(str(regex),file)]))
CsReport(188, "Fetching files not registered yet.").display()
registeredFiles = set(csline.path for csline in self.__cslines)
# fetch files (pathes) excluding those matching regexes
regexes.append(self.__filename)
newFiles = set([os.path.join(self.__filepath, file) \
for file in os.listdir(self.__filepath) \
if os.path.isfile(os.path.join(self.__filepath,file)) \
for regex in regexes if None == re.match(regex, file)])
regexes.remove(self.__filename)
# exclude registered files
newFiles = list(set([file for file in newFiles \
for registeredFile in registeredFiles if registeredFile != file]))
newFiles -= registeredFiles
# generate cslines of newFiles
csLines = [CsLine(self.__filepath+os.sep+file, self.__hashfunc) \
for file in newFiles]
for line in csLines:
if debugMode:
CsReport(200, "Calculating checksum of file: "+file).display()
line.generate(chunkSize)
self.append(csLines)
cslines = []
for file in newFiles:
csline = CsLine(file, self.__hashfunc)
csline.generate(chunkSize)
cslines.append(csline)
print(csline)
self.append(cslines)
def check(self, srcDir=''):
"""
......@@ -211,19 +213,19 @@ class CsFile:
3. write the result to the checksum file
"""
if debugMode:
CsReport(212, "Start checking checksums.").display()
CsReport(216, "Start checking checksums.").display()
self.read()
if 0 == len(self.__cslines) and debugMode:
CsReport(215, "CSFILE does not contain any lines.").display()
CsReport(219, "CSFILE does not contain any lines.").display()
for csline in self.__cslines:
if not len(srcDir):
if debugMode:
CsReport(219, "Performing check of file with file itself.").display()
CsReport(223, "Performing check of file with file itself.").display()
csline.check()
else:
filename = csline.path.split(os.sep)[-1]
if debugMode:
CsReport(219, "Performing check of file with source: "+ \
CsReport(228, "Performing check of file with source: "+ \
srcDir+os.sep+filename).display()
csline.check(srcDir+os.sep+filename)
self.write()
......@@ -243,43 +245,48 @@ class CsLine:
Class to handle a checksum and further data for a registered file.
"""
def __init__(self, path, hashfunc='sha256'):
self.checksum = ''
self.path = path
self.hashfunc = hashfunc
self.creationDateFile = ''
self.creationLocationChecksum = ''
self.creationDateChecksum = ''
self.dateLastCheck = ''
self.statusLastCheck = ''
def __init__(self, argList):
if not isinstance(argList, list):
raise CsFileError("Invalid argument.")
self.checksum = argList[0]
self.path = argList[1]
self.hashfunc = argList[2]
self.creationDateFile = argList[3]
self.creationLocationChecksum = argList[4]
self.creationDateChecksum = argList[5]
self.dateLastCheck = argList[6]
self.statusLastCheck = argList[7]
def __init__(self, *args):
if isinstance(args[0], list):
argList = args[0]
self.checksum = argList[0]
self.path = argList[1]
self.hashfunc = argList[2]
self.creationDateFile = argList[3]
self.creationLocationChecksum = argList[4]
self.creationDateChecksum = argList[5]
self.dateLastCheck = argList[6]
self.statusLastCheck = argList[7]
elif isinstance(args[0], str) and isinstance(args[1], str):
self.checksum = ''
self.path = args[0]
self.hashfunc = args[1]
self.creationDateFile = ''
self.creationLocationChecksum = ''
self.creationDateChecksum = ''
self.dateLastCheck = ''
self.statusLastCheck = ''
else:
CsFileError(269, "Invalid argument")
def generate(self, chunkSize):
"""
Generate the checksum and establish corresponding data for a file. The
result is a fully configured checksum line.
"""
if debugMode:
CsReport(277, "Calculating checksum for file: "+self.path).display()
# generate checksum
try:
hashfunc = hashlib.new(self.hashfunc)
blockSize = chunkSize * hashfunc.block_size
with open(self.path, "rb") as file:
for chunk in iter(lambda: file.read(blockSize), ''):
hasfunc.update(chunk)
file = open(self.path, 'rb')
data = file.read(blockSize)
while data:
hashfunc.update(data)
data = file.read(blockSize)
self.checksum = hashfunc.hexdigest()
except IOError as err:
raise CsFileError(280, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
raise CsFileError(289, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
+err.filename)
else:
file.close()
......@@ -304,23 +311,23 @@ class CsLine:
hashfunc = hashlib.new(self.hashfunc)
blockSize = chunkSize * hashfunc.block_size
if 0 == len(src):
with open(self.path, "rb") as file:
for chunk in iter(lambda: file.read(blockSize), ''):
hasfunc.update(chunk)
file = open(self.path, 'rb')
else:
with open(src, "rb") as file:
for chunk in iter(lambda: file.read(blockSize), ''):
hasfunc.update(chunk)
checksum = hashfunc.hexdigest()
file = open(src, 'rb')
data = file.read(blockSize)
while data:
hashfunc.update(data)
data = file.read(blockSize)
checksum = hashfunc.hexdigest()
except IOError as err:
# Maybe better to avoid Exception here and just put the status to notice
raise CsFileError(275, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
raise CsFileError(324, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
+err.filename)
else:
file.close()
# checks
if not len(self.checksum):
raise CsFileError(280, "Caclulation of checksum was not successful.")
raise CsFileError(330, "Caclulation of checksum was not successful.")
if checksum == self.checksum:
self.statusLastCheck = 'ok'
else:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment