Commit 91e90641 authored by Daniel Armbruster's avatar Daniel Armbruster Committed by thomas.forbriger
Browse files

create checksumfile in arbitrary destination, logging does work now

This is a legacy commit from before 2015-05-18.
It may be incomplete as well as inconsistent.
See COPYING.legacy and README.history for details.

Up to now logging only is piped to stdout. Besides no tool uses the pidlock
mechanism yet.


SVN Path:     http://gpitrsvn.gpi.uni-karlsruhe.de/repos/TFSoftware/trunk
SVN Revision: 4338
SVN UUID:     67feda4a-a26e-11df-9d6e-31afc202ad0c
parent a4d16cec
......@@ -38,6 +38,7 @@ import getopt
import os
import sys
import re
import pwd
import logging
import csfile
......@@ -63,7 +64,8 @@ class Usage(Error):
usage_text = "Version: "+__version__+"\nLicense: "+__license__+ \
"\n"+__subversion__+"\nAuthor: "+__author__+ """
Usage: csbackgen [-v|--verbose] [-e REGEX [-e REGEX [...]]]
[-R|--notrecursive] [-d|--debug] [-f|--followlinks] PATH
[-R|--notrecursive] [-d|--debug] [-f|--followlinks]
[-t|--target DIR] PATH
or: csbackgen -h|--help\n"""
sys.stderr.write("csbackgen: " + self.msg + "\n")
sys.stderr.write(usage_text)
......@@ -76,7 +78,8 @@ def help():
help_text = "Version: "+__version__+"\nLicense: "+__license__+"\n"+ \
__subversion__+"\nAuthor: "+__author__+"""
Usage: csbackgen [-v|--verbose] [-e REGEX [-e REGEX [...]]]
[-R|--notrecursive] [-d|--debug] [-f|--followlinks] PATH
[-R|--notrecursive] [-d|--debug] [-f|--followlinks]
[-t|--target DIR] PATH
or: csbackgen -h|--help
-------------------------------------------------------------------------------
-v|--verbose Be verbose.
......@@ -87,6 +90,7 @@ def help():
-d|--debug Debug mode. Be really verbose.
-f|--followlinks Follow symbolic links. Only available if option -R is not
set. Note that this option can lead to infinite recursion.
-t|--target Target directory for checksumfile.
PATH Path to generate checksumfile for.
-------------------------------------------------------------------------------
csbackgen.py will either generate a checksumfile if still no checksumfile is
......@@ -99,25 +103,26 @@ csbackgen.py is working in its update mode.\n"""
def main(argv=None):
# configure logger
logInfo = {'hostname': os.uname()[1], \
'user': str(pwd.getpwuid(os.getuid()).pw_name) \
'user': pwd.getpwuid(os.getuid()).pw_name, \
'pid': os.getpid()}
csfile.__logInfo = logInfo
csfile.moduleLoggerName = 'csbackgen.csfile'
logger = logging.LoggerAdapter(logging.getLogger('csbackgen'), logInfo)
csfile.csfileLogInfo = logInfo
csfile.csfileLoggerName = 'csbackgen.csfile'
log = logging.getLogger('csbackgen')
logger = logging.LoggerAdapter(log, logInfo)
logging.basicConfig(level=logging.DEBUG, \
format='%(asctime)-15s %(hostname)-10s %(name)-15s[%(pid)-8s]: \
(%(user)-8s) %(levelname)-8s %(message)s')
format='%(asctime)-15s %(hostname)-6s %(name)-8s[%(pid)s]: \
(%(user)-4s) %(levelname)-8s %(message)s')
console = logging.StreamHandler()
formatter = logging.Formatter( \
'%(name)-15s [%(lineno)d]: %(levelname)-15s %(message)s')
'%(name)-8s [%(lineno)d]: %(levelname)-8s %(message)s')
console.setFormatter(formatter)
if argv is None:
argv = sys.argv
try:
try:
opts, args = getopt.getopt(argv[1:], "vhe:Rdf", ["help", "verbose", \
"notrecursive", "debug", "followlinks"])
opts, args = getopt.getopt(argv[1:], "vhe:Rdft:", ["help", "verbose", \
"notrecursive", "debug", "followlinks", "target="])
except getopt.GetoptError as err:
raise Usage(err.msg)
verbose = False
......@@ -125,6 +130,7 @@ def main(argv=None):
notRecursive = False
followLinks = False
regexes = []
targetSet = False
for opt, arg in opts:
if opt in ("-v", "--verbose"):
verbose = True
......@@ -140,11 +146,14 @@ def main(argv=None):
console.setLevel(logging.DEBUG)
elif opt in ("-f", "--followlinks"):
followLinks = True
elif opt in ("-t", "--target"):
targetSet = True
targetDirectory = str(arg)
else:
raise Usage("Unhandled option chosen.")
if verbose or debugMode:
logger.addHandler(console)
log.addHandler(console)
# fetch arguments
if 1 == len(args):
......@@ -152,9 +161,15 @@ def main(argv=None):
else:
raise Usage("Invalid argument.")
if not targetSet:
targetDirectory = inputpath
if not os.path.isdir(targetDirectory):
raise Usage("Target directory not a valid directory.")
# major part
if notRecursive:
logger.info("Generating checksumfile in: %s", inputpath)
csfile.CsFile(inputpath).update(regexes)
csfile.CsFile(targetDirectory, srcpath=inputpath).update(regexes)
else:
logger.info("Collecting subdirectories.")
directories = [inputpath]
......@@ -162,7 +177,7 @@ def main(argv=None):
followLinks))
logger.info("Generating checksumfiles in subdirectories.")
for dir in directories:
csfile.CsFile(dir).update(regexes)
csfile.CsFile(targetDirectory, srcpath=dir).update(regexes)
except Usage as err:
err.display()
......
......@@ -29,12 +29,12 @@
# along with csback. If not, see <http://www.gnu.org/licenses/>.
# ----
#
# Copyright (c) 2011 by Daniel Armbruster
# Copyright (c) 2011-2012 by Daniel Armbruster
#
# REVISIONS and CHANGES
# 15/09/2011 V0.1 Daniel Armbruster
# 01/01/2012 V0.1.1 finished implementation
# 02/01/2012 V0.1.2 implemented debugMode
# 04/01/2012 V0.1.3 implemented a debug mode using the python logging module
#
# =============================================================================
""" CsFile module to handle checksum files. """
......@@ -42,6 +42,7 @@
import os
import re
import sys
import pwd
import hashlib
import logging
from datetime import datetime
......@@ -57,8 +58,8 @@ __copyright__ = "Copyright (c) 2012 by Daniel Armbruster"
chunkSize = 1024 * 128 # 128kB
__moduleLoggerName = ''
__logInfo = {}
csfileLoggerName = ''
csfileLogInfo = {}
# -----------------------------------------------------------------------------
# functions
......@@ -79,7 +80,7 @@ def getSubDirectories(path, regexes, followLinks=False):
matching = set(dir for dir in subDirs if None != re.match(regex, dir))
subDirs -= matching
except OSError as err:
raise CsFileError("[Errno "+str(err.errno)+"] "+err.strerror+": " \
raise CsFileError(83, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
+err.filename)
else:
return subDirs
......@@ -100,52 +101,55 @@ class CsFileError(Exception):
# -----------------------------------------------------------------------------
class CsFile:
"""
Provides an interface to handle a csback checksum file.
Provides an interface to handle a csback checksumfile. A checksumfile
possesses the ability to take the files generating the checksums for from a
different sourcdirectory which can be configured with the srcpath variable.
"""
def __init__(self, filepath, hashfunc='sha256'):
self.__filepath = filepath
self.__filename = ".cs"
def __init__(self, filedir, srcpath, hashfunc='sha256'):
self.filedir = filedir
self.filename = ".cs"
self.__cslines = []
self.__hashfunc = hashfunc
self.srcpath = srcpath
self.logger = logging.LoggerAdapter(logging.getLogger( \
__moduleLoggerName+".CsFile"), __logInfo)
self.createFile()
csfileLoggerName+".CsFile"), csfileLogInfo)
def createFile(self):
path = self.__filepath+os.sep+self.__filename
if not os.path.isfile(path):
def read(self):
"""
Read a checksum file.
"""
if not os.access(self.filedir, os.F_OK):
raise CsFileError(122, "Invalid directory path.")
path = os.path.join(self.filedir, self.filename)
# no checksumfile available -> create new file
if os.access(self.filedir, os.F_OK) and not os.path.isfile(path):
self.logger.info("Creating checksumfile in %s", self.filedir)
try:
csfile = open(path, 'w')
except IOError as err:
raise CsFileError(99, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
raise CsFileError(130, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
+err.filename)
else:
csfile.close()
def read(self):
"""
Read a checksum file.
"""
path = self.__filepath+os.sep+self.__filename
try:
self.logger.debug("Start reading checksumfile %s",path)
csfile = open(path)
self.__cslines = [CsLine(line.split()) for line in csfile \
if len(line.rstrip()) and line[0] != '#']
except IOError as err:
# Maybe better to create the file here and keep on going.
# Will be managed later during further dev.
raise CsFileError(118, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
+err.filename)
# checksumfile available -> read file
else:
csfile.close()
self.logger.debug("Finished reading checksumfile: %s", path)
try:
self.logger.debug("Start reading checksumfile %s",path)
csfile = open(path)
self.__cslines = [CsLine(line.split()) for line in csfile \
if len(line.rstrip()) and line[0] != '#']
except IOError as err:
raise CsFileError(142, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
+err.filename)
else:
csfile.close()
self.logger.debug("Finished reading checksumfile: %s", path)
def write(self):
"""
Write the entire checksumfile.
"""
path = self.__filepath+os.sep+self.__filename
path = os.path.join(self.filedir, self.filename)
try:
self.logger.debug("Start writing checksumfile: %s",path)
csfile = open(path, 'w')
......@@ -154,9 +158,9 @@ class CsFile:
if isinstance(csline, CsLine):
csfile.write(str(csline) + '\n')
else:
raise CsFileError(140, "Argument must be of type CsLine.")
raise CsFileError(161, "Argument must be of type CsLine.")
except IOError as err:
raise CsFileError(142, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
raise CsFileError(163, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
+err.filename)
else:
csfile.close()
......@@ -164,50 +168,50 @@ class CsFile:
def append(self, cslines):
"""
Append checksum lines to the checksum file.
Append checksum lines to the checksumfile.
"""
path = self.__filepath+os.sep+self.__filename
path = os.path.join(self.filedir, self.filename)
if 0 == len(cslines):
self.logger.debug("Empty list passed. Nothing to append.")
try:
self.logger.debug("Start appending to checksumfile: %s", path)
csfile = open(path, 'a')
for csline in cslines:
self.logger.debug("Writing line: %s", str(csline))
if isinstance(csline, CsLine):
csfile.write(str(csline) + '\n')
else:
raise CsFileError(167, "Argument must be of type CsLine.")
except IOError as err:
raise CsFileError(169, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
+err.filename)
else:
self.logger.debug("Finished appending to checksumfile: %s", path)
csfile.close()
try:
self.logger.debug("Start appending to checksumfile: %s", path)
csfile = open(path, 'a')
for csline in cslines:
self.logger.debug("Writing line: %s", str(csline))
if isinstance(csline, CsLine):
csfile.write(str(csline) + '\n')
else:
raise CsFileError(184, "Argument must be of type CsLine.")
except IOError as err:
raise CsFileError(186, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
+err.filename)
else:
self.logger.debug("Finished appending to checksumfile: %s", path)
csfile.close()
def update(self, regexes=[]):
"""
Update a checksum file. Also includes appending of not registered files to
checksum file in current directory.
"""
if not isinstance(regexes, list):
raise CsFileError(182, "Pass regular expressions in a list.")
raise CsFileError(198, "Pass regular expressions in a list.")
self.logger.debug("Updating checksumfile ...")
# fetch cslines in current csfile
self.read()
self.logger.debug("Fetching files not registered yet.")
registeredFiles = set(csline.path for csline in self.__cslines)
# fetch files (pathes)
newFiles = os.listdir(self.__filepath)
newFiles = set(os.path.join(self.__filepath, file) for file in newFiles \
if os.path.isfile(os.path.join(self.__filepath, file)))
newFiles = os.listdir(self.srcpath)
newFiles = set(os.path.join(self.srcpath, file) for file in newFiles \
if os.path.isfile(os.path.join(self.srcpath, file)))
# exclude files matching regexes
regexes.append(os.path.join(self.__filepath,self.__filename))
regexes.append(os.path.join(self.srcpath,self.filename))
for regex in regexes:
matching = set(file for file in newFiles \
if None != re.match(regex, file))
newFiles -= matching
regexes.remove(os.path.join(self.__filepath,self.__filename))
regexes.remove(os.path.join(self.srcpath,self.filename))
# exclude registered files
newFiles -= registeredFiles
# generate cslines of newFiles
......@@ -217,6 +221,8 @@ class CsFile:
csline.generate(chunkSize)
cslines.append(csline)
self.append(cslines)
path = os.path.join(self.filedir, self.filename)
self.logger.debug("Update of checksumfile: %s finished.", path)
def check(self, srcDir, beTolerant=False):
"""
......@@ -249,7 +255,7 @@ class CsFile:
Display the content of the checksum file at stdout.
"""
if not len(self.__cslines):
raise CsFileError("CSFILE does not contain any lines.", 193)
raise CsFileError(258, "CSFILE does not contain any lines.")
for line in self.__cslines:
sys.stdout.write(line)
......@@ -260,7 +266,7 @@ class CsLine:
"""
def __init__(self, *args):
self.logger = logging.LoggerAdapter(logging.getLogger( \
__moduleLoggerName+".CsLine"), __logInfo)
csfileLoggerName+".CsLine"), csfileLogInfo)
if isinstance(args[0], list):
argList = args[0]
self.checksum = argList[0]
......@@ -281,14 +287,14 @@ class CsLine:
self.dateLastCheck = ''
self.statusLastCheck = ''
else:
CsFileError(273, "Invalid argument")
CsFileError(290, "Invalid argument(s).")
def generate(self, chunkSize):
"""
Generate the checksum and establish corresponding data for a file. The
result is a fully configured checksum line.
"""
self.logger.debug("Calculating checksum for file: %", self.path)
self.logger.debug("Calculating checksum for file: %s", self.path)
# generate checksum
try:
hashfunc = hashlib.new(self.hashfunc)
......@@ -300,7 +306,7 @@ class CsLine:
data = file.read(blockSize)
self.checksum = hashfunc.hexdigest()
except IOError as err:
raise CsFileError(293, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
raise CsFileError(309, "[Errno "+str(err.errno)+"] "+err.strerror+": " \
+err.filename)
else:
file.close()
......@@ -359,12 +365,10 @@ class CsLine:
# Tests
if __name__ == '__main__':
try:
debugMode = True
file = CsFile('/home/daniel/')
file.read()
except CsFileError as err:
err.display()
sys.exit()
# ----- END OF csfile.py -----
......@@ -157,10 +157,10 @@ if __name__ == '__main__':
pidhandler = PidHandler()
# announce pid of popen 1
popen1 = subprocess.Popen('echo "Process 1"; exit 0', shell=True)
pidhandler.cancel(popen1.pid)
popen1.wait()
print(popen1.pid)
pidhandler.announce(popen1.pid)
pidhandler.cancel(popen1.pid)
popen1.wait()
# announce another pid
popen2 = subprocess.Popen('echo "Process 2"; exit 0', shell=True)
popen2.wait()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment