Commit f91743a7 authored by Daniel Armbruster's avatar Daniel Armbruster Committed by thomas.forbriger
Browse files

merge feature branch back into trunk

This is a legacy commit from before 2015-05-18.
It may be incomplete as well as inconsistent.
See COPYING.legacy and README.history for details.


SVN Path:     http://gpitrsvn.gpi.uni-karlsruhe.de/repos/TFSoftware/trunk
SVN Revision: 5047
SVN UUID:     67feda4a-a26e-11df-9d6e-31afc202ad0c
parent 72cd9b68
......@@ -33,7 +33,7 @@ TUN_INTERFACE=tun0
# approx. 80% of 256 kBit/second
BWLIMIT=32
EXCLUDE="--exclude='.lock'"
BFO_BASEDIR=/data/BFO
BFO_BASEDIR=/media/BFO_archive
LSDF_BASEDIR=\
gpi-seis@gpilsdf.gpi.uni-karlsruhe.de:/gpfs/lsdf/gpi/GPI/SEIS/projects/BFO
# NOTE: If adding an additional data directory an exclude pattern must be added,
......@@ -98,7 +98,7 @@ source $HOME/.keychain/$HOSTNAME-sh
for i in ${!DATADIRS[*]}
do
# copy data (compressed)
${RSYNC} -q -z -a --protocol=29 --bwlimit ${BWLIMIT} ${EXCLUDE} \
${RSYNC} -nv -z -a --protocol=29 --bwlimit ${BWLIMIT} ${EXCLUDE} \
${EXCLUDE_PATTERNS[$i]} ${BFO_BASEDIR}/${DATADIRS[$i]} \
${LSDF_BASEDIR}/${DATADIRS[$i]}
......
......@@ -24,10 +24,14 @@
# 15/01/2012 V0.1 Daniel Armbruster
# 16/01/2012 V0.2 added package rule
# 28/02/2012 V0.3 smaller modifications - added csbackobs.py
# 27/04/2012 V0.4 added lsYears.py
# 27/04/2012 V0.4 added lsYears.py - removed with V0.5 (deprecated)
# 13/02/2013 V0.5 added csbackresc.py and ssh/scp specific checks
# 14/02/2013 V0.6 added csclean.sh
# 28/02/2013 V0.7 added csbackscp.py
#
# ----------------------------------------------------------------------------
PROGRAMS=csback2cron.py csbackntfy.py csbackgen.py csbackchk.py csbackobs.py \
lsYears.py
csbackresc.py csbackscp.py csclean.sh
.PHONY: all
all: install doc
......@@ -45,11 +49,14 @@ CRON=$(shell ps -ef | grep -v grep | grep -cw cron)
SYSLOGNG=$(shell ps -ef | grep -v grep | grep -cw syslog)
LOGROTATE=$(shell if [ -d /etc/logrotate.d/ ]; then echo logrotate; fi)
PYTHON=$(shell env which python)
SSH=$(shell env which ssh)
SCP=$(shell env which scp)
# ----------------------------------------------------------------------------
.PHONY: clean
clean:
/bin/rm -rvf $(addprefix $(LOCBINDIR)/,$(patsubst %.py,%,$(PROGRAMS)))
/bin/rm -rvf $(addprefix $(LOCBINDIR)/,$(patsubst %.sh,%,$(PROGRAMS)))
$(MAKE) -C doc clean
# ----------------------------------------------------------------------------
......@@ -60,7 +67,7 @@ CHECKPROG=$(if $($(1)),,$(error ERROR: missing program $(1)))
CHECKPROGS=$(foreach prog,$(1),$(call CHECKPROG,$(prog)))
$(call CHECKVARS, LOCBINDIR)
$(call CHECKPROGS, RSYNC CRON SYSLOGNG LOGROTATE PYTHON)
$(call CHECKPROGS, RSYNC CRON SYSLOGNG LOGROTATE PYTHON SSH SCP)
# ----------------------------------------------------------------------------
PWD=$(shell env pwd)
......@@ -72,7 +79,13 @@ $(LOCBINDIR)/%: %.py
ln -fs $(addprefix $(PWD)/,$<) $@
chmod +x $(addprefix $(PWD)/,$<)
installscripts: $(addprefix $(LOCBINDIR)/, $(patsubst %.py,%,$(PROGRAMS)))
$(LOCBINDIR)/%: %.sh
mkdir -pv $(LOCBINDIR)
ln -fs $(addprefix $(PWD)/,$<) $@
chmod +x $(addprefix $(PWD)/,$<)
installscripts: $(addprefix $(LOCBINDIR)/, $(patsubst %.py,%,$(PROGRAMS)) \
$(patsubst %.sh,%,$(PROGRAMS)))
# ----------------------------------------------------------------------------
pdfdoc:
......
this is <README>
===============================================================================
csback - A toolkit written in python to manage backups
csback - A toolkit written in python to manage backups.
--------------------------
$Id$
===============================================================================
......@@ -39,7 +39,7 @@ documentation which comes along with this package.
Documentation:
----
This package comes along with a pdf documentation which is located in
doc/manuals.pdf.
doc/manual.pdf.
----- END OF README -----
......@@ -42,6 +42,13 @@
# 27/02/2012 V0.9.4 external programs are executed with csbackobs from now on
# 08/05/2012 V0.9.5 introducing /dev/null disposing for outputstreams
# 09/05/2012 V0.9.6 location flag for programs introduced
# 12/02/2013 V0.9.7 use adapter design pattern delegating the output to
# dev/null
# 12/02/2013 V0.9.8 introduce keychain specific flag
# 05/03/2013 V0.9.9 add secure copy option; improve error handling
# 06/03/2013 V1.0 select copy command for csbackscp by specialcommands flag
# 07/03/2013 V1.1 provide '-newer' option for copy commands
# 12/03/2013 V1.1.1 add keychain option to secure copy process
#
# =============================================================================
"""
......@@ -64,43 +71,54 @@ elif sys.version_info >= (3,):
from configparser import NoOptionError
else:
sys.stderr.write("csback2cron: Incompatible python version.\n")
import csbackErrorCodes as eCodes
__version__ = "V0.9.6"
__version__ = "V1.1.1"
__subversion__ = "$Id$"
__license__ = "GPLv2"
__license__ = "GPLv2+"
__author__ = "Daniel Armbruster"
__copyright__ = "Copyright (c) 2012 by Daniel Armbruster"
DEFAULTPATH = os.path.expanduser("~/.csback/csbackrc")
# -----------------------------------------------------------------------------
class Error(Exception):
def __init__(self, msg):
self.msg = msg
def display(self):
sys.stderr.write("csback2cron (ERROR): " + self.msg + "\n")
class Usage(Error):
def display(self):
usage_text = "Version: "+__version__+"\nLicense: "+__license__+ \
# variables
# ---------
USAGE_TEXT = "Version: "+__version__+"\nLicense: "+__license__+ \
"\n"+__subversion__+"\nAuthor: "+__author__+ """
Usage: csback2cron [-v|--verbose] [-o|--overwrite] [-i|--infile ARG]
[-n|--null] [-l|--location arg] <CRONTABFILENAME>
or: csback2cron -h|--help
Note: If [-i|--infile CONFIGFILE] isn't passed as a argument, csback2cron
uses ~/.csback/csbackrc as default.\n"""
sys.stderr.write("csback2cron: " + self.msg + "\n")
sys.stderr.write(usage_text)
DEFAULTPATH = os.path.expanduser("~/.csback/csbackrc")
# -----------------------------------------------------------------------------
# exception handling
# ---------
class Error(Exception):
def __init__(self, msg, errorCode):
self.msg = str(msg)
self.errorCode = int(errorCode)
def __str__(self):
return "csback2cron (ERROR): {0} (CODE {1}).\n".format(self.msg, \
self.errorCode)
class Usage(Error):
def __str__(self):
return "csback2cron (ERROR): {0} (CODE {1}).\n".format( \
self.msg, self.errorCode)+USAGE_TEXT
# -----------------------------------------------------------------------------
def help():
"""
Printing helptext to stdout.
"""
help_text = "Version: "+__version__+"\nLicense: "+__license__+"\n"+ \
__subversion__+"\nAuthor: "+__author__+"""
Usage: csback2cron [-v|--verbose] [-o|--overwrite] [-i|--infile CONFIGFILE]
[-n|--null] [-l|--location arg] <CRONTABFILENAME>
or: csback2cron -h|--help
help_text = USAGE_TEXT+"""
-------------------------------------------------------------------------------
-v|--verbose Be verbose.
-h|--help Display this help.
......@@ -132,13 +150,11 @@ class Converter():
self.addCronExpr = True
self.scriptLocation = scriptLocation
def convert(self, disposeDevNull=False):
def convert(self):
"""
Abstract function (similar to GoF template function design pattern)
"""
self.convertDict()
if disposeDevNull:
self.line += " >/dev/null 2>&1"
def convertDict(self):
raise NotImplementedError("function must be defined!")
......@@ -247,20 +263,83 @@ class BackupConverter(Converter):
class CopyConverter(Converter):
"""
Class which implements a copy section dictionary converter
Class which implements a copy section dictionary converter (with rsync)
"""
def convertDict(self):
if self.addCronExpr:
self.line = self.sectionDict['cronexpr']
self.line +=' '+self.scriptLocation+'csbackobs "rsync -q'
if self.sectionDict['keychain']:
self.line += ' source $HOME/.keychain/${HOSTNAME}-sh;'
self.line += self.scriptLocation+' csbackobs "'
if self.sectionDict['newer']:
# check if src contains ':' -> remote source directory
# access via remote shell
if self.sectionDict['srcdir'].find(':') > 0:
src = self.sectionDict['srcdir'].split(':')
self.line += 'ssh '+src[0]+' find '+src[1]
else:
self.line += 'find '+self.sectionDict['srcdir']
# complete find command
pos = self.sectionDict['newer'].find(" ")
newer = [self.sectionDict['newer'][0:pos], \
self.sectionDict['newer'][pos+1:]]
self.line += ' -newer'+newer[0]+" '"+newer[1]+"'"+ \
r" -type f -printf '%P\n' | "
self.line += "rsync -q"
for special in self.sectionDict['special']:
self.line += ' '+special.strip()
if self.sectionDict['newer']:
self.line += ' --files-from=-'
for regex in self.sectionDict['exclude']:
self.line += " --exclude='"+regex.strip()+"'"
self.line += \
' '+self.sectionDict['srcdir']+' '+self.sectionDict['targetdir']
self.line +='"'
class SecureCopyConverter(Converter):
"""
Class which implements a copy section dictionary converter (with csbackscp)
"""
def convertDict(self):
if self.addCronExpr:
self.line = self.sectionDict['cronexpr']
if self.sectionDict['keychain']:
self.line += ' source $HOME/.keychain/${HOSTNAME}-sh;'
self.line += self.scriptLocation+' csbackscp -L -l'
if self.sectionDict['newer']:
self.line += ' -N "'+self.sectionDict['newer']+'"'
for regex in self.sectionDict['exclude']:
self.line += " --exclude='"+regex.strip()+"'"
for special in self.sectionDict['special']:
self.line += " --copy '{0}'".format(special)
self.line += \
' '+self.sectionDict['srcdir']+' '+self.sectionDict['targetdir']
self.line +='"'
# -----------------------------------------------------------------------------
class DevNullAdapter(Converter):
"""
Delegate output to /dev/null device. Adapter design pattern for converter
class in use.
"""
def __init__(self, converter):
self.converter = converter
def convert(self):
"""
Abstract function (similar to GoF template function design pattern)
"""
self.converter.convertDict()
self.converter.line += " >/dev/null 2>&1"
def __getattr__(self, attr):
"""
Everything else is delegated to the object
"""
return getattr(self.converter, attr)
# -----------------------------------------------------------------------------
class Processor():
"""
......@@ -274,6 +353,7 @@ class Processor():
self.__verbose = verbose
self.mail = {}
self.copies = []
self.secureCopies = []
self.backups = []
self.tests = []
self.result = []
......@@ -293,7 +373,7 @@ class Processor():
try:
self.config.read(self.configfile)
except ParsingError as err:
raise Error("{0}".format(err.message))
raise Error("{0}".format(err.message), eCodes.CRON_ParsingError)
# fetch mail section
if self.config.has_section('mail'):
try:
......@@ -312,9 +392,9 @@ class Processor():
else:
self.mail['logging'] = False
except NoOptionError as err:
raise Error("{0}", err.message)
raise Error("{0}".format(err.message), eCodes.CRON_MissingOption)
except ValueError:
raise Error("Argument error in [mail] section.")
raise Error("Argument error in [mail] section.", eCodes.CRON_ArgError)
else:
self.mail = {}
# fetch copies (keys) section
......@@ -325,7 +405,8 @@ class Processor():
# fetch every copy section
for key in copyKeys:
if not self.config.has_section(key):
raise Error("Section {0} in configfile not defined.".format(key))
raise Error("Section {0} in configfile not defined.".format(key), \
eCodes.CRON_UndefinedSection)
copy = {'id': key}
try:
copy['cronexpr'] = self.config.get(key, 'cronexpr').strip()
......@@ -335,18 +416,37 @@ class Processor():
if self.config.has_option(key, 'exclude'):
copy['exclude'].extend(self.config.get( \
key, 'exclude', raw=1).split(", "))
copy['special'] = []
copySpecial=''
if self.config.has_option(key, 'specialcommands'):
copy['special'].extend(self.config.get( \
key, 'specialcommands', raw=1).split(", "))
copySpecial = self.config.get(key, 'specialcommands', raw=1)
copy['keychain'] = False
if self.config.has_option(key, 'keychain') and \
self.config.get(key, 'keychain').strip() == 'yes':
copy['keychain'] = True
copy['secure'] = False
if self.config.has_option(key, 'secure') and \
self.config.get(key, 'secure').strip() == 'yes':
copy['secure'] = True
copy['newer'] = None
if self.config.has_option(key, 'newer'):
copy['newer'] = self.config.get(key, 'newer', raw=1)
except NoOptionError as err:
raise Error("{0}".format(err.message))
raise Error("{0}".format(err.message), eCodes.CRON_MissingOption)
except ValueError:
raise Error("Argument error in [{0}] section.".format(key))
raise Error("Argument error in [{0}] section.".format(key), \
eCodes.CRON_ArgError)
else:
self.copies.append(copy)
copy['special'] = []
if copy['secure']:
if len(copySpecial):
copy['special'].insert(0, copySpecial)
self.secureCopies.append(copy)
else:
if len(copySpecial):
copy['special'].extend(copySpecial.split(", "))
self.copies.append(copy)
except NoOptionError as err:
raise Error("{0}".format(err.message))
raise Error("{0}".format(err.message), eCodes.CRON_MissingOption)
except NoSectionError:
self.copies = []
# fetch backups (keys) section
......@@ -357,7 +457,8 @@ class Processor():
# fetch every backup section
for key in backupKeys:
if not self.config.has_section(key):
raise Error("Section {0} in configfile not defined.".format(key))
raise Error("Section {0} in configfile not defined.".format(key), \
eCodes.CRON_UndefinedSection)
backup = {'id': key}
try:
backup['cronexpr'] = self.config.get(key, 'cronexpr').strip()
......@@ -405,13 +506,14 @@ class Processor():
backup['test'] = self.config.getboolean(key, 'test')
backup['tolerant'] = self.config.getboolean(key, 'tolerant')
except NoOptionError as err:
raise Error("{0}".format(err.message))
raise Error("{0}".format(err.message), eCodes.CRON_MissingOption)
except ValueError:
raise Error("Argument error in [{0}] section.".format(key))
raise Error("Argument error in [{0}] section.".format(key), \
eCodes.CRON_ArgError)
else:
self.backups.append(backup)
except NoOptionError as err:
raise Error("{0}".format(err.message))
raise Error("{0}".format(err.message), eCodes.CRON_MissingOption)
except NoSectionError:
self.backups = []
# fetch tests (keys) section
......@@ -422,7 +524,8 @@ class Processor():
# fetch every test section
for key in testKeys:
if not self.config.has_section(key):
raise Error("Section {0} in configfile not defined.".format(key))
raise Error("Section {0} in configfile not defined.".format(key), \
eCodes.CRON_UndefinedSection)
test = {'id': key}
try:
test['cronexpr'] = self.config.get(key, 'cronexpr').strip()
......@@ -440,13 +543,14 @@ class Processor():
test['followlinks'] = self.config.getboolean(key, 'followlinks')
test['tolerant'] = self.config.getboolean(key, 'tolerant')
except NoOptionError as err:
raise Error("{0}".format(err.message))
raise Error("{0}".format(err.message), eCodes.CRON_MissingOption)
except ValueError:
raise Error("Argument error in [{0}] section.".format(key))
raise Error("Argument error in [{0}] section.".format(key), \
eCodes.CRON_ArgError)
else:
self.tests.append(test)
except NoOptionError as err:
raise Error("{0}".format(err.message))
raise Error("{0}".format(err.message), eCodes.CRON_MissingOption)
except NoSectionError:
self.tests = []
if self.__verbose:
......@@ -458,33 +562,56 @@ class Processor():
"""
if self.__verbose:
sys.stdout.write("csback2cron: Conversion ... \n")
#convert copy sections
#convert copy sections (copy with rsync)
if len(self.copies) and self.__verbose:
sys.stdout.write("csback2cron: Converting copy sections ...\n")
for copy in self.copies:
copyConv = CopyConverter(copy, scriptLocation)
copyConv.convert(disposeDevNull)
if disposeDevNull:
copyConv = DevNullAdapter(CopyConverter(copy, scriptLocation))
else:
copyConv = CopyConverter(copy, scriptLocation)
copyConv.convert()
self.result.append(str(copyConv)+"\n")
# convert copy sections (copy with csbackscp)
if len(self.secureCopies) and self.__verbose:
sys.stdout.write("csback2cron: Converting secure copy sections ...\n")
for copy in self.secureCopies:
if disposeDevNull:
secureCopyConv = DevNullAdapter(SecureCopyConverter( \
copy, scriptLocation))
else:
secureCopyConv = SecureCopyConverter(copy, scriptLocation)
secureCopyConv.convert()
self.result.append(str(secureCopyConv)+"\n")
# convert backup sections
if len(self.backups) and self.__verbose:
sys.stdout.write("csback2cron: Converting backup sections ...\n")
for backup in self.backups:
backupConv = BackupConverter(backup, scriptLocation)
backupConv.convert(disposeDevNull)
if disposeDevNull:
backupConv = DevNullAdapter(BackupConverter(backup, scriptLocation))
else:
backupConv = BackupConverter(backup, scriptLocation)
backupConv.convert()
self.result.append(str(backupConv)+"\n")
# convert test sections
if len(self.tests) and self.__verbose:
sys.stdout.write("csback2cron: Converting test sections ...\n")
for test in self.tests:
testConv = TestConverter(test, scriptLocation)
testConv.convert(disposeDevNull)
if disposeDevNull:
testConv = DevNullAdapter(TestConverter(test, scriptLocation))
else:
testConv = TestConverter(test, scriptLocation)
testConv.convert()
self.result.append(str(testConv)+"\n")
# convert mail section
if len(self.mail) and self.__verbose:
sys.stdout.write("csback2cron: Converting mail section ...\n")
if len(self.mail):
mailConv = MailConverter(self.mail, scriptLocation)
mailConv.convert(disposeDevNull)
if disposeDevNull:
mailConv = DevNullAdapter(MailConverter(self.mail, scriptLocation))
else:
mailConv = MailConverter(self.mail, scriptLocation)
mailConv.convert()
self.result.append(str(mailConv)+"\n")
def write(self):
......@@ -520,7 +647,7 @@ def main(argv=None):
opts, args = getopt.getopt(argv[1:], "hovni:l:", ["help", "overwrite",\
"verbose", "infile=", "null", "location="])
except getopt.GetoptError as err:
raise Usage(err.msg)
raise Usage(err.msg, eCodes.GLOBAL_UsageError)
# fetch arguments
inputfile = DEFAULTPATH
verbose = False
......@@ -542,28 +669,31 @@ def main(argv=None):
elif opt in ("-l", "--location"):
scriptLocation = str(arg).rstrip(os.sep)+os.sep
else:
raise Usage("Unhandled option chosen.")
raise Usage("Unhandled option chosen.", eCodes.GLOBAL_UnhandledOption)
if len(args) == 1:
outputfile = args[0]
else:
raise Usage("Invalid arguments.")
raise Usage("Invalid commandline arguments.", \
eCodes.GLOBAL_InvalidCmdlineArgs)
# perform some checks
if not overwrite and os.path.isfile(outputfile):
raise Usage(outputfile +\
" already exists. Enable [-o] to overwrite file.")
" already exists. Enable [-o] to overwrite file.", \
eCodes.CRON_ExistentCrontab)
# create a processor and read, write and convert the content of the
# configuration file using python's ConfigParser module
processor = Processor(inputfile, outputfile, verbose=verbose)
processor.read()
processor.convert(disposeDevNull, scriptLocation)
processor.write()
except Error as err:
err.display()
return 2
except Usage as err:
err.display()
return 2
sys.stderr.write(str(err))
return err.errorCode
except Error as err:
sys.stderr.write(str(err))
return err.errorCode
# -----------------------------------------------------------------------------
if __name__ == "__main__":
......
#!/usr/bin/env python
# This is <csbackErrorCodes.py>
# ----------------------------------------------------------------------------
# $Id$
#
# Copyright (c) 2013 by Daniel Armbruster (BFO Schiltach)
#
# Purpose: Collection csback error codes.
#
# ----
# This file is part of csback.
#
# csback is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# csback is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with csback. If not, see <http://www.gnu.org/licenses/>.
# ----
#
# REVISIONS and CHANGES
# 23/02/2013 V1.0 Daniel Armbruster
#
# ============================================================================
#
# ----------------------------------------------------------------------------
# global error codes (used by several csback tools and modules)
# ----------------------------------------------------------------------------
# general usage error while using program
GLOBAL_UsageError=1
# unhandled commandline option chosen
GLOBAL_UnhandledOption=2
# program called with invalid commandline arguments
GLOBAL_InvalidCmdlineArgs=3
# unhandled exception caught
GLOBAL_UnhandledException=9
# ----------------------------------------------------------------------------
# csbackntfy specific error codes
# ----------------------------------------------------------------------------
# an IOError occurred while parsing a log file
NTFY_IOError=101
# invalid email address
NTFY_InvalidMailAdd=102
# ----------------------------------------------------------------------------
# csback2cron specific error codes
# ----------------------------------------------------------------------------
# error while parsing CONFIGFILE
CRON_ParsingError=201
# missing required option in section
CRON_MissingOption=202
# argument error while parsing CONFIGFILE
CRON_ArgError=203
# undefined section in CONFIGFILE
CRON_UndefinedSection=204
# already existent crontab file
CRON_ExistentCrontab=205
# ----------------------------------------------------------------------------
# csbackresc specific error codes
# ----------------------------------------------------------------------------
# exit status if tag is unkown
RESC_UnknownTag=302
# exit status if ID is missing
RESC_MissingId=303
# exit status if entry property missing
RESC_MissingProberty=304
# exit status for ssh problems
RESC_Ssh=305
# exit status for scp problems
RESC_Scp=306
# exit status for status file problems
RESC_WriteStatus=307
# exit status if directory is not available
RESC_InvalidDir=308
# exit status if file is not available
RESC_InvalidFile=309
# exit status if arbitrary file error
RESC_FileIo=310
# ----------------------------------------------------------------------------
# csbackobs specific error codes
# ----------------------------------------------------------------------------
# error while executing commands
OBS_WhileExecutingCmds=401