From dfe2f4d100a8ed43444c69b85005dbb76c0d5e3a Mon Sep 17 00:00:00 2001 From: brent s Date: Sat, 21 May 2022 06:11:40 -0400 Subject: [PATCH] cleaning up Rob's remote branch for inclusion into upstream --- .gitignore | 9 -- backup.py | 61 +++++---- config.xml | 34 ----- config.xsd | 272 +++++++++++++++++++------------------- sample.config.snippet.xml | 9 +- sample.config.xml | 53 +++++--- 6 files changed, 213 insertions(+), 225 deletions(-) delete mode 100644 config.xml diff --git a/.gitignore b/.gitignore index fd69eda..ff3b78a 100644 --- a/.gitignore +++ b/.gitignore @@ -25,12 +25,3 @@ __pycache__/ *.sqlite3 *.deb .idea/ -files/* -repo/* -repo\:testrepo/* -config -hints.21 -index.21 -integrity.21 -nonce -README \ No newline at end of file diff --git a/backup.py b/backup.py index f14dc79..674bae1 100755 --- a/backup.py +++ b/backup.py @@ -20,9 +20,13 @@ import re import subprocess import sys import tempfile -from xmlrpc.client import Server + # TODO: virtual env? -from lxml import etree # A lot safer and easier to use than the stdlib xml module. +# https://pypi.org/project/isodate/ +import isodate +# https://pypi.org/project/lxml/ +# A lot safer and easier to use than the stdlib xml module. +from lxml import etree try: # https://www.freedesktop.org/software/systemd/python-systemd/journal.html#journalhandler-class from systemd import journal @@ -30,6 +34,7 @@ try: except ImportError: has_systemd = False + ### LOG LEVEL MAPPINGS ### loglvls = {'critical': logging.CRITICAL, 'error': logging.ERROR, @@ -38,7 +43,7 @@ loglvls = {'critical': logging.CRITICAL, 'debug': logging.DEBUG} ### DEFAULT NAMESPACE ### -dflt_ns = 'http://git.square-r00t.net/BorgExtend/tree/storage/backups/borg/' +dflt_ns = 'http://git.root2.io/r00t2/borgextend/' ### THE GUTS ### @@ -79,6 +84,8 @@ class Backup(object): if self.args['verbose']: handlers.append(logging.StreamHandler()) if has_systemd: + # There are two different modules with the same import floating around. + # We can use either, but we need to figure out which one it is first. try: h = journal.JournalHandler() except AttributeError: @@ -120,6 +127,9 @@ class Backup(object): if not reponames: reponames = [] repos = [] + dfltRetention = None + if server.attrib.get('pruneRetention') is not None: + dfltRetention = isodate.parse_duration(server.attrib.get('pruneRetention')) for repo in server.findall('{0}repo'.format(self.ns)): if reponames and repo.attrib['name'] not in reponames: continue @@ -154,6 +164,11 @@ class Backup(object): r['plugins'][pname]['params'][paramname] = json.loads(param.text) else: r['plugins'][pname]['params'][paramname] = param.text + retention = repo.attrib.get('pruneRetention') + if retention is not None: + r['retention'] = isodate.parse_duration(retention) + else: + r['retention'] = dfltRetention repos.append(r) return(repos) self.logger.debug('VARS (before args cleanup): {0}'.format(vars(self))) @@ -181,6 +196,7 @@ class Backup(object): self.repos[sname][x] = server.attrib[x] self.repos[sname]['repos'] = getRepo(server, reponames = self.args['repo']) self.logger.debug('VARS (after args cleanup): {0}'.format(vars(self))) + self.logger.debug('REPOS: {0}'.format(dict(self.repos))) return() def createRepo(self): @@ -455,7 +471,7 @@ class Backup(object): return() def prune(self): - # TODO: support "--strip-components N"? + # https://borgbackup.readthedocs.io/en/stable/usage/prune.html self.logger.info('START: prune') for server in self.repos: _env = os.environ.copy() @@ -465,6 +481,13 @@ class Backup(object): _env['LC_CTYPE'] = 'en_US.UTF-8' _user = self.repos[server].get('user', pwd.getpwuid(os.geteuid()).pw_name) for repo in self.repos[server]['repos']: + if repo.get('retention') is None: + # No prune duration was set. Skip. + continue + if isinstance(repo['retention'], datetime.timedelta): + retentionSeconds = repo['retention'].total_seconds() + else: # it's an isodate.Duration + retentionSeconds = repo['retention'].totimedelta(datetime.datetime.now()).total_seconds() _loc_env = _env.copy() if 'password' not in repo: print('Password not supplied for {0}:{1}.'.format(server, repo['name'])) @@ -477,27 +500,14 @@ class Backup(object): '--log-json', '--{0}'.format(self.args['loglevel']), 'prune', - '--stats'] - if self.repos[server]['keepYearly'][0].isnumeric() and int(self.repos[server]['keepYearly'][0]) > 0: - _cmd.extend(['--keep-yearly', self.repos[server]['keepYearly'].lower()[0]]) - if self.repos[server]['keepMonthly'][0].isnumeric() and int(self.repos[server]['keepMonthly'][0]) > 0: - _cmd.extend(['--keep-monthly', self.repos[server]['keepMonthly'].lower()[0]]) - if self.repos[server]['keepWeekly'][0].isnumeric() and int(self.repos[server]['keepWeekly'][0]) > 0: - _cmd.extend(['--keep-weekly', self.repos[server]['keepWeekly'].lower()[0]]) - if self.repos[server]['keepDaily'][0].isnumeric() and int(self.repos[server]['keepDaily'][0]) > 0: - _cmd.extend(['--keep-daily', self.repos[server]['keepDaily'].lower()[0]]) - if self.repos[server]['keepHourly'][0].isnumeric() and int(self.repos[server]['keepHourly'][0]) > 0: - _cmd.extend(['--keep-hourly', self.repos[server]['keepHourly'].lower()[0]]) - if self.repos[server]['keepMinutely'][0].isnumeric() and int(self.repos[server]['keepMinutely'][0]) > 0: - _cmd.extend(['--keep-minutely', self.repos[server]['keepMinutely'].lower()[0]]) - if self.repos[server]['keepSecondly'][0].isnumeric() and int(self.repos[server]['keepSecondly'][0]) > 0: - _cmd.extend(['--keep-secondly', self.repos[server]['keepSecondly'].lower()[0]]) + '--stats', + '--keep-secondly', int(retentionSeconds)] if self.repos[server]['remote'].lower()[0] in ('1', 't'): repo_tgt = '{0}@{1}'.format(_user, server) else: repo_tgt = os.path.abspath(os.path.expanduser(server)) _cmd.append('{0}:{1}'.format(repo_tgt, - repo['name'])) + repo['name'])) self.logger.debug('VARS: {0}'.format(vars())) # We don't use self.cmdExec() here because we want to explicitly # pass the env and format the log line differently. @@ -759,10 +769,10 @@ def parseArgs(): remoteargs, fileargs]) pruneargs = subparsers.add_parser('prune', - help = ('Prune backups to schedule.'), - parents = [commonargs, - remoteargs, - fileargs]) + help = ('Prune backups to schedule.'), + parents = [commonargs, + remoteargs, + fileargs]) cvrtargs = subparsers.add_parser('convert', help = ('Convert the legacy JSON format to the new XML format and quit')) ### OPERATION-SPECIFIC OPTIONS ### @@ -812,6 +822,7 @@ def parseArgs(): 'repo under their respective server(s).')) return (args) + def convertConf(cfgfile): oldcfgfile = re.sub('\.xml$', '.json', cfgfile) try: @@ -857,7 +868,7 @@ def convertConf(cfgfile): namespaces = {None: dflt_ns, 'xsi': 'http://www.w3.org/2001/XMLSchema-instance'} xsi = {('{http://www.w3.org/2001/' - 'XMLSchema-instance}schemaLocation'): ('http://git.square-r00t.net/BorgExtend/plain/config.xsd')} + 'XMLSchema-instance}schemaLocation'): ('http://git.r00t2.io/r00t2/borgextend/src/branch/master/config.xsd')} genname = 'LXML (http://lxml.de/)' root = etree.Element('borg', nsmap = namespaces, attrib = xsi) root.append(etree.Comment(('Generated by {0} on {1} from {2} via {3}').format(sys.argv[0], diff --git a/config.xml b/config.xml deleted file mode 100644 index c7023e4..0000000 --- a/config.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - - - /home/nosbig/git-repos/nosbig/BorgExtend/files - - /home/nosbig/git-repos/nosbig/BorgExtend/files/b.txt - - - - - diff --git a/config.xsd b/config.xsd index c1cdeed..f359bc1 100644 --- a/config.xsd +++ b/config.xsd @@ -1,147 +1,141 @@ - - - - - - - - - - - + + + + + + - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/sample.config.snippet.xml b/sample.config.snippet.xml index 107342a..e1710e1 100644 --- a/sample.config.snippet.xml +++ b/sample.config.snippet.xml @@ -1,4 +1,9 @@ - + /dev/null - \ No newline at end of file + diff --git a/sample.config.xml b/sample.config.xml index a4527a6..7c8a1f8 100644 --- a/sample.config.xml +++ b/sample.config.xml @@ -1,24 +1,43 @@ - - + + + - - + + +