cleaning up Rob's remote branch for inclusion into upstream

.gitignore

@@ -25,12 +25,3 @@ __pycache__/
 *.sqlite3
 *.deb
 .idea/
-files/*
-repo/*
-repo\:testrepo/*
-config
-hints.21
-index.21
-integrity.21
-nonce
-README

backup.py

@@ -20,9 +20,13 @@ import re
 import subprocess
 import sys
 import tempfile
-from xmlrpc.client import Server
+
 # TODO: virtual env?
-from lxml import etree  # A lot safer and easier to use than the stdlib xml module.
+# https://pypi.org/project/isodate/
+import isodate
+# https://pypi.org/project/lxml/
+# A lot safer and easier to use than the stdlib xml module.
+from lxml import etree
 try:
     # https://www.freedesktop.org/software/systemd/python-systemd/journal.html#journalhandler-class
     from systemd import journal
@@ -30,6 +34,7 @@ try:
 except ImportError:
     has_systemd = False
 
+
 ### LOG LEVEL MAPPINGS ###
 loglvls = {'critical': logging.CRITICAL,
            'error': logging.ERROR,
@@ -38,7 +43,7 @@ loglvls = {'critical': logging.CRITICAL,
            'debug': logging.DEBUG}
 
 ### DEFAULT NAMESPACE ###
-dflt_ns = 'http://git.square-r00t.net/BorgExtend/tree/storage/backups/borg/'
+dflt_ns = 'http://git.root2.io/r00t2/borgextend/'
 
 
 ### THE GUTS ###
@@ -79,6 +84,8 @@ class Backup(object):
         if self.args['verbose']:
             handlers.append(logging.StreamHandler())
         if has_systemd:
+            # There are two different modules with the same import floating around.
+            # We can use either, but we need to figure out which one it is first.
             try:
                 h = journal.JournalHandler()
             except AttributeError:
@@ -120,6 +127,9 @@ class Backup(object):
             if not reponames:
                 reponames = []
             repos = []
+            dfltRetention = None
+            if server.attrib.get('pruneRetention') is not None:
+                dfltRetention = isodate.parse_duration(server.attrib.get('pruneRetention'))
             for repo in server.findall('{0}repo'.format(self.ns)):
                 if reponames and repo.attrib['name'] not in reponames:
                     continue
@@ -154,6 +164,11 @@ class Backup(object):
                                 r['plugins'][pname]['params'][paramname] = json.loads(param.text)
                             else:
                                 r['plugins'][pname]['params'][paramname] = param.text
+                retention = repo.attrib.get('pruneRetention')
+                if retention is not None:
+                    r['retention'] = isodate.parse_duration(retention)
+                else:
+                    r['retention'] = dfltRetention
                 repos.append(r)
             return(repos)
         self.logger.debug('VARS (before args cleanup): {0}'.format(vars(self)))
@@ -181,6 +196,7 @@ class Backup(object):
                     self.repos[sname][x] = server.attrib[x]
                 self.repos[sname]['repos'] = getRepo(server, reponames = self.args['repo'])
         self.logger.debug('VARS (after args cleanup): {0}'.format(vars(self)))
+        self.logger.debug('REPOS: {0}'.format(dict(self.repos)))
         return()
 
     def createRepo(self):
@@ -455,7 +471,7 @@ class Backup(object):
         return()
     
     def prune(self):
-        # TODO: support "--strip-components N"?
+        # https://borgbackup.readthedocs.io/en/stable/usage/prune.html
         self.logger.info('START: prune')
         for server in self.repos:
             _env = os.environ.copy()
@@ -465,6 +481,13 @@ class Backup(object):
             _env['LC_CTYPE'] = 'en_US.UTF-8'
             _user = self.repos[server].get('user', pwd.getpwuid(os.geteuid()).pw_name)
             for repo in self.repos[server]['repos']:
+                if repo.get('retention') is None:
+                    # No prune duration was set. Skip.
+                    continue
+                if isinstance(repo['retention'], datetime.timedelta):
+                    retentionSeconds = repo['retention'].total_seconds()
+                else:  # it's an isodate.Duration
+                    retentionSeconds = repo['retention'].totimedelta(datetime.datetime.now()).total_seconds()
                 _loc_env = _env.copy()
                 if 'password' not in repo:
                     print('Password not supplied for {0}:{1}.'.format(server, repo['name']))
@@ -477,21 +500,8 @@ class Backup(object):
                         '--log-json',
                         '--{0}'.format(self.args['loglevel']),
                         'prune',
-                        '--stats']
+                        '--stats',
-                if self.repos[server]['keepYearly'][0].isnumeric() and int(self.repos[server]['keepYearly'][0]) > 0:
+                        '--keep-secondly', int(retentionSeconds)]
-                    _cmd.extend(['--keep-yearly', self.repos[server]['keepYearly'].lower()[0]])
-                if self.repos[server]['keepMonthly'][0].isnumeric() and int(self.repos[server]['keepMonthly'][0]) > 0:
-                    _cmd.extend(['--keep-monthly', self.repos[server]['keepMonthly'].lower()[0]])
-                if self.repos[server]['keepWeekly'][0].isnumeric() and int(self.repos[server]['keepWeekly'][0]) > 0:
-                    _cmd.extend(['--keep-weekly', self.repos[server]['keepWeekly'].lower()[0]])
-                if self.repos[server]['keepDaily'][0].isnumeric() and int(self.repos[server]['keepDaily'][0]) > 0:
-                    _cmd.extend(['--keep-daily', self.repos[server]['keepDaily'].lower()[0]])
-                if self.repos[server]['keepHourly'][0].isnumeric() and int(self.repos[server]['keepHourly'][0]) > 0:
-                    _cmd.extend(['--keep-hourly', self.repos[server]['keepHourly'].lower()[0]])
-                if self.repos[server]['keepMinutely'][0].isnumeric() and int(self.repos[server]['keepMinutely'][0]) > 0:
-                    _cmd.extend(['--keep-minutely', self.repos[server]['keepMinutely'].lower()[0]])
-                if self.repos[server]['keepSecondly'][0].isnumeric() and int(self.repos[server]['keepSecondly'][0]) > 0:
-                    _cmd.extend(['--keep-secondly', self.repos[server]['keepSecondly'].lower()[0]])
                 if self.repos[server]['remote'].lower()[0] in ('1', 't'):
                     repo_tgt = '{0}@{1}'.format(_user, server)
                 else:
@@ -812,6 +822,7 @@ def parseArgs():
                                   'repo under their respective server(s).'))
     return (args)
 
+
 def convertConf(cfgfile):
     oldcfgfile = re.sub('\.xml$', '.json', cfgfile)
     try:
@@ -857,7 +868,7 @@ def convertConf(cfgfile):
     namespaces = {None: dflt_ns,
                   'xsi': 'http://www.w3.org/2001/XMLSchema-instance'}
     xsi = {('{http://www.w3.org/2001/'
-            'XMLSchema-instance}schemaLocation'): ('http://git.square-r00t.net/BorgExtend/plain/config.xsd')}
+            'XMLSchema-instance}schemaLocation'): ('http://git.r00t2.io/r00t2/borgextend/src/branch/master/config.xsd')}
     genname = 'LXML (http://lxml.de/)'
     root = etree.Element('borg', nsmap = namespaces, attrib = xsi)
     root.append(etree.Comment(('Generated by {0} on {1} from {2} via {3}').format(sys.argv[0],

config.xml

@@ -1,34 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<borg xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-      xmlns="http://git.square-r00t.net/BorgExtend/tree/storage/backups/borg/"
-      xsi:schemaLocation="http://git.square-r00t.net/BorgExtend/plain/config.xsd"
-      xmlns:xi="http://www.w3.org/2001/XInclude">
-    <!-- You can have multiple server elements, but each one *MUST* have a unique "target" attribute. -->
-    <!-- "target" = either the local filesystem path (absolute or relative to execution) or the remote host
-         "remote" = 1/true if "target" is a remote host or 0/false if it's a local filepath
-         "rsh" = (remote host only) the ssh command to use. The default is given below.
-         "user" = (remote host only) the ssh user to use.
-         "dummy" = a boolean; if you need to create a "dummy" server, set this to "true".
-                   It will *not* be parsed or executed upon.
-                   It won't even be created by an init operation or show up in a repolist operation. -->
-    <server target="repo" remote="false" keepSecondly="3" keepMinutely="2">
-        <!-- You can (and probably will) have multiple repos for each server. -->
-        <!-- "name" = the repositoriy name.
-             "password" = the repository's password for the key. If not specified, you will be prompted
-                          to enter it interactively and securely.
-             "dummy" = see server[@dummy] explanation.
-             "compression" = see https://borgbackup.readthedocs.io/en/stable/usage/create.html (-C option) -->
-        <repo name="testrepo" password="SuperSecretPassword" compression="lzma,9">
-            <!-- Each path entry is a path to back up.
-                 See https://borgbackup.readthedocs.io/en/stable/usage/create.html
-                 Note that globbing, etc. is *disabled* for security reasons, so you will need to specify all
-                 directories explicitly. -->
-		<path>/home/nosbig/git-repos/nosbig/BorgExtend/files</path>
-            <!-- Each exclude entry should be a subdirectory of a <path> (otherwise it wouldn't match, obviously). -->
-		<exclude>/home/nosbig/git-repos/nosbig/BorgExtend/files/b.txt</exclude>
-        </repo>
-        <!-- You can also include other snippets. Either absolute paths, paths relative to your backup.xml file,
-             or a URL. -->
-        <xi:include href="sample.config.snippet.xml"/>
-    </server>
-</borg>

config.xsd

@@ -1,22 +1,48 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
+<xs:schema targetNamespace="http://git.root2.io/r00t2/borgextend/"
-           targetNamespace="http://git.square-r00t.net/BorgExtend/tree/"
+           xmlns="http://git.root2.io/r00t2/borgextend/"
-           xmlns="http://git.square-r00t.net/BorgExtend/tree/"
+           xmlns:borg="http://git.root2.io/r00t2/borgextend/"
-           xmlns:borg="http://git.square-r00t.net/BorgExtend/tree/"
+           xmlns:xs="http://www.w3.org/2001/XMLSchema"
            elementFormDefault="qualified"
            attributeFormDefault="unqualified">
 
+  <!--
+      Sort of imported in from t_unix_posixUserGroup in http://xml.r00t2.io/schema/lib/types/unix.xsd
+  -->
   <xs:simpleType name="posixuser">
     <xs:restriction base="xs:token">
       <xs:pattern value="[a-z_]([a-z0-9_-]{0,31}|[a-z0-9_-]{0,30}$)"/>
     </xs:restriction>
   </xs:simpleType>
+
   <xs:simpleType name="blocktext">
     <xs:restriction base="xs:string">
       <xs:whiteSpace value="preserve"/>
     </xs:restriction>
   </xs:simpleType>
 
+  <xs:complexType name="prepScript">
+    <xs:simpleContent>
+      <xs:extension base="blocktext">
+        <xs:attribute name="inline" use="optional" default="false" type="xs:boolean"/>
+      </xs:extension>
+    </xs:simpleContent>
+  </xs:complexType>
+
+  <xs:complexType name="param">
+    <xs:simpleContent>
+      <xs:extension base="blocktext">
+        <xs:attribute name="key"
+                      type="xs:token"
+                      use="required"/>
+        <xs:attribute name="json"
+                      type="xs:boolean"
+                      default="0"
+                      use="optional"/>
+      </xs:extension>
+    </xs:simpleContent>
+  </xs:complexType>
+
   <!-- START ROOT -->
   <xs:element name="borg">
     <xs:complexType>
@@ -42,17 +68,8 @@
                     <!-- This gets messy. We essentially preserve whitespace, allowing
                          either an inline script to be executed (written to a temp file) or
                          a path to an external script/command to be specified. -->
-                                        <xs:element name="prep" minOccurs="0"
+                    <xs:element name="prep" type="prepScript" minOccurs="0"
-                                                    maxOccurs="unbounded">
+                                maxOccurs="unbounded"/>
-                                            <xs:complexType>
-                                                <xs:simpleContent>
-                                                    <xs:extension base="borg:blocktext">
-                                                        <xs:attribute name="inline" type="xs:boolean"
-                                                                      default="0"/>
-                                                    </xs:extension>
-                                                </xs:simpleContent>
-                                            </xs:complexType>
-                                        </xs:element>
                     <!-- END PREP -->
                     <!-- START PLUGIN -->
                     <xs:element name="plugins" minOccurs="0"
@@ -63,20 +80,7 @@
                             <xs:complexType>
                               <xs:sequence>
                                 <xs:element name="param" minOccurs="0"
-                                                                            maxOccurs="unbounded">
+                                            maxOccurs="unbounded" type="param">
-                                                                    <xs:complexType>
-                                                                        <xs:simpleContent>
-                                                                            <xs:extension base="borg:blocktext">
-                                                                                <xs:attribute name="key"
-                                                                                              type="xs:token"
-                                                                                              use="required"/>
-                                                                                <xs:attribute name="json"
-                                                                                              type="xs:boolean"
-                                                                                              default="0"
-                                                                                              use="optional"/>
-                                                                            </xs:extension>
-                                                                        </xs:simpleContent>
-                                                                    </xs:complexType>
                                 </xs:element>
                               </xs:sequence>
                               <xs:attribute name="name" type="xs:string" use="required"/>
@@ -96,9 +100,11 @@
                   <!-- This specifies if a repo is a "dummy" configuration.
                        Useful for testing and placeholder. -->
                   <xs:attribute name="dummy" type="xs:boolean" use="optional" default="false"/>
+                  <!-- This is used to specify the per-repo retention period for pruning. -->
+                  <xs:attribute name="pruneRetention" type="xs:duration" use="optional"/>
                 </xs:complexType>
                 <xs:unique name="uniquePath">
-                                    <xs:selector xpath="borg:path"/>
+                  <xs:selector xpath="path"/>
                   <xs:field xpath="."/>
                 </xs:unique>
               </xs:element>
@@ -114,24 +120,12 @@
             <xs:attribute name="rsh" type="xs:string" use="optional"/>
             <!-- Only used if "target" is a remote host. -->
             <!-- The remote host SSH user. -->
-                        <xs:attribute name="user" type="borg:posixuser" use="optional"/>
+            <xs:attribute name="user" type="posixuser" use="optional"/>
             <!-- This specifies if a server is a "dummy" configuration.
                  Useful for testing and placeholder. -->
             <xs:attribute name="dummy" type="xs:boolean" use="optional" default="false"/>
-                        <!-- Used to define the number of yearly backups to keep. -->
+            <!-- This is used to specify the server-wide default retention period for pruning. -->
-                        <xs:attribute name="keepYearly" type="xs:int" use="optional" />
+            <xs:attribute name="pruneRetention" type="xs:duration" use="optional"/>
-                        <!-- Used to define the number of monthly backups to keep. -->
-                        <xs:attribute name="keepMonthly" type="xs:int" use="optional" />
-                        <!-- Used to define the number of weekly backups to keep. -->
-                        <xs:attribute name="keepWeekly" type="xs:int" use="optional" />
-                        <!-- Used to define the number of daily backups to keep. -->
-                        <xs:attribute name="keepDaily" type="xs:int" use="optional" />
-                        <!-- Used to define the number of hourly backups to keep. -->
-                        <xs:attribute name="keepHourly" type="xs:int" use="optional" />
-                        <!-- Used to define the number of minutely backups to keep. -->
-                        <xs:attribute name="keepMinutely" type="xs:int" use="optional" />
-                        <!-- Used to define the number of secondly backups to keep. -->
-                        <xs:attribute name="keepSecondly" type="xs:int" use="optional" />
           </xs:complexType>
         </xs:element>
         <!-- END SERVER -->
@@ -139,7 +133,7 @@
       <xs:attribute name="borgpath" default="borg" use="optional"/>
     </xs:complexType>
     <xs:unique name="uniqueServer">
-            <xs:selector xpath="borg:server"/>
+      <xs:selector xpath="server"/>
       <xs:field xpath="@target"/>
     </xs:unique>
   </xs:element>

sample.config.snippet.xml

@@ -1,4 +1,9 @@
 <?xml version="1.0" encoding="UTF-8" ?>
-        <repo name="testrepo2" password="AnotherSuperSecretPassword" dummy="true">
+        <repo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+              xmlns="http://git.root2.io/r00t2/borgextend/"
+              xsi:schemaLocation="http://git.root2.io/r00t2/borgextend/ http://git.r00t2.io/r00t2/borgextend/src/branch/master/config.xsd"
+              name="testrepo2"
+              password="AnotherSuperSecretPassword"
+              dummy="true">
             <path>/dev/null</path>
         </repo>

sample.config.xml

@@ -1,24 +1,43 @@
 <?xml version="1.0" encoding="UTF-8" ?>
 <borg xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-      xmlns="http://git.square-r00t.net/BorgExtend/tree/storage/backups/borg/"
+      xmlns="http://git.root2.io/r00t2/borgextend/"
-      xsi:schemaLocation="http://git.square-r00t.net/BorgExtend/plain/config.xsd"
+      xsi:schemaLocation="http://git.root2.io/r00t2/borgextend/ http://git.r00t2.io/r00t2/borgextend/src/branch/master/config.xsd"
       xmlns:xi="http://www.w3.org/2001/XInclude">
     <!-- You can have multiple server elements, but each one *MUST* have a unique "target" attribute. -->
-    <!-- "target" = either the local filesystem path (absolute or relative to execution) or the remote host
+    <!--
+        "target" = either the local filesystem path (absolute or relative to execution) or the remote host
         "remote" = 1/true if "target" is a remote host or 0/false if it's a local filepath
         "rsh" = (remote host only) the ssh command to use. The default is given below.
         "user" = (remote host only) the ssh user to use.
         "dummy" = a boolean; if you need to create a "dummy" server, set this to "true".
                   It will *not* be parsed or executed upon.
-                   It won't even be created by an init operation or show up in a repolist operation. -->
+                  It won't even be created by an init operation or show up in a repolist operation.
-    <server target="fq.dn.tld" remote="true" rsh="ssh -p 22" user="root">
+        "pruneRetention" = an ISO 8601 duration to be used as the default rentention period during a prune operation.
+    -->
+    <!--
+        The pruneRetention attribute's format (ISO 8601 Duration) can be found here:
+            https://en.wikipedia.org/wiki/ISO_8601#Durations).
+        It must be positive. The "alternative"/"extended" variant (the one using colons) should be supported
+        (but may be unpredictable).
+        It is optional, but can be used as a fallback during a prune operation if a repo child does not specify one.
+        The below example would prune anything older than 1 month, 2 minutes, and 30 seconds.
+    -->
+    <server target="fq.dn.tld" remote="true" rsh="ssh -p 22" user="root" pruneRetention="P1MT2M30S">
         <!-- You can (and probably will) have multiple repos for each server. -->
-        <!-- "name" = the repositoriy name.
+        <!--
+            "name" = the repository name.
             "password" = the repository's password for the key. If not specified, you will be prompted
                          to enter it interactively and securely.
             "dummy" = see server[@dummy] explanation.
-             "compression" = see https://borgbackup.readthedocs.io/en/stable/usage/create.html (-C option) -->
+            "compression" = see https://borgbackup.readthedocs.io/en/stable/usage/create.html (-C option)
-        <repo name="testrepo" password="SuperSecretPassword" compression="lzma,9">
+            "pruneRetention" = an ISO 8601 duration to be used as the rentention period during a prune operation.
+                               If not specified, uses ../server/[@pruneRetention].
+                               If ../server/[@pruneRetention] was not specified, this repo will be skipped during a prune.
+        -->
+        <repo name="testrepo"
+              password="SuperSecretPassword"
+              compression="lzma,9"
+              pruneRetention="P1Y"><!-- One year. -->
             <!-- Each path entry is a path to back up.
                  See https://borgbackup.readthedocs.io/en/stable/usage/create.html
                  Note that globbing, etc. is *disabled* for security reasons, so you will need to specify all
@@ -76,6 +95,8 @@
         </repo>
         <!-- You can also include other snippets. Either absolute paths, paths relative to your backup.xml file,
              or a URL. -->
+        <!--
         <xi:include href="sample.config.snippet.xml"/>
+        -->
     </server>
 </borg>