adding in some delay checking calculation and parsing of xs:duration types.

example.config.xml

@@ -95,7 +95,14 @@
       -->
       <short>c</short><!-- checksum -->
     </rsyncArgs>
-    <upstream>
+    <!--
+      Upstreams have an optional attribute, "delayCheck", which is an ISO 8601 duration type.
+      https://en.wikipedia.org/wiki/ISO_8601#Durations
+      e.g. 5 minutes would be P0Y0M0DT0H5M0S or just PT5M.
+      It is used to determine if your upstream is "out of date" (e.g. will be skipped if its last check date is older
+      than the specified amount of time). Obviously this is only checked if you have a specified lastRemoteUpdate value.
+    -->
+    <upstream delayCheck="P0Y0M2DT0H0M0S">
       <!--
         The following example uses "rsync://arch.mirror.constant.com/archlinux/"
         (https://www.archlinux.org/mirrors/constant.com/1008/)

repomirror/constants.py

@@ -1,5 +1,6 @@
 PROTO_DEF_PORTS = {'ftp': 21,
                    'rsync': 873}
+
 RSYNC_DEF_ARGS = ['--recursive',
                   '--times',
                   '--links',
@@ -10,7 +11,17 @@ RSYNC_DEF_ARGS = ['--recursive',
                   '--safe-links',
                   '--delete-excluded',
                   '--exclude=.*']
-# How many days an upstream should have last synced by before it's considered stale.
+
-## TODO: make this part of the upstream config? repo config?
+# These are needed to convert years/months to timedeltas.
-DAYS_WARN = 2
+# The following are averaged definitions for time units *in days* according to Google Calculator.
-VERSION = '1.0.4'
+YEAR = 365.2422
+MONTH = 30.4167
+# The following are approximations based on ISO 8601 defintions *in days*.
+# https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm
+# YEAR = 365.25
+# MONTH = 30.6
+
+# We no longer do this by default.
+# # How many days an upstream should have last synced by before it's considered stale.
+# DAYS_WARN = 2
+VERSION = '1.1.0'

repomirror/sync.py

@@ -3,6 +3,7 @@ import logging
 import pwd
 import grp
 import os
+import re
 import socket
 import sys
 import warnings
@@ -23,6 +24,17 @@ if os.isatty(sys.stdin.fileno()):
 else:
     _is_cron = True
 
+_duration_re = re.compile(('^P'
+                           '((?P<years>[0-9]+(\.[0-9]+)?)Y)?'
+                           '((?P<months>[0-9]+(\.[0-9]+)?)M)?'
+                           '((?P<days>[0-9]+(\.[0-9]+)?)D)?'
+                           'T?'
+                           '((?P<hours>[0-9]+(\.[0-9]+)?)H)?'
+                           '((?P<minutes>[0-9]+(\.[0-9]+)?)M)?'
+                           '((?P<seconds>[0-9]+(\.[0-9]+)?)S)?'
+                           '$'))
+
+
 
 def get_owner(owner_xml):
     owner = {}
@@ -149,8 +161,10 @@ class Upstream(object):
         self.domain = self.xml.find('domain').text
         self.path = self.xml.find('path').text
         self.dest = os.path.abspath(os.path.expanduser(dest))
+        self.delay = None
         self.owner = owner
         self.filechecks = filechecks
+        self._get_delaychk()
         self.has_new = False
         # These are optional.
         port = self.xml.find('port')
@@ -171,6 +185,18 @@ class Upstream(object):
             self.fetcher = fetcher.FTP(self.domain, self.port, self.path, self.dest, owner = self.owner)
         self._check_conn()
 
+    def _get_delaychk(self):
+        delay = self.xml.attrib.get('delayCheck')
+        if not delay:
+            return(None)
+        r = _duration_re.search(delay)
+        times = {k: (float(v) if v else 0.0) for k, v in r.groupdict().items()}
+        years = float(times.pop('years'))
+        months = float(times.pop('months'))
+        times['days'] = (times['days'] + (years * constants.YEAR) + (months * constants.MONTH))
+        self.delay = datetime.timedelta(**times)
+        return(None)
+
     def _check_conn(self):
         sock = socket.socket()
         sock.settimeout(7)

reposync

@@ -1,56 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import os
-import pathlib
-import sys
-##
-##
-_cur_file = pathlib.Path(os.path.abspath(os.path.expanduser(__file__))).resolve()
-_cur_path = os.path.dirname(_cur_file)
-sys.path.insert(1, _cur_path)
-import repomirror
-
-
-if os.geteuid() == 0:
-    _def_logdir = '/var/log/repo'
-else:
-    _def_logdir = '~/.cache/repologs'
-
-
-def parseArgs():
-    args = argparse.ArgumentParser(description = 'Sync repositories for various distributions to local paths')
-    args.add_argument('-c', '--config',
-                      default = '~/.config/repomirror.xml',
-                      dest = 'cfg',
-                      help = ('The path to the config file. If it does not exist, a bare version will be created. '
-                              'Default: ~/.config/repomirror.xml'))
-    # args.add_argument('-n', '--dry-run',
-    #                   action = 'store_true',
-    #                   dest = 'dummy',
-    #                   help = ('If specified, do not actually sync anything (other than timestamp files if '
-    #                           'applicable to determine logic); do not actually sync any repositories. Useful for '
-    #                           'generating logs to determine potential issues before they happen'))
-    args.add_argument('-d', '--distro',
-                      dest = 'distro',
-                      action = 'append',
-                      help = ('If specified, only sync the specified distro in the config file (otherwise sync all '
-                              'specified). May be given multiple times'))
-    args.add_argument('-l', '--logdir',
-                      default = _def_logdir,
-                      dest = 'logdir',
-                      help = ('The path to the directory where logs should be written. The actual log files will be '
-                              'named after their respective distro names in the config file. '
-                              'Default: {0}'.format(_def_logdir)))
-    return(args)
-
-
-def main():
-    args = parseArgs().parse_args()
-    r = repomirror.Sync(**vars(args))
-    r.sync()
-    return(None)
-
-
-if __name__ == '__main__':
-    main()

reposync

@@ -0,0 +1 @@
+reposync.py

reposync.py

@@ -1 +0,0 @@
-reposync

reposync.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import pathlib
+import sys
+##
+##
+_cur_file = pathlib.Path(os.path.abspath(os.path.expanduser(__file__))).resolve()
+_cur_path = os.path.dirname(_cur_file)
+sys.path.insert(1, _cur_path)
+import repomirror
+
+
+if os.geteuid() == 0:
+    _def_logdir = '/var/log/repo'
+else:
+    _def_logdir = '~/.cache/repologs'
+
+
+def parseArgs():
+    args = argparse.ArgumentParser(description = 'Sync repositories for various distributions to local paths')
+    args.add_argument('-c', '--config',
+                      default = '~/.config/repomirror.xml',
+                      dest = 'cfg',
+                      help = ('The path to the config file. If it does not exist, a bare version will be created. '
+                              'Default: ~/.config/repomirror.xml'))
+    # args.add_argument('-n', '--dry-run',
+    #                   action = 'store_true',
+    #                   dest = 'dummy',
+    #                   help = ('If specified, do not actually sync anything (other than timestamp files if '
+    #                           'applicable to determine logic); do not actually sync any repositories. Useful for '
+    #                           'generating logs to determine potential issues before they happen'))
+    args.add_argument('-d', '--distro',
+                      dest = 'distro',
+                      action = 'append',
+                      help = ('If specified, only sync the specified distro in the config file (otherwise sync all '
+                              'specified). May be given multiple times'))
+    args.add_argument('-l', '--logdir',
+                      default = _def_logdir,
+                      dest = 'logdir',
+                      help = ('The path to the directory where logs should be written. The actual log files will be '
+                              'named after their respective distro names in the config file. '
+                              'Default: {0}'.format(_def_logdir)))
+    return(args)
+
+
+def main():
+    args = parseArgs().parse_args()
+    r = repomirror.Sync(**vars(args))
+    r.sync()
+    return(None)
+
+
+if __name__ == '__main__':
+    main()