Automated commit (/opt/dev/infra/gitclass.py)

README

@@ -10,7 +10,6 @@ Features:
 * Changing of directory/file ownership per-distribution
 * Mount-checking per-distribution - a distribution will be skipped if its designated mountpoint is not mounted
 * Synchronization checks - timestamp files can be read and written and are used to determine if a sync should take place or not
-** TODO: customizable "staleness" of repos (e.g. sync if timestamp X is older than timestamp Y on server)
 
 
 Configuration/Deployment:

example.config.xml

@@ -43,18 +43,22 @@
     <lastLocalSync timeFormat="UNIX_EPOCH">/srv/repos/arch/lastsync</lastLocalSync>
     <!--
       The path to a file on the upstream(s) that gives a time when it last updated.
-      The optional timeFormat attribute behavior is the same as above.
+      The syntax and options are the same as lastLocalCheck/lastLocalSync.
       If neither this nor lastRemoteSync is provided, a sync will be attempted regardless of when the last one was
       attempted.
     -->
+    <!--
+      Remote timestamps take an additional optional boolean attribute, "mtime". If true, the mtime of the remote file
+      will be checked instead of the content of the file (and thus timeFormat is ignored).
+    -->
     <lastRemoteUpdate timeFormat="UNIX_EPOCH">/lastupdate</lastRemoteUpdate>
     <!--
       The path to a file on the upstream(s) that gives a time when it last synced from its upstream.
-      The optional timeFormat attribute behavior is the same as above.
+      The syntax and options are the same as lastRemoteCheck.
       If neither this nor lastRemoteUpdate is provided, a sync will be attempted regardless of when the last one was
-      attempted.
+      attempted. It follows the same rules as lastRemoteUpdate for syntax.
     -->
-    <lastRemoteSync timeFormat="UNIX_EPOCH">/lastsync</lastRemoteSync>
+    <lastRemoteSync mtime="true" timeFormat="UNIX_EPOCH">/lastsync</lastRemoteSync>
     <!--
       The path that must be currently mounted for sync to proceed.
       This is required.
@@ -113,7 +117,14 @@
       It is used to determine if your upstream is "out of date" (e.g. will be skipped if its last check date is older
       than the specified amount of time). Obviously this is only checked if you have a specified lastRemoteUpdate value.
     -->
-    <upstream delayCheck="P0Y0M2DT0H0M0S">
+    <!--
+      You can optionally specify an offset via the "offset" attribute in the same format as "delayCheck" if your
+      upstream's remote files are using a different timezone instead of UTC.
+      e.g.:
+        * If your upstream uses UTC-4 for its timestamp files', you would use "-PT4H".
+        * If your upstream uses UTC+6 for its timestamp files, you would use either "+PT4H" or just "PT4H".
+    -->
+    <upstream delayCheck="P0Y0M2DT0H0M0S" offset="-PT0S">
       <!--
         The following example uses "rsync://arch.mirror.constant.com/archlinux/"
         (https://www.archlinux.org/mirrors/constant.com/1008/)

repomirror/sync.py

@@ -24,7 +24,7 @@ if os.isatty(sys.stdin.fileno()):
 else:
     _is_cron = True
 
-_duration_re = re.compile(('^P'
+_duration_re = re.compile(('^(?P<mod>[-+])?P'
                            '((?P<years>[0-9]+(\.[0-9]+)?)Y)?'
                            '((?P<months>[0-9]+(\.[0-9]+)?)M)?'
                            '((?P<days>[0-9]+(\.[0-9]+)?)D)?'
@@ -57,6 +57,19 @@ def get_owner(owner_xml):
     return(owner)
 
 
+def get_duration(duration_str):
+    r = _duration_re.search(duration_str)
+    times = {k: (float(v) if v else 0.0) for k, v in r.groupdict().items()}
+    mod = times.pop('mod')
+    if not mod:
+        mod = '+'
+    years = float(times.pop('years'))
+    months = float(times.pop('months'))
+    times['days'] = (times['days'] + (years * constants.YEAR) + (months * constants.MONTH))
+    delay = datetime.timedelta(**times)
+    return((mod, delay))
+
+
 class Args(object):
     def __init__(self, args_xml):
         self.xml = args_xml
@@ -104,12 +117,15 @@ class Mount(object):
 
 class TimestampFile(object):
     def __init__(self, ts_xml, owner_xml = None):
+        self.xml = ts_xml
         self.fmt = ts_xml.attrib.get('timeFormat', 'UNIX_EPOCH')
         if self.fmt == 'UNIX_EPOCH':
             self.fmt = '%s'
         elif self.fmt == 'MICROSECOND_EPOCH':
             self.fmt = '%s.%f'
         _logger.debug('Set timestamp format string to {0}'.format(self.fmt))
+        self.mtime = (True if self.xml.attrib.get('mtime', 'false').lower().startswith(('t', '1')) else False)
+        _logger.debug('Using mtime: {0}'.format(self.mtime))
         self.owner_xml = owner_xml
         self.owner = {}
         if self.owner_xml is not None:
@@ -126,13 +142,16 @@ class TimestampFile(object):
         else:
             path = self.path
         if os.path.isfile(path):
-            with open(path, 'r') as fh:
+            if self.mtime:
-                ts_raw = fh.read().strip()
+                timestamp = datetime.datetime.fromtimestamp(float(os.stat(path).st_mtime))
-            if '%s' in self.fmt:
-                timestamp = datetime.datetime.fromtimestamp(float(ts_raw))
             else:
-                timestamp = datetime.datetime.strptime(ts_raw, self.fmt)
+                with open(path, 'r') as fh:
-            _logger.debug('Read timestamp {0} from {1}'.format(str(timestamp), self.path))
+                    ts_raw = fh.read().strip()
+                if '%s' in self.fmt:
+                    timestamp = datetime.datetime.fromtimestamp(float(ts_raw))
+                else:
+                    timestamp = datetime.datetime.strptime(ts_raw, self.fmt)
+                _logger.debug('Read timestamp {0} from {1}'.format(str(timestamp), self.path))
         return(timestamp)
 
     def write(self):
@@ -148,6 +167,9 @@ class TimestampFile(object):
         os.chmod(self.path, mode = 0o0644)
         if self.owner:
             os.chown(self.path, **self.owner)
+        if self.mtime:
+            now = float(datetime.datetime.utcnow().timestamp())
+            os.utime(self.path, (now, now))
         _logger.debug('Wrote timestamp to {0}'.format(self.path))
         return(None)
 
@@ -161,9 +183,11 @@ class Upstream(object):
         self.path = self.xml.find('path').text
         self.dest = os.path.abspath(os.path.expanduser(dest))
         self.delay = None
+        self.offset = None
         self.owner = owner
         self.filechecks = filechecks
         self._get_delaychk()
+        self._get_offset()
         self.has_new = False
         # These are optional.
         port = self.xml.find('port')
@@ -185,18 +209,6 @@ class Upstream(object):
             self.fetcher = fetcher.FTP(self.domain, self.port, self.path, self.dest, owner = self.owner)
         self._check_conn()
 
-    def _get_delaychk(self):
-        delay = self.xml.attrib.get('delayCheck')
-        if not delay:
-            return(None)
-        r = _duration_re.search(delay)
-        times = {k: (float(v) if v else 0.0) for k, v in r.groupdict().items()}
-        years = float(times.pop('years'))
-        months = float(times.pop('months'))
-        times['days'] = (times['days'] + (years * constants.YEAR) + (months * constants.MONTH))
-        self.delay = datetime.timedelta(**times)
-        return(None)
-
     def _check_conn(self):
         sock = socket.socket()
         sock.settimeout(7)
@@ -208,6 +220,20 @@ class Upstream(object):
             self.available = False
         return(None)
 
+    def _get_delaychk(self):
+        delay = self.xml.attrib.get('delayCheck')
+        if not delay:
+            return(None)
+        mod, self.delay = get_duration(delay)
+        return(None)
+
+    def _get_offset(self):
+        offset = self.xml.attrib.get('offset')
+        if not offset:
+            return(None)
+        self.offset = get_duration(offset)
+        return(None)
+
     def sync(self):
         self.fetcher.fetch()
         return(None)