summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorbrent s <r00t@square-r00t.net>2020-07-23 12:05:36 -0400
committerbrent s <r00t@square-r00t.net>2020-07-23 12:05:36 -0400
commit4bdb31541b324c5a6896bedf6c8413414b2b9193 (patch)
tree0052159850ee45d44bbad8e60aa39df72fa5db88
parenta3203ab03a1124036191079e0acf3d7ff04fe234 (diff)
downloadRepoMirror-4bdb31541b324c5a6896bedf6c8413414b2b9193.tar.xz
adding in some delay checking calculation and parsing of xs:duration types.
-rw-r--r--example.config.xml9
-rw-r--r--repomirror/constants.py19
-rw-r--r--repomirror/sync.py26
l---------[-rwxr-xr-x]reposync57
-rwxr-xr-x[l---------]reposync.py57
5 files changed, 106 insertions, 62 deletions
diff --git a/example.config.xml b/example.config.xml
index 9301809..1451cfe 100644
--- a/example.config.xml
+++ b/example.config.xml
@@ -95,7 +95,14 @@
-->
<short>c</short><!-- checksum -->
</rsyncArgs>
- <upstream>
+ <!--
+ Upstreams have an optional attribute, "delayCheck", which is an ISO 8601 duration type.
+ https://en.wikipedia.org/wiki/ISO_8601#Durations
+ e.g. 5 minutes would be P0Y0M0DT0H5M0S or just PT5M.
+ It is used to determine if your upstream is "out of date" (e.g. will be skipped if its last check date is older
+ than the specified amount of time). Obviously this is only checked if you have a specified lastRemoteUpdate value.
+ -->
+ <upstream delayCheck="P0Y0M2DT0H0M0S">
<!--
The following example uses "rsync://arch.mirror.constant.com/archlinux/"
(https://www.archlinux.org/mirrors/constant.com/1008/)
diff --git a/repomirror/constants.py b/repomirror/constants.py
index d11525a..207ab87 100644
--- a/repomirror/constants.py
+++ b/repomirror/constants.py
@@ -1,5 +1,6 @@
PROTO_DEF_PORTS = {'ftp': 21,
'rsync': 873}
+
RSYNC_DEF_ARGS = ['--recursive',
'--times',
'--links',
@@ -10,7 +11,17 @@ RSYNC_DEF_ARGS = ['--recursive',
'--safe-links',
'--delete-excluded',
'--exclude=.*']
-# How many days an upstream should have last synced by before it's considered stale.
-## TODO: make this part of the upstream config? repo config?
-DAYS_WARN = 2
-VERSION = '1.0.4'
+
+# These are needed to convert years/months to timedeltas.
+# The following are averaged definitions for time units *in days* according to Google Calculator.
+YEAR = 365.2422
+MONTH = 30.4167
+# The following are approximations based on ISO 8601 defintions *in days*.
+# https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm
+# YEAR = 365.25
+# MONTH = 30.6
+
+# We no longer do this by default.
+# # How many days an upstream should have last synced by before it's considered stale.
+# DAYS_WARN = 2
+VERSION = '1.1.0'
diff --git a/repomirror/sync.py b/repomirror/sync.py
index 3e9b16b..0226b3a 100644
--- a/repomirror/sync.py
+++ b/repomirror/sync.py
@@ -3,6 +3,7 @@ import logging
import pwd
import grp
import os
+import re
import socket
import sys
import warnings
@@ -23,6 +24,17 @@ if os.isatty(sys.stdin.fileno()):
else:
_is_cron = True
+_duration_re = re.compile(('^P'
+ '((?P<years>[0-9]+(\.[0-9]+)?)Y)?'
+ '((?P<months>[0-9]+(\.[0-9]+)?)M)?'
+ '((?P<days>[0-9]+(\.[0-9]+)?)D)?'
+ 'T?'
+ '((?P<hours>[0-9]+(\.[0-9]+)?)H)?'
+ '((?P<minutes>[0-9]+(\.[0-9]+)?)M)?'
+ '((?P<seconds>[0-9]+(\.[0-9]+)?)S)?'
+ '$'))
+
+
def get_owner(owner_xml):
owner = {}
@@ -149,8 +161,10 @@ class Upstream(object):
self.domain = self.xml.find('domain').text
self.path = self.xml.find('path').text
self.dest = os.path.abspath(os.path.expanduser(dest))
+ self.delay = None
self.owner = owner
self.filechecks = filechecks
+ self._get_delaychk()
self.has_new = False
# These are optional.
port = self.xml.find('port')
@@ -171,6 +185,18 @@ class Upstream(object):
self.fetcher = fetcher.FTP(self.domain, self.port, self.path, self.dest, owner = self.owner)
self._check_conn()
+ def _get_delaychk(self):
+ delay = self.xml.attrib.get('delayCheck')
+ if not delay:
+ return(None)
+ r = _duration_re.search(delay)
+ times = {k: (float(v) if v else 0.0) for k, v in r.groupdict().items()}
+ years = float(times.pop('years'))
+ months = float(times.pop('months'))
+ times['days'] = (times['days'] + (years * constants.YEAR) + (months * constants.MONTH))
+ self.delay = datetime.timedelta(**times)
+ return(None)
+
def _check_conn(self):
sock = socket.socket()
sock.settimeout(7)
diff --git a/reposync b/reposync
index a78bde5..97ffdba 100755..120000
--- a/reposync
+++ b/reposync
@@ -1,56 +1 @@
-#!/usr/bin/env python3
-
-import argparse
-import os
-import pathlib
-import sys
-##
-##
-_cur_file = pathlib.Path(os.path.abspath(os.path.expanduser(__file__))).resolve()
-_cur_path = os.path.dirname(_cur_file)
-sys.path.insert(1, _cur_path)
-import repomirror
-
-
-if os.geteuid() == 0:
- _def_logdir = '/var/log/repo'
-else:
- _def_logdir = '~/.cache/repologs'
-
-
-def parseArgs():
- args = argparse.ArgumentParser(description = 'Sync repositories for various distributions to local paths')
- args.add_argument('-c', '--config',
- default = '~/.config/repomirror.xml',
- dest = 'cfg',
- help = ('The path to the config file. If it does not exist, a bare version will be created. '
- 'Default: ~/.config/repomirror.xml'))
- # args.add_argument('-n', '--dry-run',
- # action = 'store_true',
- # dest = 'dummy',
- # help = ('If specified, do not actually sync anything (other than timestamp files if '
- # 'applicable to determine logic); do not actually sync any repositories. Useful for '
- # 'generating logs to determine potential issues before they happen'))
- args.add_argument('-d', '--distro',
- dest = 'distro',
- action = 'append',
- help = ('If specified, only sync the specified distro in the config file (otherwise sync all '
- 'specified). May be given multiple times'))
- args.add_argument('-l', '--logdir',
- default = _def_logdir,
- dest = 'logdir',
- help = ('The path to the directory where logs should be written. The actual log files will be '
- 'named after their respective distro names in the config file. '
- 'Default: {0}'.format(_def_logdir)))
- return(args)
-
-
-def main():
- args = parseArgs().parse_args()
- r = repomirror.Sync(**vars(args))
- r.sync()
- return(None)
-
-
-if __name__ == '__main__':
- main()
+reposync.py \ No newline at end of file
diff --git a/reposync.py b/reposync.py
index 9b00562..a78bde5 120000..100755
--- a/reposync.py
+++ b/reposync.py
@@ -1 +1,56 @@
-reposync \ No newline at end of file
+#!/usr/bin/env python3
+
+import argparse
+import os
+import pathlib
+import sys
+##
+##
+_cur_file = pathlib.Path(os.path.abspath(os.path.expanduser(__file__))).resolve()
+_cur_path = os.path.dirname(_cur_file)
+sys.path.insert(1, _cur_path)
+import repomirror
+
+
+if os.geteuid() == 0:
+ _def_logdir = '/var/log/repo'
+else:
+ _def_logdir = '~/.cache/repologs'
+
+
+def parseArgs():
+ args = argparse.ArgumentParser(description = 'Sync repositories for various distributions to local paths')
+ args.add_argument('-c', '--config',
+ default = '~/.config/repomirror.xml',
+ dest = 'cfg',
+ help = ('The path to the config file. If it does not exist, a bare version will be created. '
+ 'Default: ~/.config/repomirror.xml'))
+ # args.add_argument('-n', '--dry-run',
+ # action = 'store_true',
+ # dest = 'dummy',
+ # help = ('If specified, do not actually sync anything (other than timestamp files if '
+ # 'applicable to determine logic); do not actually sync any repositories. Useful for '
+ # 'generating logs to determine potential issues before they happen'))
+ args.add_argument('-d', '--distro',
+ dest = 'distro',
+ action = 'append',
+ help = ('If specified, only sync the specified distro in the config file (otherwise sync all '
+ 'specified). May be given multiple times'))
+ args.add_argument('-l', '--logdir',
+ default = _def_logdir,
+ dest = 'logdir',
+ help = ('The path to the directory where logs should be written. The actual log files will be '
+ 'named after their respective distro names in the config file. '
+ 'Default: {0}'.format(_def_logdir)))
+ return(args)
+
+
+def main():
+ args = parseArgs().parse_args()
+ r = repomirror.Sync(**vars(args))
+ r.sync()
+ return(None)
+
+
+if __name__ == '__main__':
+ main()