summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorbrent s <r00t@square-r00t.net>2020-07-23 14:39:47 -0400
committerbrent s <r00t@square-r00t.net>2020-07-23 14:39:47 -0400
commitbc9a6532345108c8d8930c2173341620cc4ce8db (patch)
tree1a567a0bc4d28c87c62f29bcb39a96c5df64bdd0
parent4bdb31541b324c5a6896bedf6c8413414b2b9193 (diff)
downloadRepoMirror-bc9a6532345108c8d8930c2173341620cc4ce8db.tar.xz
updating to support configurable delay checking and rsync exit code ignoring
-rw-r--r--example.config.xml21
-rw-r--r--repomirror/constants.py4
-rw-r--r--repomirror/fetcher/rsync.py10
-rw-r--r--repomirror/sync.py20
4 files changed, 37 insertions, 18 deletions
diff --git a/example.config.xml b/example.config.xml
index 1451cfe..7044500 100644
--- a/example.config.xml
+++ b/example.config.xml
@@ -69,16 +69,15 @@
<long>links</long>
<long>hard-links</long>
<long>delete-after</long>
+ <long>perms</long>
<long>delay-updates</long>
- <long>copy-links</long>
<long>safe-links</long>
<long>delete-excluded</long>
- <long value=".*">exclude</long>
</rsyncArgs>
- These arguments should be sane for most, if not all, rsync-driven repository mirroring. The last one (exclude) may
- be removed in future versions.
+ These arguments should be sane for most, if not all, rsync-driven repository mirroring.
If arguments are provided, the defaults are overwritten so if you need the above, be sure to specify them.
- See the rsync man page (rsync(1)) for more details and a listing of supported flags on your system.
+ See the rsync man page (rsync(1)) for more details and a listing of supported flags on your system
+ (§ "OPTION SUMMARY", § "OPTIONS").
-->
<rsyncArgs>
<!--
@@ -94,8 +93,20 @@
A "short" option (single hyphen).
-->
<short>c</short><!-- checksum -->
+ <!--
+ An option that requires to be enclosed in quotes. (This one excludes hidden files/directories.)
+ -->
+ <long value="&quot;.*&quot;">exclude</long>
</rsyncArgs>
<!--
+ rsyncIgnore is an optional element that ignores certain return codes/exit statuses of rsync (otherwise they are
+ raised as warnings). It consists of a space-separated list of return codes that warnings should be suppressed for.
+ (Return code 0 never raises a warning, as it is success.)
+ See the rsync man page (rsync(1)) for a list of error codes and what they mean (§ "EXIT VALUES"), or refer to:
+ repomirror/fetcher/rsync_returns.py
+ -->
+ <rsyncIgnore returns="23 24"/>
+ <!--
Upstreams have an optional attribute, "delayCheck", which is an ISO 8601 duration type.
https://en.wikipedia.org/wiki/ISO_8601#Durations
e.g. 5 minutes would be P0Y0M0DT0H5M0S or just PT5M.
diff --git a/repomirror/constants.py b/repomirror/constants.py
index 207ab87..3f20d45 100644
--- a/repomirror/constants.py
+++ b/repomirror/constants.py
@@ -6,11 +6,11 @@ RSYNC_DEF_ARGS = ['--recursive',
'--links',
'--hard-links',
'--delete-after',
+ '--perms',
'--delay-updates',
- '--copy-links',
'--safe-links',
'--delete-excluded',
- '--exclude=.*']
+ '--exclude=".*"']
# These are needed to convert years/months to timedeltas.
# The following are averaged definitions for time units *in days* according to Google Calculator.
diff --git a/repomirror/fetcher/rsync.py b/repomirror/fetcher/rsync.py
index 93d63dd..5b673d3 100644
--- a/repomirror/fetcher/rsync.py
+++ b/repomirror/fetcher/rsync.py
@@ -25,6 +25,7 @@ class RSync(_base.BaseFetcher):
path,
dest,
rsync_args = None,
+ rsync_ignores = None,
owner = None,
log = True,
filechecks = None,
@@ -37,6 +38,7 @@ class RSync(_base.BaseFetcher):
else:
self.rsync_args = constants.RSYNC_DEF_ARGS[:]
_logger.debug('RSync args given: {0}'.format(self.rsync_args))
+ self.rsync_ignores = rsync_ignores[:]
if log:
# Do I want to do this in subprocess + logging module? Or keep this?
# It looks a little ugly in the log but it makes more sense than doing it via subprocess just to write it
@@ -68,10 +70,10 @@ class RSync(_base.BaseFetcher):
stderr = subprocess.PIPE)
stdout = cmd.stdout.decode('utf-8').strip()
stderr = cmd.stderr.decode('utf-8').strip()
+ rtrn = cmd.returncode
if stdout != '':
_logger.debug('STDOUT: {0}'.format(stdout))
- if stderr != '' or cmd.returncode != 0:
- rtrn = cmd.returncode
+ if stderr != '' or (rtrn != 0 and rtrn not in self.rsync_ignores):
err = rsync_returns.returns[rtrn]
errmsg = 'Rsync to {0}:{1} returned'.format(self.domain, self.port)
debugmsg = 'Rsync command {0} returned'.format(' '.join(cmd_str))
@@ -100,10 +102,10 @@ class RSync(_base.BaseFetcher):
stderr = subprocess.PIPE)
stdout = cmd.stdout.decode('utf-8').strip()
stderr = cmd.stderr.decode('utf-8').strip()
+ rtrn = cmd.returncode
if stdout != '':
_logger.debug('STDOUT: {0}'.format(stdout))
- if stderr != '' or cmd.returncode != 0:
- rtrn = cmd.returncode
+ if stderr != '' or (rtrn != 0 and rtrn not in self.rsync_ignores):
err = rsync_returns.returns[rtrn]
errmsg = 'Rsync to {0}:{1} returned'.format(self.domain, self.port)
debugmsg = 'Rsync command {0} returned'.format(' '.join(cmd_str))
diff --git a/repomirror/sync.py b/repomirror/sync.py
index 0226b3a..0e249ee 100644
--- a/repomirror/sync.py
+++ b/repomirror/sync.py
@@ -35,7 +35,6 @@ _duration_re = re.compile(('^P'
'$'))
-
def get_owner(owner_xml):
owner = {}
user = owner_xml.find('user')
@@ -154,7 +153,7 @@ class TimestampFile(object):
class Upstream(object):
- def __init__(self, upstream_xml, dest, rsync_args = None, owner = None, filechecks = None):
+ def __init__(self, upstream_xml, dest, rsync_args = None, owner = None, filechecks = None, rsync_ignores = None):
self.xml = upstream_xml
# These are required for all upstreams.
self.sync_type = self.xml.find('syncType').text.lower()
@@ -179,6 +178,7 @@ class Upstream(object):
self.path,
self.dest,
rsync_args = rsync_args,
+ rsync_ignores = rsync_ignores,
filechecks = self.filechecks,
owner = self.owner)
else:
@@ -225,6 +225,7 @@ class Distro(object):
'sync': None}}
self.timestamps = {}
self.rsync_args = None
+ self.rsync_ignores = None
self.owner = None
self.upstreams = []
self.lockfile = '/var/run/repomirror/{0}.lck'.format(self.name)
@@ -243,12 +244,17 @@ class Distro(object):
e = self.xml.find('lastRemote{0}'.format(i))
if e is not None:
self.filechecks['remote'][i.lower()] = TimestampFile(e)
+ self.rsync_ignores = []
+ rsyncig_xml = self.xml.find('rsyncIgnore')
+ if rsyncig_xml is not None:
+ self.rsync_ignores = [int(i.strip()) for i in rsyncig_xml.attrib['returns'].split()]
for u in self.xml.findall('upstream'):
self.upstreams.append(Upstream(u,
self.dest,
rsync_args = self.rsync_args,
owner = self.owner,
- filechecks = self.filechecks))
+ filechecks = self.filechecks,
+ rsync_ignores = self.rsync_ignores))
def check(self):
for k, v in self.filechecks['local'].items():
@@ -285,11 +291,11 @@ class Distro(object):
else:
_logger.info('No remote update timestamp; syncing.')
u.has_new = True
- if sync:
+ if sync and u.delay:
td = datetime.datetime.utcnow() - sync
- if td.days > constants.DAYS_WARN:
- _logger.warning(('Upstream {0} has not synced for {1} or more days; this '
- 'repository may be out of date.').format(u.fetcher.url, constants.DAYS_WARN))
+ if td.days > u.delay:
+ _logger.warning(('Upstream {0} has not synced for {1} or longer; this '
+ 'repository may be out of date.').format(u.fetcher.url, u.delay))
warnings.warn('Upstream may be out of date')
return(None)