updating to support configurable delay checking and rsync exit code ignoring
This commit is contained in:
brent s
parent
4bdb31541b
commit
bc9a653234
GPG Key ID:
8C004C2F93481F6B
|
|
@ -69,16 +69,15 @@
|
|||
<long>links</long>
|
||||
<long>hard-links</long>
|
||||
<long>delete-after</long>
|
||||
<long>perms</long>
|
||||
<long>delay-updates</long>
|
||||
<long>copy-links</long>
|
||||
<long>safe-links</long>
|
||||
<long>delete-excluded</long>
|
||||
<long value=".*">exclude</long>
|
||||
</rsyncArgs>
|
||||
These arguments should be sane for most, if not all, rsync-driven repository mirroring. The last one (exclude) may
|
||||
be removed in future versions.
|
||||
These arguments should be sane for most, if not all, rsync-driven repository mirroring.
|
||||
If arguments are provided, the defaults are overwritten so if you need the above, be sure to specify them.
|
||||
See the rsync man page (rsync(1)) for more details and a listing of supported flags on your system.
|
||||
See the rsync man page (rsync(1)) for more details and a listing of supported flags on your system
|
||||
(§ "OPTION SUMMARY", § "OPTIONS").
|
||||
-->
|
||||
<rsyncArgs>
|
||||
<!--
|
||||
|
|
@ -94,7 +93,19 @@
|
|||
A "short" option (single hyphen).
|
||||
-->
|
||||
<short>c</short><!-- checksum -->
|
||||
<!--
|
||||
An option that requires to be enclosed in quotes. (This one excludes hidden files/directories.)
|
||||
-->
|
||||
<long value="".*"">exclude</long>
|
||||
</rsyncArgs>
|
||||
<!--
|
||||
rsyncIgnore is an optional element that ignores certain return codes/exit statuses of rsync (otherwise they are
|
||||
raised as warnings). It consists of a space-separated list of return codes that warnings should be suppressed for.
|
||||
(Return code 0 never raises a warning, as it is success.)
|
||||
See the rsync man page (rsync(1)) for a list of error codes and what they mean (§ "EXIT VALUES"), or refer to:
|
||||
repomirror/fetcher/rsync_returns.py
|
||||
-->
|
||||
<rsyncIgnore returns="23 24"/>
|
||||
<!--
|
||||
Upstreams have an optional attribute, "delayCheck", which is an ISO 8601 duration type.
|
||||
https://en.wikipedia.org/wiki/ISO_8601#Durations
|
||||
|
|
|
|||
|
|
@ -6,11 +6,11 @@ RSYNC_DEF_ARGS = ['--recursive',
|
|||
'--links',
|
||||
'--hard-links',
|
||||
'--delete-after',
|
||||
'--perms',
|
||||
'--delay-updates',
|
||||
'--copy-links',
|
||||
'--safe-links',
|
||||
'--delete-excluded',
|
||||
'--exclude=.*']
|
||||
'--exclude=".*"']
|
||||
|
||||
# These are needed to convert years/months to timedeltas.
|
||||
# The following are averaged definitions for time units *in days* according to Google Calculator.
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ class RSync(_base.BaseFetcher):
|
|||
path,
|
||||
dest,
|
||||
rsync_args = None,
|
||||
rsync_ignores = None,
|
||||
owner = None,
|
||||
log = True,
|
||||
filechecks = None,
|
||||
|
|
@ -37,6 +38,7 @@ class RSync(_base.BaseFetcher):
|
|||
else:
|
||||
self.rsync_args = constants.RSYNC_DEF_ARGS[:]
|
||||
_logger.debug('RSync args given: {0}'.format(self.rsync_args))
|
||||
self.rsync_ignores = rsync_ignores[:]
|
||||
if log:
|
||||
# Do I want to do this in subprocess + logging module? Or keep this?
|
||||
# It looks a little ugly in the log but it makes more sense than doing it via subprocess just to write it
|
||||
|
|
@ -68,10 +70,10 @@ class RSync(_base.BaseFetcher):
|
|||
stderr = subprocess.PIPE)
|
||||
stdout = cmd.stdout.decode('utf-8').strip()
|
||||
stderr = cmd.stderr.decode('utf-8').strip()
|
||||
rtrn = cmd.returncode
|
||||
if stdout != '':
|
||||
_logger.debug('STDOUT: {0}'.format(stdout))
|
||||
if stderr != '' or cmd.returncode != 0:
|
||||
rtrn = cmd.returncode
|
||||
if stderr != '' or (rtrn != 0 and rtrn not in self.rsync_ignores):
|
||||
err = rsync_returns.returns[rtrn]
|
||||
errmsg = 'Rsync to {0}:{1} returned'.format(self.domain, self.port)
|
||||
debugmsg = 'Rsync command {0} returned'.format(' '.join(cmd_str))
|
||||
|
|
@ -100,10 +102,10 @@ class RSync(_base.BaseFetcher):
|
|||
stderr = subprocess.PIPE)
|
||||
stdout = cmd.stdout.decode('utf-8').strip()
|
||||
stderr = cmd.stderr.decode('utf-8').strip()
|
||||
rtrn = cmd.returncode
|
||||
if stdout != '':
|
||||
_logger.debug('STDOUT: {0}'.format(stdout))
|
||||
if stderr != '' or cmd.returncode != 0:
|
||||
rtrn = cmd.returncode
|
||||
if stderr != '' or (rtrn != 0 and rtrn not in self.rsync_ignores):
|
||||
err = rsync_returns.returns[rtrn]
|
||||
errmsg = 'Rsync to {0}:{1} returned'.format(self.domain, self.port)
|
||||
debugmsg = 'Rsync command {0} returned'.format(' '.join(cmd_str))
|
||||
|
|
|
|||
|
|
@ -35,7 +35,6 @@ _duration_re = re.compile(('^P'
|
|||
'$'))
|
||||
|
||||
|
||||
|
||||
def get_owner(owner_xml):
|
||||
owner = {}
|
||||
user = owner_xml.find('user')
|
||||
|
|
@ -154,7 +153,7 @@ class TimestampFile(object):
|
|||
|
||||
|
||||
class Upstream(object):
|
||||
def __init__(self, upstream_xml, dest, rsync_args = None, owner = None, filechecks = None):
|
||||
def __init__(self, upstream_xml, dest, rsync_args = None, owner = None, filechecks = None, rsync_ignores = None):
|
||||
self.xml = upstream_xml
|
||||
# These are required for all upstreams.
|
||||
self.sync_type = self.xml.find('syncType').text.lower()
|
||||
|
|
@ -179,6 +178,7 @@ class Upstream(object):
|
|||
self.path,
|
||||
self.dest,
|
||||
rsync_args = rsync_args,
|
||||
rsync_ignores = rsync_ignores,
|
||||
filechecks = self.filechecks,
|
||||
owner = self.owner)
|
||||
else:
|
||||
|
|
@ -225,6 +225,7 @@ class Distro(object):
|
|||
'sync': None}}
|
||||
self.timestamps = {}
|
||||
self.rsync_args = None
|
||||
self.rsync_ignores = None
|
||||
self.owner = None
|
||||
self.upstreams = []
|
||||
self.lockfile = '/var/run/repomirror/{0}.lck'.format(self.name)
|
||||
|
|
@ -243,12 +244,17 @@ class Distro(object):
|
|||
e = self.xml.find('lastRemote{0}'.format(i))
|
||||
if e is not None:
|
||||
self.filechecks['remote'][i.lower()] = TimestampFile(e)
|
||||
self.rsync_ignores = []
|
||||
rsyncig_xml = self.xml.find('rsyncIgnore')
|
||||
if rsyncig_xml is not None:
|
||||
self.rsync_ignores = [int(i.strip()) for i in rsyncig_xml.attrib['returns'].split()]
|
||||
for u in self.xml.findall('upstream'):
|
||||
self.upstreams.append(Upstream(u,
|
||||
self.dest,
|
||||
rsync_args = self.rsync_args,
|
||||
owner = self.owner,
|
||||
filechecks = self.filechecks))
|
||||
filechecks = self.filechecks,
|
||||
rsync_ignores = self.rsync_ignores))
|
||||
|
||||
def check(self):
|
||||
for k, v in self.filechecks['local'].items():
|
||||
|
|
@ -285,11 +291,11 @@ class Distro(object):
|
|||
else:
|
||||
_logger.info('No remote update timestamp; syncing.')
|
||||
u.has_new = True
|
||||
if sync:
|
||||
if sync and u.delay:
|
||||
td = datetime.datetime.utcnow() - sync
|
||||
if td.days > constants.DAYS_WARN:
|
||||
_logger.warning(('Upstream {0} has not synced for {1} or more days; this '
|
||||
'repository may be out of date.').format(u.fetcher.url, constants.DAYS_WARN))
|
||||
if td.days > u.delay:
|
||||
_logger.warning(('Upstream {0} has not synced for {1} or longer; this '
|
||||
'repository may be out of date.').format(u.fetcher.url, u.delay))
|
||||
warnings.warn('Upstream may be out of date')
|
||||
return(None)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue