updating to support configurable delay checking and rsync exit code ignoring

This commit is contained in:
brent s 2020-07-23 14:39:47 -04:00
parent 4bdb31541b
commit bc9a653234
Signed by: bts
GPG Key ID: 8C004C2F93481F6B
4 changed files with 37 additions and 18 deletions

View File

@ -69,16 +69,15 @@
<long>links</long>
<long>hard-links</long>
<long>delete-after</long>
<long>perms</long>
<long>delay-updates</long>
<long>copy-links</long>
<long>safe-links</long>
<long>delete-excluded</long>
<long value=".*">exclude</long>
</rsyncArgs>
These arguments should be sane for most, if not all, rsync-driven repository mirroring. The last one (exclude) may
be removed in future versions.
These arguments should be sane for most, if not all, rsync-driven repository mirroring.
If arguments are provided, the defaults are overwritten so if you need the above, be sure to specify them.
See the rsync man page (rsync(1)) for more details and a listing of supported flags on your system.
See the rsync man page (rsync(1)) for more details and a listing of supported flags on your system
(§ "OPTION SUMMARY", § "OPTIONS").
-->
<rsyncArgs>
<!--
@ -94,7 +93,19 @@
A "short" option (single hyphen).
-->
<short>c</short><!-- checksum -->
<!--
An option that requires to be enclosed in quotes. (This one excludes hidden files/directories.)
-->
<long value="&quot;.*&quot;">exclude</long>
</rsyncArgs>
<!--
rsyncIgnore is an optional element that ignores certain return codes/exit statuses of rsync (otherwise they are
raised as warnings). It consists of a space-separated list of return codes that warnings should be suppressed for.
(Return code 0 never raises a warning, as it is success.)
See the rsync man page (rsync(1)) for a list of error codes and what they mean (§ "EXIT VALUES"), or refer to:
repomirror/fetcher/rsync_returns.py
-->
<rsyncIgnore returns="23 24"/>
<!--
Upstreams have an optional attribute, "delayCheck", which is an ISO 8601 duration type.
https://en.wikipedia.org/wiki/ISO_8601#Durations

View File

@ -6,11 +6,11 @@ RSYNC_DEF_ARGS = ['--recursive',
'--links',
'--hard-links',
'--delete-after',
'--perms',
'--delay-updates',
'--copy-links',
'--safe-links',
'--delete-excluded',
'--exclude=.*']
'--exclude=".*"']

# These are needed to convert years/months to timedeltas.
# The following are averaged definitions for time units *in days* according to Google Calculator.

View File

@ -25,6 +25,7 @@ class RSync(_base.BaseFetcher):
path,
dest,
rsync_args = None,
rsync_ignores = None,
owner = None,
log = True,
filechecks = None,
@ -37,6 +38,7 @@ class RSync(_base.BaseFetcher):
else:
self.rsync_args = constants.RSYNC_DEF_ARGS[:]
_logger.debug('RSync args given: {0}'.format(self.rsync_args))
self.rsync_ignores = rsync_ignores[:]
if log:
# Do I want to do this in subprocess + logging module? Or keep this?
# It looks a little ugly in the log but it makes more sense than doing it via subprocess just to write it
@ -68,10 +70,10 @@ class RSync(_base.BaseFetcher):
stderr = subprocess.PIPE)
stdout = cmd.stdout.decode('utf-8').strip()
stderr = cmd.stderr.decode('utf-8').strip()
rtrn = cmd.returncode
if stdout != '':
_logger.debug('STDOUT: {0}'.format(stdout))
if stderr != '' or cmd.returncode != 0:
rtrn = cmd.returncode
if stderr != '' or (rtrn != 0 and rtrn not in self.rsync_ignores):
err = rsync_returns.returns[rtrn]
errmsg = 'Rsync to {0}:{1} returned'.format(self.domain, self.port)
debugmsg = 'Rsync command {0} returned'.format(' '.join(cmd_str))
@ -100,10 +102,10 @@ class RSync(_base.BaseFetcher):
stderr = subprocess.PIPE)
stdout = cmd.stdout.decode('utf-8').strip()
stderr = cmd.stderr.decode('utf-8').strip()
rtrn = cmd.returncode
if stdout != '':
_logger.debug('STDOUT: {0}'.format(stdout))
if stderr != '' or cmd.returncode != 0:
rtrn = cmd.returncode
if stderr != '' or (rtrn != 0 and rtrn not in self.rsync_ignores):
err = rsync_returns.returns[rtrn]
errmsg = 'Rsync to {0}:{1} returned'.format(self.domain, self.port)
debugmsg = 'Rsync command {0} returned'.format(' '.join(cmd_str))

View File

@ -35,7 +35,6 @@ _duration_re = re.compile(('^P'
'$'))



def get_owner(owner_xml):
owner = {}
user = owner_xml.find('user')
@ -154,7 +153,7 @@ class TimestampFile(object):


class Upstream(object):
def __init__(self, upstream_xml, dest, rsync_args = None, owner = None, filechecks = None):
def __init__(self, upstream_xml, dest, rsync_args = None, owner = None, filechecks = None, rsync_ignores = None):
self.xml = upstream_xml
# These are required for all upstreams.
self.sync_type = self.xml.find('syncType').text.lower()
@ -179,6 +178,7 @@ class Upstream(object):
self.path,
self.dest,
rsync_args = rsync_args,
rsync_ignores = rsync_ignores,
filechecks = self.filechecks,
owner = self.owner)
else:
@ -225,6 +225,7 @@ class Distro(object):
'sync': None}}
self.timestamps = {}
self.rsync_args = None
self.rsync_ignores = None
self.owner = None
self.upstreams = []
self.lockfile = '/var/run/repomirror/{0}.lck'.format(self.name)
@ -243,12 +244,17 @@ class Distro(object):
e = self.xml.find('lastRemote{0}'.format(i))
if e is not None:
self.filechecks['remote'][i.lower()] = TimestampFile(e)
self.rsync_ignores = []
rsyncig_xml = self.xml.find('rsyncIgnore')
if rsyncig_xml is not None:
self.rsync_ignores = [int(i.strip()) for i in rsyncig_xml.attrib['returns'].split()]
for u in self.xml.findall('upstream'):
self.upstreams.append(Upstream(u,
self.dest,
rsync_args = self.rsync_args,
owner = self.owner,
filechecks = self.filechecks))
filechecks = self.filechecks,
rsync_ignores = self.rsync_ignores))

def check(self):
for k, v in self.filechecks['local'].items():
@ -285,11 +291,11 @@ class Distro(object):
else:
_logger.info('No remote update timestamp; syncing.')
u.has_new = True
if sync:
if sync and u.delay:
td = datetime.datetime.utcnow() - sync
if td.days > constants.DAYS_WARN:
_logger.warning(('Upstream {0} has not synced for {1} or more days; this '
'repository may be out of date.').format(u.fetcher.url, constants.DAYS_WARN))
if td.days > u.delay:
_logger.warning(('Upstream {0} has not synced for {1} or longer; this '
'repository may be out of date.').format(u.fetcher.url, u.delay))
warnings.warn('Upstream may be out of date')
return(None)