adding in some delay checking calculation and parsing of xs:duration types.
This commit is contained in:
brent s
parent
a3203ab03a
commit
4bdb31541b
GPG Key ID:
8C004C2F93481F6B
@ -95,7 +95,14 @@
|
||||
-->
|
||||
<short>c</short><!-- checksum -->
|
||||
</rsyncArgs>
|
||||
<upstream>
|
||||
<!--
|
||||
Upstreams have an optional attribute, "delayCheck", which is an ISO 8601 duration type.
|
||||
https://en.wikipedia.org/wiki/ISO_8601#Durations
|
||||
e.g. 5 minutes would be P0Y0M0DT0H5M0S or just PT5M.
|
||||
It is used to determine if your upstream is "out of date" (e.g. will be skipped if its last check date is older
|
||||
than the specified amount of time). Obviously this is only checked if you have a specified lastRemoteUpdate value.
|
||||
-->
|
||||
<upstream delayCheck="P0Y0M2DT0H0M0S">
|
||||
<!--
|
||||
The following example uses "rsync://arch.mirror.constant.com/archlinux/"
|
||||
(https://www.archlinux.org/mirrors/constant.com/1008/)
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
PROTO_DEF_PORTS = {'ftp': 21,
|
||||
'rsync': 873}
|
||||
|
||||
RSYNC_DEF_ARGS = ['--recursive',
|
||||
'--times',
|
||||
'--links',
|
||||
@ -10,7 +11,17 @@ RSYNC_DEF_ARGS = ['--recursive',
|
||||
'--safe-links',
|
||||
'--delete-excluded',
|
||||
'--exclude=.*']
|
||||
# How many days an upstream should have last synced by before it's considered stale.
|
||||
## TODO: make this part of the upstream config? repo config?
|
||||
DAYS_WARN = 2
|
||||
VERSION = '1.0.4'
|
||||
|
||||
# These are needed to convert years/months to timedeltas.
|
||||
# The following are averaged definitions for time units *in days* according to Google Calculator.
|
||||
YEAR = 365.2422
|
||||
MONTH = 30.4167
|
||||
# The following are approximations based on ISO 8601 defintions *in days*.
|
||||
# https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm
|
||||
# YEAR = 365.25
|
||||
# MONTH = 30.6
|
||||
|
||||
# We no longer do this by default.
|
||||
# # How many days an upstream should have last synced by before it's considered stale.
|
||||
# DAYS_WARN = 2
|
||||
VERSION = '1.1.0'
|
||||
|
||||
@ -3,6 +3,7 @@ import logging
|
||||
import pwd
|
||||
import grp
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import sys
|
||||
import warnings
|
||||
@ -23,6 +24,17 @@ if os.isatty(sys.stdin.fileno()):
|
||||
else:
|
||||
_is_cron = True
|
||||
|
||||
_duration_re = re.compile(('^P'
|
||||
'((?P<years>[0-9]+(\.[0-9]+)?)Y)?'
|
||||
'((?P<months>[0-9]+(\.[0-9]+)?)M)?'
|
||||
'((?P<days>[0-9]+(\.[0-9]+)?)D)?'
|
||||
'T?'
|
||||
'((?P<hours>[0-9]+(\.[0-9]+)?)H)?'
|
||||
'((?P<minutes>[0-9]+(\.[0-9]+)?)M)?'
|
||||
'((?P<seconds>[0-9]+(\.[0-9]+)?)S)?'
|
||||
'$'))
|
||||
|
||||
|
||||
|
||||
def get_owner(owner_xml):
|
||||
owner = {}
|
||||
@ -149,8 +161,10 @@ class Upstream(object):
|
||||
self.domain = self.xml.find('domain').text
|
||||
self.path = self.xml.find('path').text
|
||||
self.dest = os.path.abspath(os.path.expanduser(dest))
|
||||
self.delay = None
|
||||
self.owner = owner
|
||||
self.filechecks = filechecks
|
||||
self._get_delaychk()
|
||||
self.has_new = False
|
||||
# These are optional.
|
||||
port = self.xml.find('port')
|
||||
@ -171,6 +185,18 @@ class Upstream(object):
|
||||
self.fetcher = fetcher.FTP(self.domain, self.port, self.path, self.dest, owner = self.owner)
|
||||
self._check_conn()
|
||||
|
||||
def _get_delaychk(self):
|
||||
delay = self.xml.attrib.get('delayCheck')
|
||||
if not delay:
|
||||
return(None)
|
||||
r = _duration_re.search(delay)
|
||||
times = {k: (float(v) if v else 0.0) for k, v in r.groupdict().items()}
|
||||
years = float(times.pop('years'))
|
||||
months = float(times.pop('months'))
|
||||
times['days'] = (times['days'] + (years * constants.YEAR) + (months * constants.MONTH))
|
||||
self.delay = datetime.timedelta(**times)
|
||||
return(None)
|
||||
|
||||
def _check_conn(self):
|
||||
sock = socket.socket()
|
||||
sock.settimeout(7)
|
||||
|
||||
56
reposync
56
reposync
@ -1,56 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
##
|
||||
##
|
||||
_cur_file = pathlib.Path(os.path.abspath(os.path.expanduser(__file__))).resolve()
|
||||
_cur_path = os.path.dirname(_cur_file)
|
||||
sys.path.insert(1, _cur_path)
|
||||
import repomirror
|
||||
|
||||
|
||||
if os.geteuid() == 0:
|
||||
_def_logdir = '/var/log/repo'
|
||||
else:
|
||||
_def_logdir = '~/.cache/repologs'
|
||||
|
||||
|
||||
def parseArgs():
|
||||
args = argparse.ArgumentParser(description = 'Sync repositories for various distributions to local paths')
|
||||
args.add_argument('-c', '--config',
|
||||
default = '~/.config/repomirror.xml',
|
||||
dest = 'cfg',
|
||||
help = ('The path to the config file. If it does not exist, a bare version will be created. '
|
||||
'Default: ~/.config/repomirror.xml'))
|
||||
# args.add_argument('-n', '--dry-run',
|
||||
# action = 'store_true',
|
||||
# dest = 'dummy',
|
||||
# help = ('If specified, do not actually sync anything (other than timestamp files if '
|
||||
# 'applicable to determine logic); do not actually sync any repositories. Useful for '
|
||||
# 'generating logs to determine potential issues before they happen'))
|
||||
args.add_argument('-d', '--distro',
|
||||
dest = 'distro',
|
||||
action = 'append',
|
||||
help = ('If specified, only sync the specified distro in the config file (otherwise sync all '
|
||||
'specified). May be given multiple times'))
|
||||
args.add_argument('-l', '--logdir',
|
||||
default = _def_logdir,
|
||||
dest = 'logdir',
|
||||
help = ('The path to the directory where logs should be written. The actual log files will be '
|
||||
'named after their respective distro names in the config file. '
|
||||
'Default: {0}'.format(_def_logdir)))
|
||||
return(args)
|
||||
|
||||
|
||||
def main():
|
||||
args = parseArgs().parse_args()
|
||||
r = repomirror.Sync(**vars(args))
|
||||
r.sync()
|
||||
return(None)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -1 +0,0 @@
|
||||
reposync
|
||||
56
reposync.py
Executable file
56
reposync.py
Executable file
@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
##
|
||||
##
|
||||
_cur_file = pathlib.Path(os.path.abspath(os.path.expanduser(__file__))).resolve()
|
||||
_cur_path = os.path.dirname(_cur_file)
|
||||
sys.path.insert(1, _cur_path)
|
||||
import repomirror
|
||||
|
||||
|
||||
if os.geteuid() == 0:
|
||||
_def_logdir = '/var/log/repo'
|
||||
else:
|
||||
_def_logdir = '~/.cache/repologs'
|
||||
|
||||
|
||||
def parseArgs():
|
||||
args = argparse.ArgumentParser(description = 'Sync repositories for various distributions to local paths')
|
||||
args.add_argument('-c', '--config',
|
||||
default = '~/.config/repomirror.xml',
|
||||
dest = 'cfg',
|
||||
help = ('The path to the config file. If it does not exist, a bare version will be created. '
|
||||
'Default: ~/.config/repomirror.xml'))
|
||||
# args.add_argument('-n', '--dry-run',
|
||||
# action = 'store_true',
|
||||
# dest = 'dummy',
|
||||
# help = ('If specified, do not actually sync anything (other than timestamp files if '
|
||||
# 'applicable to determine logic); do not actually sync any repositories. Useful for '
|
||||
# 'generating logs to determine potential issues before they happen'))
|
||||
args.add_argument('-d', '--distro',
|
||||
dest = 'distro',
|
||||
action = 'append',
|
||||
help = ('If specified, only sync the specified distro in the config file (otherwise sync all '
|
||||
'specified). May be given multiple times'))
|
||||
args.add_argument('-l', '--logdir',
|
||||
default = _def_logdir,
|
||||
dest = 'logdir',
|
||||
help = ('The path to the directory where logs should be written. The actual log files will be '
|
||||
'named after their respective distro names in the config file. '
|
||||
'Default: {0}'.format(_def_logdir)))
|
||||
return(args)
|
||||
|
||||
|
||||
def main():
|
||||
args = parseArgs().parse_args()
|
||||
r = repomirror.Sync(**vars(args))
|
||||
r.sync()
|
||||
return(None)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
Reference in New Issue
Block a user