adding in some delay checking calculation and parsing of xs:duration types.

This commit is contained in:
brent s 2020-07-23 12:05:36 -04:00
parent a3203ab03a
commit 4bdb31541b
Signed by: bts
GPG Key ID: 8C004C2F93481F6B
5 changed files with 106 additions and 62 deletions

View File

@ -95,7 +95,14 @@
--> -->
<short>c</short><!-- checksum --> <short>c</short><!-- checksum -->
</rsyncArgs> </rsyncArgs>
<upstream> <!--
Upstreams have an optional attribute, "delayCheck", which is an ISO 8601 duration type.
https://en.wikipedia.org/wiki/ISO_8601#Durations
e.g. 5 minutes would be P0Y0M0DT0H5M0S or just PT5M.
It is used to determine if your upstream is "out of date" (e.g. will be skipped if its last check date is older
than the specified amount of time). Obviously this is only checked if you have a specified lastRemoteUpdate value.
-->
<upstream delayCheck="P0Y0M2DT0H0M0S">
<!-- <!--
The following example uses "rsync://arch.mirror.constant.com/archlinux/" The following example uses "rsync://arch.mirror.constant.com/archlinux/"
(https://www.archlinux.org/mirrors/constant.com/1008/) (https://www.archlinux.org/mirrors/constant.com/1008/)

View File

@ -1,5 +1,6 @@
PROTO_DEF_PORTS = {'ftp': 21, PROTO_DEF_PORTS = {'ftp': 21,
'rsync': 873} 'rsync': 873}

RSYNC_DEF_ARGS = ['--recursive', RSYNC_DEF_ARGS = ['--recursive',
'--times', '--times',
'--links', '--links',
@ -10,7 +11,17 @@ RSYNC_DEF_ARGS = ['--recursive',
'--safe-links', '--safe-links',
'--delete-excluded', '--delete-excluded',
'--exclude=.*'] '--exclude=.*']
# How many days an upstream should have last synced by before it's considered stale.
## TODO: make this part of the upstream config? repo config? # These are needed to convert years/months to timedeltas.
DAYS_WARN = 2 # The following are averaged definitions for time units *in days* according to Google Calculator.
VERSION = '1.0.4' YEAR = 365.2422
MONTH = 30.4167
# The following are approximations based on ISO 8601 defintions *in days*.
# https://webspace.science.uu.nl/~gent0113/calendar/isocalendar.htm
# YEAR = 365.25
# MONTH = 30.6

# We no longer do this by default.
# # How many days an upstream should have last synced by before it's considered stale.
# DAYS_WARN = 2
VERSION = '1.1.0'

View File

@ -3,6 +3,7 @@ import logging
import pwd import pwd
import grp import grp
import os import os
import re
import socket import socket
import sys import sys
import warnings import warnings
@ -23,6 +24,17 @@ if os.isatty(sys.stdin.fileno()):
else: else:
_is_cron = True _is_cron = True


_duration_re = re.compile(('^P'
'((?P<years>[0-9]+(\.[0-9]+)?)Y)?'
'((?P<months>[0-9]+(\.[0-9]+)?)M)?'
'((?P<days>[0-9]+(\.[0-9]+)?)D)?'
'T?'
'((?P<hours>[0-9]+(\.[0-9]+)?)H)?'
'((?P<minutes>[0-9]+(\.[0-9]+)?)M)?'
'((?P<seconds>[0-9]+(\.[0-9]+)?)S)?'
'$'))




def get_owner(owner_xml): def get_owner(owner_xml):
owner = {} owner = {}
@ -149,8 +161,10 @@ class Upstream(object):
self.domain = self.xml.find('domain').text self.domain = self.xml.find('domain').text
self.path = self.xml.find('path').text self.path = self.xml.find('path').text
self.dest = os.path.abspath(os.path.expanduser(dest)) self.dest = os.path.abspath(os.path.expanduser(dest))
self.delay = None
self.owner = owner self.owner = owner
self.filechecks = filechecks self.filechecks = filechecks
self._get_delaychk()
self.has_new = False self.has_new = False
# These are optional. # These are optional.
port = self.xml.find('port') port = self.xml.find('port')
@ -171,6 +185,18 @@ class Upstream(object):
self.fetcher = fetcher.FTP(self.domain, self.port, self.path, self.dest, owner = self.owner) self.fetcher = fetcher.FTP(self.domain, self.port, self.path, self.dest, owner = self.owner)
self._check_conn() self._check_conn()


def _get_delaychk(self):
delay = self.xml.attrib.get('delayCheck')
if not delay:
return(None)
r = _duration_re.search(delay)
times = {k: (float(v) if v else 0.0) for k, v in r.groupdict().items()}
years = float(times.pop('years'))
months = float(times.pop('months'))
times['days'] = (times['days'] + (years * constants.YEAR) + (months * constants.MONTH))
self.delay = datetime.timedelta(**times)
return(None)

def _check_conn(self): def _check_conn(self):
sock = socket.socket() sock = socket.socket()
sock.settimeout(7) sock.settimeout(7)

View File

@ -1,56 +0,0 @@
#!/usr/bin/env python3

import argparse
import os
import pathlib
import sys
##
##
_cur_file = pathlib.Path(os.path.abspath(os.path.expanduser(__file__))).resolve()
_cur_path = os.path.dirname(_cur_file)
sys.path.insert(1, _cur_path)
import repomirror


if os.geteuid() == 0:
_def_logdir = '/var/log/repo'
else:
_def_logdir = '~/.cache/repologs'


def parseArgs():
args = argparse.ArgumentParser(description = 'Sync repositories for various distributions to local paths')
args.add_argument('-c', '--config',
default = '~/.config/repomirror.xml',
dest = 'cfg',
help = ('The path to the config file. If it does not exist, a bare version will be created. '
'Default: ~/.config/repomirror.xml'))
# args.add_argument('-n', '--dry-run',
# action = 'store_true',
# dest = 'dummy',
# help = ('If specified, do not actually sync anything (other than timestamp files if '
# 'applicable to determine logic); do not actually sync any repositories. Useful for '
# 'generating logs to determine potential issues before they happen'))
args.add_argument('-d', '--distro',
dest = 'distro',
action = 'append',
help = ('If specified, only sync the specified distro in the config file (otherwise sync all '
'specified). May be given multiple times'))
args.add_argument('-l', '--logdir',
default = _def_logdir,
dest = 'logdir',
help = ('The path to the directory where logs should be written. The actual log files will be '
'named after their respective distro names in the config file. '
'Default: {0}'.format(_def_logdir)))
return(args)


def main():
args = parseArgs().parse_args()
r = repomirror.Sync(**vars(args))
r.sync()
return(None)


if __name__ == '__main__':
main()

1
reposync Symbolic link
View File

@ -0,0 +1 @@
reposync.py

View File

@ -1 +0,0 @@
reposync

56
reposync.py Executable file
View File

@ -0,0 +1,56 @@
#!/usr/bin/env python3

import argparse
import os
import pathlib
import sys
##
##
_cur_file = pathlib.Path(os.path.abspath(os.path.expanduser(__file__))).resolve()
_cur_path = os.path.dirname(_cur_file)
sys.path.insert(1, _cur_path)
import repomirror


if os.geteuid() == 0:
_def_logdir = '/var/log/repo'
else:
_def_logdir = '~/.cache/repologs'


def parseArgs():
args = argparse.ArgumentParser(description = 'Sync repositories for various distributions to local paths')
args.add_argument('-c', '--config',
default = '~/.config/repomirror.xml',
dest = 'cfg',
help = ('The path to the config file. If it does not exist, a bare version will be created. '
'Default: ~/.config/repomirror.xml'))
# args.add_argument('-n', '--dry-run',
# action = 'store_true',
# dest = 'dummy',
# help = ('If specified, do not actually sync anything (other than timestamp files if '
# 'applicable to determine logic); do not actually sync any repositories. Useful for '
# 'generating logs to determine potential issues before they happen'))
args.add_argument('-d', '--distro',
dest = 'distro',
action = 'append',
help = ('If specified, only sync the specified distro in the config file (otherwise sync all '
'specified). May be given multiple times'))
args.add_argument('-l', '--logdir',
default = _def_logdir,
dest = 'logdir',
help = ('The path to the directory where logs should be written. The actual log files will be '
'named after their respective distro names in the config file. '
'Default: {0}'.format(_def_logdir)))
return(args)


def main():
args = parseArgs().parse_args()
r = repomirror.Sync(**vars(args))
r.sync()
return(None)


if __name__ == '__main__':
main()