From 99468dd4a8ca34cc1e5ae197871a66649445ccf2 Mon Sep 17 00:00:00 2001 From: brent s Date: Thu, 18 Jun 2020 00:29:12 -0400 Subject: [PATCH] v1.0.1 --- .gitignore | 6 ++ README | 45 +++++++++++---- repomirror/constants.py | 1 + repomirror/utils/__init__.py | 0 .../utils/find_fastest_upstream/__init__.py | 0 .../utils}/find_fastest_upstream/archlinux.py | 0 .../utils}/find_fastest_upstream/centos.py | 0 .../utils}/find_fastest_upstream/classes.py | 0 .../utils}/find_fastest_upstream/constants.py | 0 reposync | 56 ++++++++++++++++++ reposync.py | 57 +------------------ setup.py | 55 ++++++++++++++++++ 12 files changed, 154 insertions(+), 66 deletions(-) create mode 100644 repomirror/utils/__init__.py create mode 100644 repomirror/utils/find_fastest_upstream/__init__.py rename {utils => repomirror/utils}/find_fastest_upstream/archlinux.py (100%) rename {utils => repomirror/utils}/find_fastest_upstream/centos.py (100%) rename {utils => repomirror/utils}/find_fastest_upstream/classes.py (100%) rename {utils => repomirror/utils}/find_fastest_upstream/constants.py (100%) create mode 100755 reposync mode change 100755 => 120000 reposync.py create mode 100644 setup.py diff --git a/.gitignore b/.gitignore index ff3b78a..cdfae7d 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,13 @@ screenlog* *~ .~lock.* .editix +build_instructions __pycache__/ +build/ +dist/ +repomirror.egg-info/ +repomirror/__pycache__/ +repomirror/fetcher/__pycache__/ *.pyc *.tar *.tar.bz2 diff --git a/README b/README index ebbb928..6c50cac 100644 --- a/README +++ b/README @@ -1,13 +1,38 @@ This project will let you mirror a Linux distribution's repository/repositories (or BSD, etc.). -Features: -- Logging -- RSync or FTP supported (RSync mirrors require rsync to be installed and in the $PATH environment variable) (HTTP/HTTPS cannot and will not be supported as there's no real way to do it reliably and efficiently.) -- Locking - prevent a specific distribution from syncing if it is currently being synced but allow other distributions to sync -- Customizable rsync arguments -- Changing of directory/file ownership per-distribution -- Mount-checking per-distribution - a distribution will be skipped if its designated mountpoint is not mounted -- Synchronization checks - timestamp files can be read and written and are used to determine if a sync should take place or not --- TODO: customizable "staleness" of repos (e.g. sync if timestamp X is older than timestamp Y on server) -See example.config.xml for more information. +Features: + +* Logging +* RSync or FTP supported (RSync mirrors require rsync to be installed and in the $PATH environment variable) (HTTP/HTTPS cannot and will not be supported as there's no real way to do it reliably and efficiently.) +* Locking - prevent a specific distribution from syncing if it is currently being synced but allow other distributions to sync +* Customizable rsync arguments +* Changing of directory/file ownership per-distribution +* Mount-checking per-distribution - a distribution will be skipped if its designated mountpoint is not mounted +* Synchronization checks - timestamp files can be read and written and are used to determine if a sync should take place or not +** TODO: customizable "staleness" of repos (e.g. sync if timestamp X is older than timestamp Y on server) + + +Configuration/Deployment: + +See example.config.xml for details on a configuration file, as it's extensively commented and won't be covered in this README. + +You may take special interest in repomirror/utils/find_fastest_upstream/ scripts as they're pre-written to find the fastest (in theory) upstream you can use. Currently only Arch Linux and CentOS have scripts written, but I'll gladly try to add other distros if you open a feature request (see below). + + +Bugs/Feature Requests: + +Please use my tracker at https://bugs.square-r00t.net/index.php?project=14 or, if you prefer to not create an account, +simply email me at bts(at)square-r00t[dot]net. + + +Sources: + +This project can be found at/cloned from: + +* https://git.square-r00t.net/RepoMirror +* git://git.square-r00t.net/repomirror.git + +And has a mirrored repository on GitHub (Issues/Wiki/etc. disabled) at: + +* https://github.com/johnnybubonic/repomirror diff --git a/repomirror/constants.py b/repomirror/constants.py index 97f4beb..965aa1b 100644 --- a/repomirror/constants.py +++ b/repomirror/constants.py @@ -12,3 +12,4 @@ RSYNC_DEF_ARGS = ['--recursive', '--exclude=.*'] # How many days an upstream should have last synced by before it's considered stale. DAYS_WARN = 2 +VERSION = '1.0.1' diff --git a/repomirror/utils/__init__.py b/repomirror/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/repomirror/utils/find_fastest_upstream/__init__.py b/repomirror/utils/find_fastest_upstream/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/utils/find_fastest_upstream/archlinux.py b/repomirror/utils/find_fastest_upstream/archlinux.py similarity index 100% rename from utils/find_fastest_upstream/archlinux.py rename to repomirror/utils/find_fastest_upstream/archlinux.py diff --git a/utils/find_fastest_upstream/centos.py b/repomirror/utils/find_fastest_upstream/centos.py similarity index 100% rename from utils/find_fastest_upstream/centos.py rename to repomirror/utils/find_fastest_upstream/centos.py diff --git a/utils/find_fastest_upstream/classes.py b/repomirror/utils/find_fastest_upstream/classes.py similarity index 100% rename from utils/find_fastest_upstream/classes.py rename to repomirror/utils/find_fastest_upstream/classes.py diff --git a/utils/find_fastest_upstream/constants.py b/repomirror/utils/find_fastest_upstream/constants.py similarity index 100% rename from utils/find_fastest_upstream/constants.py rename to repomirror/utils/find_fastest_upstream/constants.py diff --git a/reposync b/reposync new file mode 100755 index 0000000..a78bde5 --- /dev/null +++ b/reposync @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 + +import argparse +import os +import pathlib +import sys +## +## +_cur_file = pathlib.Path(os.path.abspath(os.path.expanduser(__file__))).resolve() +_cur_path = os.path.dirname(_cur_file) +sys.path.insert(1, _cur_path) +import repomirror + + +if os.geteuid() == 0: + _def_logdir = '/var/log/repo' +else: + _def_logdir = '~/.cache/repologs' + + +def parseArgs(): + args = argparse.ArgumentParser(description = 'Sync repositories for various distributions to local paths') + args.add_argument('-c', '--config', + default = '~/.config/repomirror.xml', + dest = 'cfg', + help = ('The path to the config file. If it does not exist, a bare version will be created. ' + 'Default: ~/.config/repomirror.xml')) + # args.add_argument('-n', '--dry-run', + # action = 'store_true', + # dest = 'dummy', + # help = ('If specified, do not actually sync anything (other than timestamp files if ' + # 'applicable to determine logic); do not actually sync any repositories. Useful for ' + # 'generating logs to determine potential issues before they happen')) + args.add_argument('-d', '--distro', + dest = 'distro', + action = 'append', + help = ('If specified, only sync the specified distro in the config file (otherwise sync all ' + 'specified). May be given multiple times')) + args.add_argument('-l', '--logdir', + default = _def_logdir, + dest = 'logdir', + help = ('The path to the directory where logs should be written. The actual log files will be ' + 'named after their respective distro names in the config file. ' + 'Default: {0}'.format(_def_logdir))) + return(args) + + +def main(): + args = parseArgs().parse_args() + r = repomirror.Sync(**vars(args)) + r.sync() + return(None) + + +if __name__ == '__main__': + main() diff --git a/reposync.py b/reposync.py deleted file mode 100755 index a78bde5..0000000 --- a/reposync.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import os -import pathlib -import sys -## -## -_cur_file = pathlib.Path(os.path.abspath(os.path.expanduser(__file__))).resolve() -_cur_path = os.path.dirname(_cur_file) -sys.path.insert(1, _cur_path) -import repomirror - - -if os.geteuid() == 0: - _def_logdir = '/var/log/repo' -else: - _def_logdir = '~/.cache/repologs' - - -def parseArgs(): - args = argparse.ArgumentParser(description = 'Sync repositories for various distributions to local paths') - args.add_argument('-c', '--config', - default = '~/.config/repomirror.xml', - dest = 'cfg', - help = ('The path to the config file. If it does not exist, a bare version will be created. ' - 'Default: ~/.config/repomirror.xml')) - # args.add_argument('-n', '--dry-run', - # action = 'store_true', - # dest = 'dummy', - # help = ('If specified, do not actually sync anything (other than timestamp files if ' - # 'applicable to determine logic); do not actually sync any repositories. Useful for ' - # 'generating logs to determine potential issues before they happen')) - args.add_argument('-d', '--distro', - dest = 'distro', - action = 'append', - help = ('If specified, only sync the specified distro in the config file (otherwise sync all ' - 'specified). May be given multiple times')) - args.add_argument('-l', '--logdir', - default = _def_logdir, - dest = 'logdir', - help = ('The path to the directory where logs should be written. The actual log files will be ' - 'named after their respective distro names in the config file. ' - 'Default: {0}'.format(_def_logdir))) - return(args) - - -def main(): - args = parseArgs().parse_args() - r = repomirror.Sync(**vars(args)) - r.sync() - return(None) - - -if __name__ == '__main__': - main() diff --git a/reposync.py b/reposync.py new file mode 120000 index 0000000..9b00562 --- /dev/null +++ b/reposync.py @@ -0,0 +1 @@ +reposync \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..6f6cb79 --- /dev/null +++ b/setup.py @@ -0,0 +1,55 @@ +import re +from setuptools import setup +from repomirror import constants + + +_req_re = re.compile(r'(<|=|>)') +with open('README', 'r') as fh: + long_desc = fh.read().strip() +with open('requirements.txt', 'r') as fh: + reqs = [_req_re.split(i.strip())[0] for i in fh.read().strip().splitlines() + if not i.startswith('#') + and i.strip() != ''] + + +setup( + author_email = 'bts@square-r00t.net', + author = 'Brent S.', + classifiers = [ + 'Environment :: Console', + 'Intended Audience :: Developers', + 'Intended Audience :: Information Technology', + 'Intended Audience :: System Administrators', + 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)', + 'Operating System :: POSIX :: BSD', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3 :: Only', + 'Topic :: Internet', + 'Topic :: Software Development :: Build Tools', + 'Topic :: System :: Archiving :: Mirroring', + 'Topic :: System :: Installation/Setup', + 'Topic :: System :: Software Distribution', + 'Topic :: System :: Systems Administration' + ], + description = 'Clone/mirror multiple Linux distro/BSD flavor/etc. repositories to a local server', + long_description = long_desc, + long_description_content_type = 'text/plain', + name = 'repomirror', + packages = ['repomirror', 'repomirror.fetcher', 'repomirror.utils'], + project_urls = { + 'Documentation': 'https://git.square-r00t.net/RepoMirror/tree/README', + 'Source': 'https://git.square-r00t.net/RepoMirror/', + 'Tracker': 'https://bugs.square-r00t.net/index.php?project=14' + }, + install_requires = reqs, + python_requires = '>=3.7', + scripts = [ + 'reposync' + ], + url = 'https://git.square-r00t.net/RepoMirror/', + version = str(constants.VERSION) + )