whoo doggie. should check this in.
This commit is contained in:
brent s
parent
0695b86add
commit
e9b7b52bb0
|
|
@ -1,3 +1,7 @@
|
|||
import os
|
||||
import grp
|
||||
import pwd
|
||||
|
||||
def xmlBool(xmlobj):
|
||||
if isinstance(xmlobj, bool):
|
||||
return (xmlobj)
|
||||
|
|
@ -7,3 +11,25 @@ def xmlBool(xmlobj):
|
|||
return(False)
|
||||
else:
|
||||
return(None)
|
||||
|
||||
|
||||
def getSudoGroup():
|
||||
is_sudo = False
|
||||
if os.environ.get('SUDO_GID'):
|
||||
gid = int(os.environ['SUDO_GID'])
|
||||
is_sudo = True
|
||||
else:
|
||||
gid = os.getegid()
|
||||
group = grp.getgrgid(gid)
|
||||
return((group, gid, is_sudo))
|
||||
|
||||
|
||||
def getSudoUser():
|
||||
is_sudo = False
|
||||
if os.environ.get('SUDO_UID'):
|
||||
uid = int(os.environ['SUDO_UID'])
|
||||
is_sudo = True
|
||||
else:
|
||||
uid = os.geteuid()
|
||||
user = pwd.getpwuid(os.geteuid())
|
||||
return((user, uid, is_sudo))
|
||||
|
|
|
|||
159
ARB/mirror.py
159
ARB/mirror.py
|
|
@ -1,39 +1,174 @@
|
|||
import os
|
||||
import grp
|
||||
import pathlib
|
||||
import pwd
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
##
|
||||
import paramiko
|
||||
##
|
||||
import arb_util
|
||||
|
||||
|
||||
class Mirror(object):
|
||||
def __init__(self, mirror_xml, ns = '', *args, **kwargs):
|
||||
self.xml = mirror_xml
|
||||
self.ns = ns
|
||||
if os.environ.get('SUDO_USER'):
|
||||
_uname = os.environ['SUDO_USER']
|
||||
else:
|
||||
_uname = pwd.getpwuid(os.geteuid()).pw_name
|
||||
self.user = pwd.getpwnam(mirror_xml.attrib.get('user', _uname))
|
||||
self.fmode = int(self.xml.attrib.get('fileMode', '0600'), 8)
|
||||
self.dmode = int(self.xml.attrib.get('dirMode', '0700'), 8)
|
||||
user, uid, self.is_sudo = arb_util.getSudoUser()
|
||||
self.user = pwd.getpwnam(mirror_xml.attrib.get('user', user.pw_name))
|
||||
try:
|
||||
self.fmode = int(mirror_xml.attrib.get('fileMode'), 8)
|
||||
except TypeError:
|
||||
self.fmode = None
|
||||
try:
|
||||
self.dmode = int(mirror_xml.attrib.get('dirMode'), 8)
|
||||
except TypeError:
|
||||
self.dmode = None
|
||||
self.dest = self.xml.text
|
||||
|
||||
def sync(self):
|
||||
# no-op; this is handled in the subclasses since it's unique to them.
|
||||
pass
|
||||
return(True)
|
||||
|
||||
|
||||
class LocalMirror(Mirror):
|
||||
def __init__(self, mirror_xml, ns = '', *args, **kwargs):
|
||||
super().__init__(mirror_xml, ns = ns, *args, **kwargs)
|
||||
if os.environ.get('SUDO_GID'):
|
||||
_grpnm = os.environ['SUDO_GID']
|
||||
else:
|
||||
_grpnm = grp.getgrgid(os.getegid()).gr_name
|
||||
self.group = grp.getgrnam(mirror_xml.attrib.get('group', _grpnm))
|
||||
if self.user.pw_uid == arb_util.getSudoUser()[1]:
|
||||
self.user = None
|
||||
group, gid, is_sudo = arb_util.getSudoGroup()
|
||||
self.group = grp.getgrnam(mirror_xml.attrib.get('group', group.gr_name))
|
||||
if self.group.gr_gid == gid:
|
||||
self.group = None
|
||||
self.dest = os.path.abspath(os.path.expanduser(self.dest))
|
||||
|
||||
def sync(self, source):
|
||||
source = os.path.abspath(os.path.expanduser(source))
|
||||
for root, dirs, files in os.walk(source):
|
||||
for d in dirs:
|
||||
dpath = os.path.join(root, d)
|
||||
reldpath = pathlib.PurePosixPath(dpath).relative_to(source)
|
||||
destdpath = os.path.join(self.dest, reldpath)
|
||||
if os.path.exists(destdpath):
|
||||
shutil.rmtree(destdpath)
|
||||
shutil.copytree(dpath, destdpath, symlinks = True, ignore_dangling_symlinks = True)
|
||||
for f in files:
|
||||
fpath = os.path.join(root, f)
|
||||
relfpath = pathlib.PurePosixPath(fpath).relative_to(source)
|
||||
destfpath = os.path.join(self.dest, relfpath)
|
||||
shutil.copy2(fpath, destfpath)
|
||||
break # We only need one iteration since copytree is recursive
|
||||
# Now we set the user/group ownership and the file/dir modes.
|
||||
# This first any() check is DEFINITELY a speed optimization if those perms aren't modified.
|
||||
if any((self.user, self.group, self.fmode, self.dmode)):
|
||||
if self.user:
|
||||
os.chown(self.dest, self.user.pw_uid, -1, follow_symlinks = False)
|
||||
if self.group:
|
||||
os.chown(self.dest, -1, self.group.gr_gid, follow_symlinks = False)
|
||||
if self.dmode:
|
||||
try:
|
||||
os.chmod(self.dest, self.dmode, follow_symlinks = False)
|
||||
except NotImplementedError:
|
||||
os.chmod(self.dest, self.dmode)
|
||||
for root, dirs, files in os.walk(self.dest):
|
||||
for d in dirs:
|
||||
dpath = os.path.join(root, d)
|
||||
if self.user:
|
||||
os.chown(dpath, self.user.pw_uid, -1, follow_symlinks = False)
|
||||
if self.group:
|
||||
os.chown(dpath, -1, self.group.gr_gid, follow_symlinks = False)
|
||||
if self.dmode:
|
||||
try:
|
||||
os.chmod(dpath, self.dmode, follow_symlinks = False)
|
||||
except NotImplementedError:
|
||||
os.chmod(dpath, self.dmode)
|
||||
for f in files:
|
||||
fpath = os.path.join(root, f)
|
||||
if self.user:
|
||||
os.chown(fpath, self.user.pw_uid, -1, follow_symlinks = False)
|
||||
if self.group:
|
||||
os.chown(fpath, -1, self.group.gr_gid, follow_symlinks = False)
|
||||
if self.fmode:
|
||||
try:
|
||||
os.chmod(fpath, self.fmode, follow_symlinks = False)
|
||||
except NotImplementedError:
|
||||
os.chmod(fpath, self.fmode)
|
||||
return(True)
|
||||
|
||||
|
||||
class RemoteMirror(Mirror):
|
||||
def __init__(self, mirror_xml, ns = '', *args, **kwargs):
|
||||
super().__init__(mirror_xml, ns = ns, *args, **kwargs)
|
||||
self.server = mirror_xml.attrib['server']
|
||||
self.hardened = arb_util.xmlBool(mirror_xml.attrib.get('hardened', False))
|
||||
self.port = int(mirror_xml.attrib.get('port', 22))
|
||||
self.keyfile = os.path.abspath(os.path.expanduser(mirror_xml.attrib.get('key', '~/.ssh/id_rsa')))
|
||||
self.remote_user = mirror_xml.attrib.get('remoteUser')
|
||||
self.remote_group = mirror_xml.attrib.get('remoteGroup')
|
||||
self.ssh = None
|
||||
self.transport = None
|
||||
|
||||
def _initSSH(self):
|
||||
has_ssh = False
|
||||
if self.ssh and self.transport.is_active() and self.transport.is_alive():
|
||||
has_ssh = True
|
||||
if not has_ssh:
|
||||
userhostkeys = os.path.abspath(os.path.expanduser('~/.ssh/known_hosts'))
|
||||
self.ssh = paramiko.SSHClient()
|
||||
self.ssh.load_system_host_keys()
|
||||
if os.path.isfile(userhostkeys):
|
||||
self.ssh.load_system_host_keys(userhostkeys)
|
||||
self.ssh.set_missing_host_key_policy((paramiko.RejectPolicy
|
||||
if self.hardened else
|
||||
paramiko.AutoAddPolicy))
|
||||
self.ssh.connect(hostname = self.server,
|
||||
port = self.port,
|
||||
username = self.user.pw_name,
|
||||
key_filename = self.keyfile)
|
||||
self.transport = self.ssh.get_transport()
|
||||
return()
|
||||
|
||||
def _closeSSH(self):
|
||||
if self.transport:
|
||||
self.transport.close()
|
||||
if self.ssh:
|
||||
self.ssh.close()
|
||||
return()
|
||||
|
||||
def sync(self, source):
|
||||
source = os.path.abspath(os.path.expanduser(source))
|
||||
cmd = ['rsync',
|
||||
'--archive',
|
||||
# '--delete', # TODO: yes? no? configurable?
|
||||
'--rsh="ssh -p {0} -l {1}"'.format(self.port, self.user.pw_name),
|
||||
source,
|
||||
'{0}@{1}:{2}'.format(self.user.pw_name, self.server, self.dest)]
|
||||
# TODO: log output?
|
||||
rsync_out = subprocess.run(cmd, stderr = subprocess.PIPE, stdout = subprocess.PIPE)
|
||||
# This first if is technically unnecessary, but it can offer a *slight* speed benefit. VERY slight.
|
||||
# As in so negligible, only Jthan would care about it.
|
||||
if any((self.remote_user, self.remote_group, self.fmode, self.dmode)):
|
||||
if self.remote_user:
|
||||
self._initSSH()
|
||||
stdin, stdout, stderr = self.ssh.exec_command('chown -R {0} {1}'.format(self.remote_user,
|
||||
self.dest))
|
||||
if self.remote_group:
|
||||
self._initSSH()
|
||||
stdin, stdout, stderr = self.ssh.exec_command('chgrp -R {0} {1}'.format(self.remote_group,
|
||||
self.dest))
|
||||
if self.fmode:
|
||||
self._initSSH()
|
||||
stdin, stdout, stderr = self.ssh.exec_command(
|
||||
('find {0} -type f -print0 | '
|
||||
'xargs --null --no-run-if-empty chmod {1}').format(self.dest,
|
||||
re.sub('^0o', '', oct(self.fmode))))
|
||||
if self.dmode:
|
||||
self._initSSH()
|
||||
stdin, stdout, stderr = self.ssh.exec_command(
|
||||
('find {0} -type d -print0 | '
|
||||
'xargs --null --no-run-if-empty chmod {1}').format(self.dest,
|
||||
re.sub('^0o', '', oct(self.dmode))))
|
||||
self._closeSSH()
|
||||
return(True)
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import requests
|
|||
##
|
||||
import arb_util
|
||||
|
||||
# TODO: implement alwaysBuild check!!!
|
||||
|
||||
# TODO: should this be a configuration option?
|
||||
aurbase = 'https://aur.archlinux.org'
|
||||
|
|
|
|||
37
ARB/repo.py
37
ARB/repo.py
|
|
@ -1,5 +1,6 @@
|
|||
import os
|
||||
import re
|
||||
import subprocess
|
||||
##
|
||||
import gpg
|
||||
##
|
||||
|
|
@ -17,16 +18,32 @@ class Repo(object):
|
|||
self.key = None
|
||||
self.mirrors = []
|
||||
self.packages = []
|
||||
self.packagefiles = []
|
||||
self.sigfiles = []
|
||||
_key_id = self.xml.attrib.get('gpgKeyID')
|
||||
self.key_id = (re.sub(r'\s+', '', _key_id) if _key_id else None)
|
||||
self.staging_dir = os.path.abspath(os.path.expanduser(self.xml.attrib.get('staging',
|
||||
'.')))
|
||||
self.sign_pkgs = arb_util.xmlBool(self.xml.attrib.get('signPkgs', True))
|
||||
self.sign_db = arb_util.xmlBool(self.xml.attrib.get('signDB', True))
|
||||
if any((self.sign_db, self.sign_pkgs)):
|
||||
self._initSigner()
|
||||
self._initMirrors()
|
||||
self._initPackages()
|
||||
|
||||
def _genRepo(self):
|
||||
if not self.packagefiles:
|
||||
# raise RuntimeError('.build() must be run before ._genRepo()')
|
||||
return(None)
|
||||
cmd = ['repo-add']
|
||||
if self.sign_db:
|
||||
cmd.extend(['--sign', '--key', self.key_id])
|
||||
cmd.extend(['--remove',
|
||||
os.path.join(self.staging_dir, '{0}.db.tar.xz'.format(self.name)),
|
||||
*self.packagefiles])
|
||||
repo_out = subprocess.run(cmd, stderr = subprocess.PIPE, stdout = subprocess.PIPE)
|
||||
return(True)
|
||||
|
||||
def _initMirrors(self):
|
||||
for m in self.xml.findall('{0}mirrors/{0}mirror.RemoteMirror'.format(self.ns)):
|
||||
self.mirrors.append(mirror.RemoteMirror(m, ns = self.ns))
|
||||
|
|
@ -61,6 +78,7 @@ class Repo(object):
|
|||
if squashed_key in keyforms:
|
||||
if k.can_sign:
|
||||
self.key = k
|
||||
self.key_id = k.fpr
|
||||
break
|
||||
else:
|
||||
for s in k.subkeys:
|
||||
|
|
@ -68,6 +86,7 @@ class Repo(object):
|
|||
if squashed_key in subkeyforms:
|
||||
if s.can_sign:
|
||||
self.key = s
|
||||
self.key_id = s.fpr
|
||||
break
|
||||
else:
|
||||
if k.can_sign:
|
||||
|
|
@ -77,3 +96,21 @@ class Repo(object):
|
|||
raise ValueError('Cannot find a suitable signing GPG key')
|
||||
self.gpg.signers = [self.key]
|
||||
return()
|
||||
|
||||
def build(self):
|
||||
for p in self.packages:
|
||||
self.packagefiles.extend(p.build(self.staging_dir))
|
||||
if self.sign_pkgs:
|
||||
for f in self.packagefiles:
|
||||
sigfile = '{0}.sig'.format(f)
|
||||
with open(f, 'rb') as pkg:
|
||||
with open(sigfile, 'wb') as sig:
|
||||
sig.write(self.gpg.sign(pkg.read(), mode = gpg.constants.SIG_MODE_DETACH)[0])
|
||||
self.sigfiles.append(sigfile)
|
||||
self._genRepo()
|
||||
return()
|
||||
|
||||
def sync(self):
|
||||
for m in self.mirrors:
|
||||
m.sync(self.staging_dir)
|
||||
return()
|
||||
|
|
|
|||
|
|
@ -53,6 +53,7 @@
|
|||
<xs:attribute name="user" type="archrepo:t_posixUserGroup"
|
||||
default="%same" use="optional"/>
|
||||
<xs:attribute name="server" type="xs:NMTOKEN" use="required"/>
|
||||
<xs:attribute name="hardened" type="xs:NMTOKEN" use="optional" default="false"/>
|
||||
<xs:attribute name="fileMode" type="archrepo:t_posixMode"
|
||||
use="optional" default="0600"/>
|
||||
<xs:attribute name="dirMode" type="archrepo:t_posixMode"
|
||||
|
|
|
|||
|
|
@ -49,8 +49,8 @@
|
|||
group: The group to chown the files/directories to (must be running as root user). If not
|
||||
specified, the default is the primary group for the current user (or the user calling
|
||||
sudo, if done via sudo).
|
||||
fileMode: The octal permissions to chmod the files to.
|
||||
dirMode: The octal permissions to chmod the directories to.
|
||||
fileMode: The octal permissions to chmod the files to (default is creation mode).
|
||||
dirMode: The octal permissions to chmod the directories to (default is creation mode).
|
||||
-->
|
||||
<localMirror
|
||||
user="foo"
|
||||
|
|
@ -62,14 +62,21 @@
|
|||
The remoteMirror element is for rsyncing packages to a remote mirror/repo server. Rsync must be installed
|
||||
locally (it should; it's part of base-devel) *and* the remote server. Obviously, SSH pubkey auth must also
|
||||
be set up as well for the user. They must have a valid shell on the server for chmodding/chowning.
|
||||
If you don't need to modify remoteUser/remoteGroup/fileMode/dirMode, it's recommended to use rrsync on the
|
||||
remote mirror/repo server instead (https://www.samba.org/ftp/unpacked/rsync/support/rrsync) where possible.
|
||||
Attributes:
|
||||
user: The (remote) user to sync as (e.g. for "ssh foo@bar", user would be "foo").
|
||||
user: The (remote) user to sync as (e.g. for "ssh foo@bar", user would be "foo"). The default is
|
||||
the same as localMirror[user].
|
||||
server: The server to sync to. Can be an IP address, hostname (if resolvable), or FQDN.
|
||||
hardened: Can be "1"/"true" or "0"/"false". If true, only connect to servers we know the
|
||||
host key for (either in /etc/ssh/ssh_known_hosts or ~/.ssh/known_hosts).
|
||||
port: The remote SSH port.
|
||||
key: The pubkey to use to connect.
|
||||
remoteUser: The (remote) user to chown the files/directories to (must be connecting as root user).
|
||||
If not specified, the default is the connecting user ("user" attribute).
|
||||
remoteGroup: The (remote) group to chown the files/directories to (must be connecting as root user).
|
||||
remoteUser: The (remote) user to chown the files/directories to ("user" must be "root" for this
|
||||
to work).
|
||||
If not specified, the default is the connecting user.
|
||||
remoteGroup: The (remote) group to chown the files/directories to ("user" must be "root" for this
|
||||
to work).
|
||||
If not specified, the default is the connecting user's ("user" attribute) primary
|
||||
group.
|
||||
fileMode: The octal permissions to chmod the remote files to.
|
||||
|
|
@ -78,6 +85,7 @@
|
|||
<remoteMirror
|
||||
user="foo"
|
||||
server="bar.domain.tld"
|
||||
hardened="false"
|
||||
port="22"
|
||||
key="~/.ssh/id_rsa"
|
||||
remoteUser="foo"
|
||||
|
|
|
|||
15
sync.sh
15
sync.sh
|
|
@ -1,15 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# This obviously will require some tweaking. Will roll into build.py later.
|
||||
set -e
|
||||
|
||||
server=my_repo.domain.tld
|
||||
port=2222
|
||||
user=pkgrepo
|
||||
src=~/pkgs/built/.
|
||||
# You should use rrsync to restrict to a specific directory
|
||||
dest='Arch/.'
|
||||
|
||||
echo "Syncing..."
|
||||
rsync -a --delete -e "ssh -p ${port}" ${src} ${user}@${server}:${dest}
|
||||
echo "Done."
|
||||
Loading…
Reference in New Issue