some populating...

This commit is contained in:
brent s 2019-09-18 03:01:19 -04:00
parent 168c1c3ae8
commit a90188c16f
6 changed files with 501 additions and 0 deletions

141
autorepo.xsd Normal file
View File

@ -0,0 +1,141 @@
<?xml version="1.0" encoding="UTF-8" ?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
targetNamespace="http://git.square-r00t.net/OpTools/tree/arch/autorepo/"
xmlns="http://git.square-r00t.net/OpTools/tree/arch/autorepo/tree/"
xmlns:archrepo="http://git.square-r00t.net/OpTools/tree/arch/autorepo/"
elementFormDefault="qualified"
attributeFormDefault="unqualified">

<xs:simpleType name="t_posixUserGroup">
<xs:restriction base="xs:token">
<xs:pattern value="[a-z_]([a-z0-9_-]{0,31}|[a-z0-9_-]{0,30}$)"/>
<xs:pattern value="%same"/>
<xs:whiteSpace value="collapse"/>
</xs:restriction>
</xs:simpleType>

<xs:simpleType name="t_posixMode">
<xs:restriction base="xs:positiveInteger">
<xs:pattern value="[0-7]?[0-7]{3}"/>
<xs:whiteSpace value="collapse"/>
</xs:restriction>
</xs:simpleType>

<xs:simpleType name="t_path">
<xs:restriction base="xs:string">
<xs:pattern value="(/|~/)?([A-Za-z0-9+_.-]+/)*[A-Za-z0-9+_.-]+"/>
<xs:whiteSpace value="collapse"/>
</xs:restriction>
</xs:simpleType>

<xs:simpleType name="t_port">
<xs:restriction base="xs:positiveInteger">
<!-- MAN I wish XSD let you validate based on numerical range. -->
<!-- https://stackoverflow.com/a/40213676/733214 -->
<xs:pattern value="([1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])"/>
</xs:restriction>
</xs:simpleType>

<xs:complexType name="t_localMirror">
<xs:simpleContent>
<xs:extension base="xs:anyURI">
<xs:attribute name="user" type="archrepo:t_posixUserGroup" use="optional" default="%same"/>
<xs:attribute name="group" type="archrepo:t_posixUserGroup" use="optional" default="%same"/>
<xs:attribute name="fileMode" type="archrepo:t_posixMode" use="optional" default="0600"/>
<xs:attribute name="dirMode" type="archrepo:t_posixMode" use="optional" default="0700"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>

<xs:complexType name="t_remoteMirror">
<xs:simpleContent>
<xs:extension base="xs:anyURI">
<xs:attribute name="user" type="archrepo:t_posixUserGroup"
default="%same" use="optional"/>
<xs:attribute name="server" type="xs:NMTOKEN" use="required"/>
<xs:attribute name="fileMode" type="archrepo:t_posixMode"
use="optional" default="0600"/>
<xs:attribute name="dirMode" type="archrepo:t_posixMode"
use="optional" default="0700"/>
<xs:attribute name="port" type="archrepo:t_port"
default="22" use="optional"/>
<xs:attribute name="key" type="archrepo:t_path"
default="~/.ssh/id_rsa" use="optional"/>
<xs:attribute name="remoteUser" type="archrepo:t_posixUserGroup"
default="%same" use="optional"/>
<xs:attribute name="remoteGroup" type="archrepo:t_posixUserGroup"
default="%same" use="optional"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>

<xs:simpleType name="t_gpgKeyID">
<xs:restriction base="xs:string">
<xs:pattern value="(0[Xx])?[0-9A-Fa-f]{40}"/>
<xs:pattern value="(0[Xx])?[0-9A-Fa-f]{8}"/>
<xs:pattern value="(0[Xx])?([0-9A-Fa-f]{4} ?){5} *([0-9A-Fa-f]{4}){5}"/>
<xs:whiteSpace value="collapse"/>
</xs:restriction>
</xs:simpleType>

<xs:element name="archrepo">
<xs:complexType>
<xs:choice>
<xs:element name="repo" minOccurs="1" maxOccurs="unbounded">
<xs:complexType>
<xs:all minOccurs="1">
<xs:element name="mirrors" minOccurs="1" maxOccurs="1">
<xs:complexType>
<xs:choice minOccurs="1" maxOccurs="unbounded">
<xs:element name="localMirror"
maxOccurs="unbounded"
type="archrepo:t_localMirror"/>
<xs:element name="remoteMirror"
maxOccurs="unbounded"
type="archrepo:t_remoteMirror"/>
</xs:choice>
</xs:complexType>
</xs:element>
<xs:element name="packages" minOccurs="1">
<xs:complexType>
<xs:choice minOccurs="1" maxOccurs="unbounded">
<xs:element name="aur"
maxOccurs="unbounded">
<xs:complexType>
<xs:simpleContent>
<xs:extension base="xs:string">
<xs:attribute name="alwaysBuild" default="true"
type="xs:boolean" use="optional"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
</xs:element>
<xs:element name="pkgbuild"
maxOccurs="unbounded">
<xs:complexType>
<xs:simpleContent>
<xs:extension base="xs:token">
<xs:attribute name="path" type="archrepo:t_path"
default="." use="optional"/>
<xs:attribute name="alwaysBuild" default="true"
type="xs:boolean" use="optional"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
</xs:element>
</xs:choice>
</xs:complexType>
</xs:element>
</xs:all>
<xs:attribute name="name" type="xs:token" use="required"/>
<xs:attribute name="staging" type="archrepo:t_path" use="optional" default="."/>
<xs:attribute name="signPkgs" type="xs:boolean" use="optional" default="true"/>
<xs:attribute name="signDB" type="xs:boolean" use="optional" default="true"/>
<xs:attribute name="gnupgHome" type="archrepo:t_path" use="optional" default="~/.gnupg"/>
<xs:attribute name="gpgKeyID" type="archrepo:t_gpgKeyID" use="optional"/>
</xs:complexType>
</xs:element>
</xs:choice>
</xs:complexType>
</xs:element>
</xs:schema>

191
build.py Executable file
View File

@ -0,0 +1,191 @@
#!/usr/bin/env python3

# TODO: make as flexible as the <rpms>:/bin/build.py (flesh out args), logging, etc.

import argparse
import datetime
import copy
import io
import os
import pathlib
import re
import shutil
import subprocess
import tarfile
import tempfile
import warnings
##
import gpg
import requests
from lxml import etree


# TODO: track which versions are built so we don't need to consistently rebuild ALL packages
# TODO: should this be a configuration option?
aurbase = 'https://aur.archlinux.org'

_dflts = {'cfgfile': '~/.config/optools/arch/autorepo.xml'}


class Packager(object):
def __init__(self, cfgfile = _dflts['cfgfile'], *args, **kwargs):
user_params = kwargs
self.args = copy.deepcopy(_dflts)
self.args.update(user_params)
self.origdir = os.path.abspath(os.path.expanduser(os.getcwd()))
self.gpg = None
self.args['destdir'] = os.path.abspath(os.path.expanduser(self.args['destdir']))
if not self.args['pkgs']:
self.args['pkgs'] = _dflts['pkgs']
self._initSigner()

def buildPkgs(self, auronly = None):
for p in self.args['pkgs']:
print(p)
extract_dir = tempfile.mkdtemp(prefix = '.pkgbuilder.{0}-'.format(p))
sub_extract_dir = os.path.join(extract_dir, p)
has_pkg = False
if not auronly:
has_pkg = self._getLocal(p, extract_dir)
if not has_pkg:
has_pkg = self._getAUR(p, extract_dir)
if not has_pkg:
warnings.warn('Could not find package {0}; skipping...'.format(p))
continue
# We get a list of files to compare.
prebuild_files = []
postbuild_files = []
for root, dirs, files in os.walk(sub_extract_dir):
for f in files:
prebuild_files.append(os.path.join(root, f))
os.chdir(os.path.join(extract_dir, p))
# customizepkg-scripting in AUR
try:
custpkg_out = subprocess.run(['/usr/bin/customizepkg',
'-m'],
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
except FileNotFoundError:
pass # Not installed
build_out = subprocess.run(['/usr/bin/multilib-build',
'-c',
'--',
'--',
'--skippgpcheck',
'--syncdeps',
'--noconfirm',
'--log',
'--holdver',
'--skipinteg'],
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
# with open('/tmp/build.log-{0}'.format(p), 'w') as f:
# f.write(build_out.stdout.decode('utf-8'))
for root, dirs, files in os.walk(sub_extract_dir):
for f in files:
fpath = os.path.join(root, f)
if fpath in prebuild_files:
continue
if fpath.endswith('.log'):
continue
postbuild_files.append(fpath)
postbuild_files = [i for i in postbuild_files if i.endswith('.pkg.tar.xz')]
if len(postbuild_files) != 1:
warnings.warn('Could not reliably find a built package for {0}; skipping'.format(p))
else:
fdest = os.path.join(self.args['destdir'],
os.path.basename(postbuild_files[0]))
if os.path.isfile(fdest):
os.remove(fdest)
shutil.move(postbuild_files[0], fdest)
self._sign(fdest)
os.chdir(self.origdir)
shutil.rmtree(extract_dir)
return()

def _initSigner(self):
self.gpg = gpg.Context()
# Just grab the first private key until we flesh this out.
for k in self.gpg.keylist(secret = True):
if k.can_sign:
self.gpg.signers = [k]
break
return()

def _getAUR(self, pkgnm, extract_dir):
dl_url = None
pkg_srch = requests.get(os.path.join(self.args['aurbase'],
'rpc'),
params = {
'v': 5,
'type': 'search',
'by': 'name',
'arg': pkgnm}).json()
for pkg in pkg_srch['results']:
dl_url = None
if pkg['Name'] == pkgnm:
dl_url = os.path.join(self.args['aurbase'], re.sub('^/+', '', pkg['URLPath']))
# dl_file = os.path.basename(pkg['URLPath'])
break
if not dl_url:
warnings.warn('Could not find a download path for {0}; skipping'.format(pkgnm))
return(False)
with requests.get(dl_url, stream = True) as url:
with tarfile.open(mode = 'r|*', fileobj = io.BytesIO(url.content)) as tar:
tar.extractall(extract_dir)
return(True)

def _getLocal(self, pkgnm, extract_dir):
curfile = os.path.realpath(os.path.abspath(os.path.expanduser(__file__)))
localpkg_dir = os.path.abspath(os.path.join(os.path.dirname(curfile),
'..',
'local_pkgs'))
pkgbuild_dir = os.path.join(localpkg_dir,
pkgnm)
if not os.path.isdir(pkgbuild_dir):
return(False)
shutil.copytree(pkgbuild_dir, os.path.join(extract_dir, pkgnm))
return(True)

def _sign(self, pkgfile, passphrase = None):
sigfile = '{0}.sig'.format(pkgfile)
with open(pkgfile, 'rb') as pkg:
with open(sigfile, 'wb') as sig:
# We want ascii-armoured detached sigs
sig.write(self.gpg.sign(pkg.read(), mode = gpg.constants.SIG_MODE_DETACH)[0])
return()

def createRepo(self):
pkgfiles = []
for root, dirs, files in os.walk(self.args['destdir']):
for f in files:
if f.endswith('.pkg.tar.xz'):
pkgfiles.append(os.path.join(root, f))
repo_out = subprocess.run(['/usr/bin/repo-add',
'-s',
'-R',
os.path.join(self.args['destdir'], '{0}.db.tar.xz'.format(self.args['reponame'])),
*pkgfiles],
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
return()


def parseArgs():
args = argparse.ArgumentParser(description = 'Build Pacman packages and update a local repository')
args.add_argument('-c', '--config',
dest = 'cfgfile',
default = _dflts['cfgfile'],
help = ('The path to the configuration file. Default: {0}').format(_dflts['cfgfile']))
return(args)

def main():
args = parseArgs().parse_args()
varargs = vars(args)
pkgr = Packager(**varargs)
pkgr.buildPkgs(auronly = varargs['auronly'])
pkgr.createRepo()
return()

if __name__ == '__main__':
main()

112
example.pkgs.xml Normal file
View File

@ -0,0 +1,112 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!--
The parsing supports XInclude (https://www.w3.org/TR/xinclude/).
You can use external XML snippets if that's easier/cleaner (it usually is).
-->
<archrepo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://git.square-r00t.net/OpTools/tree/arch/autorepo/"
xsi:schemaLocation="http://git.square-r00t.net/OpTools/plain/arch/autorepo/autorepo.xsd">
<!--
The repo element contains information for each repository we should build for.
Attributes:
name: The name of the repository. This is used for the db name and to generate pacman.conf snippets.
staging: The path to the staging directory. This is where we will build packages and sync to mirrors from.
signPkgs: Either "1"/"true" or "0"/"false". Whether or not we should sign packages. See signDB, gnupgHome,
and gpgKeyID.
signDB: Either "1"/"true" or "0"/"false". Whether or not we should sign the database files. See signPkgs,
gnupgHome, and gpgKeyID.
gnupgHome: The path to use for the GnuPG home (GNUPGHOME environment variable).
The order of preference follows:
1.) gnupgHome attribute (if set)
2.) $GNUPGHOME env var (if set)
3.) ~/.gnupg
See signPkgs, signDB, and gpgKeyID.
gpgKeyID: The key ID to use. It *must* have the signing ("S") capability. If it is a subkey fingerprint,
that subkey will be used. If a subkey fpr is specified but lacks the signing capability, the
(parent) key will be used (if it has signing capability). If no key ID/fingerprint/etc. is
specified, we will use the first key with signing capability found (this should be fine if you
only have one key with signing capabilities in your gnupgHome). If no suitable key is found but
signing is enabled, an error will be thrown. See signPkgs, signDB, and gnupgHome.
-->
<repo
name="testrepo"
staging="/var/tmp/arch/autorepo"
signPkgs="true"
signDB="true"
gnupgHome="~/.gnupg"
gpgKeyID="0x748231EBCBD808A14F5E85D28C004C2F93481F6B">
<!--
The mirrors element contains either localMirror elements or remoteMirror elements (see below).
There must be at least 1 of either type.
-->
<mirrors>
<!-- localMirror elements contain the path to a local mirror (exists on the same system as you're building
from). Most users will probably want this if their build box and mirror are the same machine, or if
you only want a local repository.
Attributes:
user: The user to chown the files/directories to (must be running as root user). If not
specified, the default is the current user (or the user calling sudo, if done via sudo).
group: The group to chown the files/directories to (must be running as root user). If not
specified, the default is the primary group for the current user (or the user calling
sudo, if done via sudo).
fileMode: The octal permissions to chmod the files to.
dirMode: The octal permissions to chmod the directories to.
-->
<localMirror
user="foo"
group="bar"
fileMode="0600"
dirMode="0700">/path/to/path</localMirror>
<localMirror>a/relative/path</localMirror>
<!--
The remoteMirror element is for rsyncing packages to a remote mirror/repo server. Rsync must be installed
locally (it should; it's part of base-devel) *and* the remote server. Obviously, SSH pubkey auth must also
be set up as well for the user. They must have a valid shell on the server for chmodding/chowning.
Attributes:
user: The (remote) user to sync as (e.g. for "ssh foo@bar", user would be "foo").
server: The server to sync to. Can be an IP address, hostname (if resolvable), or FQDN.
port: The remote SSH port.
key: The pubkey to use to connect.
remoteUser: The (remote) user to chown the files/directories to (must be connecting as root user).
If not specified, the default is the connecting user ("user" attribute).
remoteGroup: The (remote) group to chown the files/directories to (must be connecting as root user).
If not specified, the default is the connecting user's ("user" attribute) primary
group.
fileMode: The octal permissions to chmod the remote files to.
dirMode: The octal permissions to chmod the remote directories to.
-->
<remoteMirror
user="foo"
server="bar.domain.tld"
port="22"
key="~/.ssh/id_rsa"
remoteUser="foo"
remoteGroup="bar"
fileMode="0600"
dirMode="0700">/path/to/remote/path
</remoteMirror>
</mirrors>
<!--
The packages element contains actual packages to build into the repository.
-->
<packages>
<!--
The aur element specifies packages that should be fetched and built from the AUR.
They contain the name of the package.
Attributes:
alwaysBuild: Accepts "1"/"true" or "0"/"false". If true, always build the package even if the same
version exists already. This only works if you don't delete/empty your staging
directory, otherwise it will be built.
-->
<aur alwaysBuild="true">somepkg</aur>
<!--
The pkgbuild element specifies packages that are locally developed/designed.
They contain the name of the package.
Attributes:
path: The path to the package to build.
-->
<pkgbuild path="/path/to/pkgnm.snapshot.tar.gz" alwaysBuild="true">pkgnm</pkgbuild>
<pkgbuild path="/path/to/PKGBUILD" alwaysBuild="false">pkgnm2</pkgbuild>
</packages>
</repo>
</archrepo>

9
readme.txt Normal file
View File

@ -0,0 +1,9 @@
This has a lot of work pending. I need to factor in configuration files, etc.

But it does require the following packages to be installed, and the buildbox (not the repo mirror server itself) needs to be Arch:

- pacman (duh)
- namcap
- devtools (for https://wiki.archlinux.org/index.php/DeveloperWiki:Building_in_a_clean_chroot)

It is designed to be run as a *non-root* user. Use the regen_sudoers.py script to create a sudoers CMND_ALIAS (https://www.sudo.ws/man/1.7.10/sudoers.man.html) to add for your packaging user.

33
regen_sudoers.py Executable file
View File

@ -0,0 +1,33 @@
#!/usr/bin/env python3

import re

sudo_cmds = []

# All of these commands...
cmds = ['/usr/bin/extra-x86_64-build',
'/usr/bin/testing-x86_64-build',
'/usr/bin/staging-x86_64-build',
'/usr/bin/multilib-build',
'/usr/bin/multilib-testing-build',
'/usr/bin/multilib-staging-build',
'/usr/bin/makechrootpkg']

# Should allow all of these args.
args = ['-c',
'-c -- -- --skippgpcheck --syncdeps --noconfirm --log --holdver --skipinteg',
'-- -- --skippgpcheck --syncdeps --noconfirm --log --holdver --skipinteg']

for c in cmds:
for a in args:
sudo_cmds.append('{0} {1}'.format(c, a))

s = ''

s += 'Cmnd_Alias\tPKGBUILDER = \\\n'
for c in sudo_cmds:
s += '\t\t\t\t{0}, \\\n'.format(c)

s = re.sub(r', \\s*$', '', s)
print(s)

15
sync.sh Executable file
View File

@ -0,0 +1,15 @@
#!/bin/bash

# This obviously will require some tweaking. Will roll into build.py later.
set -e

server=my_repo.domain.tld
port=2222
user=pkgrepo
src=~/pkgs/built/.
# You should use rrsync to restrict to a specific directory
dest='Arch/.'

echo "Syncing..."
rsync -a --delete -e "ssh -p ${port}" ${src} ${user}@${server}:${dest}
echo "Done."