summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorbrent s <r00t@square-r00t.net>2019-09-18 03:49:52 -0400
committerbrent s <r00t@square-r00t.net>2019-09-18 03:49:52 -0400
commit62a7d65be53f06a652499a4d994552a216a09977 (patch)
tree5ff1dc5c1bfe27276eef61a580fc2514a3626b9c
parent6c7f0a3a6f505c40475e7a6d41ce9137dfd8f155 (diff)
downloadOpTools-62a7d65be53f06a652499a4d994552a216a09977.tar.xz
committing
-rw-r--r--arch/autorepo/autorepo.xsd141
-rwxr-xr-xarch/autorepo/build.py40
-rw-r--r--arch/autorepo/example.pkgs.xml112
3 files changed, 264 insertions, 29 deletions
diff --git a/arch/autorepo/autorepo.xsd b/arch/autorepo/autorepo.xsd
new file mode 100644
index 0000000..27622ac
--- /dev/null
+++ b/arch/autorepo/autorepo.xsd
@@ -0,0 +1,141 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
+ targetNamespace="http://git.square-r00t.net/OpTools/tree/arch/autorepo/"
+ xmlns="http://git.square-r00t.net/OpTools/tree/arch/autorepo/tree/"
+ xmlns:archrepo="http://git.square-r00t.net/OpTools/tree/arch/autorepo/"
+ elementFormDefault="qualified"
+ attributeFormDefault="unqualified">
+
+ <xs:simpleType name="t_posixUserGroup">
+ <xs:restriction base="xs:token">
+ <xs:pattern value="[a-z_]([a-z0-9_-]{0,31}|[a-z0-9_-]{0,30}$)"/>
+ <xs:pattern value="%same"/>
+ <xs:whiteSpace value="collapse"/>
+ </xs:restriction>
+ </xs:simpleType>
+
+ <xs:simpleType name="t_posixMode">
+ <xs:restriction base="xs:positiveInteger">
+ <xs:pattern value="[0-7]?[0-7]{3}"/>
+ <xs:whiteSpace value="collapse"/>
+ </xs:restriction>
+ </xs:simpleType>
+
+ <xs:simpleType name="t_path">
+ <xs:restriction base="xs:string">
+ <xs:pattern value="(/|~/)?([A-Za-z0-9+_.-]+/)*[A-Za-z0-9+_.-]+"/>
+ <xs:whiteSpace value="collapse"/>
+ </xs:restriction>
+ </xs:simpleType>
+
+ <xs:simpleType name="t_port">
+ <xs:restriction base="xs:positiveInteger">
+ <!-- MAN I wish XSD let you validate based on numerical range. -->
+ <!-- https://stackoverflow.com/a/40213676/733214 -->
+ <xs:pattern value="([1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])"/>
+ </xs:restriction>
+ </xs:simpleType>
+
+ <xs:complexType name="t_localMirror">
+ <xs:simpleContent>
+ <xs:extension base="xs:anyURI">
+ <xs:attribute name="user" type="archrepo:t_posixUserGroup" use="optional" default="%same"/>
+ <xs:attribute name="group" type="archrepo:t_posixUserGroup" use="optional" default="%same"/>
+ <xs:attribute name="fileMode" type="archrepo:t_posixMode" use="optional" default="0600"/>
+ <xs:attribute name="dirMode" type="archrepo:t_posixMode" use="optional" default="0700"/>
+ </xs:extension>
+ </xs:simpleContent>
+ </xs:complexType>
+
+ <xs:complexType name="t_remoteMirror">
+ <xs:simpleContent>
+ <xs:extension base="xs:anyURI">
+ <xs:attribute name="user" type="archrepo:t_posixUserGroup"
+ default="%same" use="optional"/>
+ <xs:attribute name="server" type="xs:NMTOKEN" use="required"/>
+ <xs:attribute name="fileMode" type="archrepo:t_posixMode"
+ use="optional" default="0600"/>
+ <xs:attribute name="dirMode" type="archrepo:t_posixMode"
+ use="optional" default="0700"/>
+ <xs:attribute name="port" type="archrepo:t_port"
+ default="22" use="optional"/>
+ <xs:attribute name="key" type="archrepo:t_path"
+ default="~/.ssh/id_rsa" use="optional"/>
+ <xs:attribute name="remoteUser" type="archrepo:t_posixUserGroup"
+ default="%same" use="optional"/>
+ <xs:attribute name="remoteGroup" type="archrepo:t_posixUserGroup"
+ default="%same" use="optional"/>
+ </xs:extension>
+ </xs:simpleContent>
+ </xs:complexType>
+
+ <xs:simpleType name="t_gpgKeyID">
+ <xs:restriction base="xs:string">
+ <xs:pattern value="(0[Xx])?[0-9A-Fa-f]{40}"/>
+ <xs:pattern value="(0[Xx])?[0-9A-Fa-f]{8}"/>
+ <xs:pattern value="(0[Xx])?([0-9A-Fa-f]{4} ?){5} *([0-9A-Fa-f]{4}){5}"/>
+ <xs:whiteSpace value="collapse"/>
+ </xs:restriction>
+ </xs:simpleType>
+
+ <xs:element name="archrepo">
+ <xs:complexType>
+ <xs:choice>
+ <xs:element name="repo" minOccurs="1" maxOccurs="unbounded">
+ <xs:complexType>
+ <xs:all minOccurs="1">
+ <xs:element name="mirrors" minOccurs="1" maxOccurs="1">
+ <xs:complexType>
+ <xs:choice minOccurs="1" maxOccurs="unbounded">
+ <xs:element name="localMirror"
+ maxOccurs="unbounded"
+ type="archrepo:t_localMirror"/>
+ <xs:element name="remoteMirror"
+ maxOccurs="unbounded"
+ type="archrepo:t_remoteMirror"/>
+ </xs:choice>
+ </xs:complexType>
+ </xs:element>
+ <xs:element name="packages" minOccurs="1">
+ <xs:complexType>
+ <xs:choice minOccurs="1" maxOccurs="unbounded">
+ <xs:element name="aur"
+ maxOccurs="unbounded">
+ <xs:complexType>
+ <xs:simpleContent>
+ <xs:extension base="xs:string">
+ <xs:attribute name="alwaysBuild" default="true"
+ type="xs:boolean" use="optional"/>
+ </xs:extension>
+ </xs:simpleContent>
+ </xs:complexType>
+ </xs:element>
+ <xs:element name="pkgbuild"
+ maxOccurs="unbounded">
+ <xs:complexType>
+ <xs:simpleContent>
+ <xs:extension base="xs:token">
+ <xs:attribute name="path" type="archrepo:t_path"
+ default="." use="optional"/>
+ <xs:attribute name="alwaysBuild" default="true"
+ type="xs:boolean" use="optional"/>
+ </xs:extension>
+ </xs:simpleContent>
+ </xs:complexType>
+ </xs:element>
+ </xs:choice>
+ </xs:complexType>
+ </xs:element>
+ </xs:all>
+ <xs:attribute name="name" type="xs:token" use="required"/>
+ <xs:attribute name="staging" type="archrepo:t_path" use="optional" default="."/>
+ <xs:attribute name="signPkgs" type="xs:boolean" use="optional" default="true"/>
+ <xs:attribute name="signDB" type="xs:boolean" use="optional" default="true"/>
+ <xs:attribute name="gnupgHome" type="archrepo:t_path" use="optional" default="~/.gnupg"/>
+ <xs:attribute name="gpgKeyID" type="archrepo:t_gpgKeyID" use="optional"/>
+ </xs:complexType>
+ </xs:element>
+ </xs:choice>
+ </xs:complexType>
+ </xs:element>
+</xs:schema> \ No newline at end of file
diff --git a/arch/autorepo/build.py b/arch/autorepo/build.py
index 7364d6b..5256069 100755
--- a/arch/autorepo/build.py
+++ b/arch/autorepo/build.py
@@ -3,6 +3,7 @@
# TODO: make as flexible as the <rpms>:/bin/build.py (flesh out args), logging, etc.
import argparse
+import datetime
import copy
import io
import os
@@ -16,19 +17,18 @@ import warnings
##
import gpg
import requests
+from lxml import etree
-# TODO: move pkgs to some kind of list/config file/whatever.
# TODO: track which versions are built so we don't need to consistently rebuild ALL packages
-# You will probably want to change these.
-_dflts = {'pkgs': ['dumpet'],
- 'reponame': 'MY_REPO',
- 'destdir': '~/pkgs/built',
- 'aurbase': 'https://aur.archlinux.org'}
+# TODO: should this be a configuration option?
+aurbase = 'https://aur.archlinux.org'
+
+_dflts = {'cfgfile': '~/.config/optools/arch/autorepo.xml'}
class Packager(object):
- def __init__(self, *args, **kwargs):
+ def __init__(self, cfgfile = _dflts['cfgfile'], *args, **kwargs):
user_params = kwargs
self.args = copy.deepcopy(_dflts)
self.args.update(user_params)
@@ -173,28 +173,10 @@ class Packager(object):
def parseArgs():
args = argparse.ArgumentParser(description = 'Build Pacman packages and update a local repository')
- args.add_argument('-p', '--package',
- dest = 'pkgs',
- action = 'append',
- help = ('If specified, only build for this package name. Can be specified multiple times. '
- '(Default is hardcoded: {0})').format(', '.join(_dflts['pkgs'])))
- args.add_argument('-r', '--repo-name',
- dest = 'reponame',
- default = _dflts['reponame'],
- help = ('The name of the repo. Default: {0}').format(_dflts['reponame']))
- args.add_argument('-d', '--dest-dir',
- dest = 'destdir',
- default = _dflts['destdir'],
- help = ('Where the built packages should go. Default: {0}').format(_dflts['destdir']))
- args.add_argument('-a', '--aur-base',
- dest = 'aurbase',
- default = _dflts['aurbase'],
- help = ('The base URL for AUR. You probably don\'t want to change this. '
- 'Default: {0}').format(_dflts['aurbase']))
- args.add_argument('-A', '--aur-only',
- dest = 'auronly',
- action = 'store_true',
- help = ('If specified, ignore local PKGBUILDs and only build from AUR'))
+ args.add_argument('-c', '--config',
+ dest = 'cfgfile',
+ default = _dflts['cfgfile'],
+ help = ('The path to the configuration file. Default: {0}').format(_dflts['cfgfile']))
return(args)
def main():
diff --git a/arch/autorepo/example.pkgs.xml b/arch/autorepo/example.pkgs.xml
new file mode 100644
index 0000000..8942485
--- /dev/null
+++ b/arch/autorepo/example.pkgs.xml
@@ -0,0 +1,112 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ The parsing supports XInclude (https://www.w3.org/TR/xinclude/).
+ You can use external XML snippets if that's easier/cleaner (it usually is).
+-->
+<archrepo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="http://git.square-r00t.net/OpTools/tree/arch/autorepo/"
+ xsi:schemaLocation="http://git.square-r00t.net/OpTools/plain/arch/autorepo/autorepo.xsd">
+ <!--
+ The repo element contains information for each repository we should build for.
+ Attributes:
+ name: The name of the repository. This is used for the db name and to generate pacman.conf snippets.
+ staging: The path to the staging directory. This is where we will build packages and sync to mirrors from.
+ signPkgs: Either "1"/"true" or "0"/"false". Whether or not we should sign packages. See signDB, gnupgHome,
+ and gpgKeyID.
+ signDB: Either "1"/"true" or "0"/"false". Whether or not we should sign the database files. See signPkgs,
+ gnupgHome, and gpgKeyID.
+ gnupgHome: The path to use for the GnuPG home (GNUPGHOME environment variable).
+ The order of preference follows:
+ 1.) gnupgHome attribute (if set)
+ 2.) $GNUPGHOME env var (if set)
+ 3.) ~/.gnupg
+ See signPkgs, signDB, and gpgKeyID.
+ gpgKeyID: The key ID to use. It *must* have the signing ("S") capability. If it is a subkey fingerprint,
+ that subkey will be used. If a subkey fpr is specified but lacks the signing capability, the
+ (parent) key will be used (if it has signing capability). If no key ID/fingerprint/etc. is
+ specified, we will use the first key with signing capability found (this should be fine if you
+ only have one key with signing capabilities in your gnupgHome). If no suitable key is found but
+ signing is enabled, an error will be thrown. See signPkgs, signDB, and gnupgHome.
+ -->
+ <repo
+ name="testrepo"
+ staging="/var/tmp/arch/autorepo"
+ signPkgs="true"
+ signDB="true"
+ gnupgHome="~/.gnupg"
+ gpgKeyID="0x748231EBCBD808A14F5E85D28C004C2F93481F6B">
+ <!--
+ The mirrors element contains either localMirror elements or remoteMirror elements (see below).
+ There must be at least 1 of either type.
+ -->
+ <mirrors>
+ <!-- localMirror elements contain the path to a local mirror (exists on the same system as you're building
+ from). Most users will probably want this if their build box and mirror are the same machine, or if
+ you only want a local repository.
+ Attributes:
+ user: The user to chown the files/directories to (must be running as root user). If not
+ specified, the default is the current user (or the user calling sudo, if done via sudo).
+ group: The group to chown the files/directories to (must be running as root user). If not
+ specified, the default is the primary group for the current user (or the user calling
+ sudo, if done via sudo).
+ fileMode: The octal permissions to chmod the files to.
+ dirMode: The octal permissions to chmod the directories to.
+ -->
+ <localMirror
+ user="foo"
+ group="bar"
+ fileMode="0600"
+ dirMode="0700">/path/to/path</localMirror>
+ <localMirror>a/relative/path</localMirror>
+ <!--
+ The remoteMirror element is for rsyncing packages to a remote mirror/repo server. Rsync must be installed
+ locally (it should; it's part of base-devel) *and* the remote server. Obviously, SSH pubkey auth must also
+ be set up as well for the user. They must have a valid shell on the server for chmodding/chowning.
+ Attributes:
+ user: The (remote) user to sync as (e.g. for "ssh foo@bar", user would be "foo").
+ server: The server to sync to. Can be an IP address, hostname (if resolvable), or FQDN.
+ port: The remote SSH port.
+ key: The pubkey to use to connect.
+ remoteUser: The (remote) user to chown the files/directories to (must be connecting as root user).
+ If not specified, the default is the connecting user ("user" attribute).
+ remoteGroup: The (remote) group to chown the files/directories to (must be connecting as root user).
+ If not specified, the default is the connecting user's ("user" attribute) primary
+ group.
+ fileMode: The octal permissions to chmod the remote files to.
+ dirMode: The octal permissions to chmod the remote directories to.
+ -->
+ <remoteMirror
+ user="foo"
+ server="bar.domain.tld"
+ port="22"
+ key="~/.ssh/id_rsa"
+ remoteUser="foo"
+ remoteGroup="bar"
+ fileMode="0600"
+ dirMode="0700">/path/to/remote/path
+ </remoteMirror>
+ </mirrors>
+ <!--
+ The packages element contains actual packages to build into the repository.
+ -->
+ <packages>
+ <!--
+ The aur element specifies packages that should be fetched and built from the AUR.
+ They contain the name of the package.
+ Attributes:
+ alwaysBuild: Accepts "1"/"true" or "0"/"false". If true, always build the package even if the same
+ version exists already. This only works if you don't delete/empty your staging
+ directory, otherwise it will be built.
+ -->
+ <aur alwaysBuild="true">somepkg</aur>
+ <!--
+ The pkgbuild element specifies packages that are locally developed/designed.
+ They contain the name of the package.
+ Attributes:
+ path: The path to the package to build.
+ -->
+ <pkgbuild path="/path/to/pkgnm.snapshot.tar.gz" alwaysBuild="true">pkgnm</pkgbuild>
+ <pkgbuild path="/path/to/PKGBUILD" alwaysBuild="false">pkgnm2</pkgbuild>
+ </packages>
+ </repo>
+</archrepo> \ No newline at end of file