summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorbrent s <r00t@square-r00t.net>2019-09-19 02:01:57 -0400
committerbrent s <r00t@square-r00t.net>2019-09-19 02:01:57 -0400
commit72298d7a4c7e0348ee3071b2413564526405437b (patch)
tree62939e8dbb184fca29f842a90ea9ceef97f55950
parent62a7d65be53f06a652499a4d994552a216a09977 (diff)
downloadOpTools-72298d7a4c7e0348ee3071b2413564526405437b.tar.xz
moving autorepo to Arch_Repo_Builder repo
-rw-r--r--arch/autorepo/autorepo.xsd141
-rwxr-xr-xarch/autorepo/build.py191
-rw-r--r--arch/autorepo/example.pkgs.xml112
-rw-r--r--arch/autorepo/readme.txt9
-rwxr-xr-xarch/autorepo/regen_sudoers.py33
-rwxr-xr-xarch/autorepo/sync.sh15
6 files changed, 0 insertions, 501 deletions
diff --git a/arch/autorepo/autorepo.xsd b/arch/autorepo/autorepo.xsd
deleted file mode 100644
index 27622ac..0000000
--- a/arch/autorepo/autorepo.xsd
+++ /dev/null
@@ -1,141 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
- targetNamespace="http://git.square-r00t.net/OpTools/tree/arch/autorepo/"
- xmlns="http://git.square-r00t.net/OpTools/tree/arch/autorepo/tree/"
- xmlns:archrepo="http://git.square-r00t.net/OpTools/tree/arch/autorepo/"
- elementFormDefault="qualified"
- attributeFormDefault="unqualified">
-
- <xs:simpleType name="t_posixUserGroup">
- <xs:restriction base="xs:token">
- <xs:pattern value="[a-z_]([a-z0-9_-]{0,31}|[a-z0-9_-]{0,30}$)"/>
- <xs:pattern value="%same"/>
- <xs:whiteSpace value="collapse"/>
- </xs:restriction>
- </xs:simpleType>
-
- <xs:simpleType name="t_posixMode">
- <xs:restriction base="xs:positiveInteger">
- <xs:pattern value="[0-7]?[0-7]{3}"/>
- <xs:whiteSpace value="collapse"/>
- </xs:restriction>
- </xs:simpleType>
-
- <xs:simpleType name="t_path">
- <xs:restriction base="xs:string">
- <xs:pattern value="(/|~/)?([A-Za-z0-9+_.-]+/)*[A-Za-z0-9+_.-]+"/>
- <xs:whiteSpace value="collapse"/>
- </xs:restriction>
- </xs:simpleType>
-
- <xs:simpleType name="t_port">
- <xs:restriction base="xs:positiveInteger">
- <!-- MAN I wish XSD let you validate based on numerical range. -->
- <!-- https://stackoverflow.com/a/40213676/733214 -->
- <xs:pattern value="([1-9][0-9]{0,3}|[1-5][0-9]{4}|6[0-4][0-9]{3}|65[0-4][0-9]{2}|655[0-2][0-9]|6553[0-5])"/>
- </xs:restriction>
- </xs:simpleType>
-
- <xs:complexType name="t_localMirror">
- <xs:simpleContent>
- <xs:extension base="xs:anyURI">
- <xs:attribute name="user" type="archrepo:t_posixUserGroup" use="optional" default="%same"/>
- <xs:attribute name="group" type="archrepo:t_posixUserGroup" use="optional" default="%same"/>
- <xs:attribute name="fileMode" type="archrepo:t_posixMode" use="optional" default="0600"/>
- <xs:attribute name="dirMode" type="archrepo:t_posixMode" use="optional" default="0700"/>
- </xs:extension>
- </xs:simpleContent>
- </xs:complexType>
-
- <xs:complexType name="t_remoteMirror">
- <xs:simpleContent>
- <xs:extension base="xs:anyURI">
- <xs:attribute name="user" type="archrepo:t_posixUserGroup"
- default="%same" use="optional"/>
- <xs:attribute name="server" type="xs:NMTOKEN" use="required"/>
- <xs:attribute name="fileMode" type="archrepo:t_posixMode"
- use="optional" default="0600"/>
- <xs:attribute name="dirMode" type="archrepo:t_posixMode"
- use="optional" default="0700"/>
- <xs:attribute name="port" type="archrepo:t_port"
- default="22" use="optional"/>
- <xs:attribute name="key" type="archrepo:t_path"
- default="~/.ssh/id_rsa" use="optional"/>
- <xs:attribute name="remoteUser" type="archrepo:t_posixUserGroup"
- default="%same" use="optional"/>
- <xs:attribute name="remoteGroup" type="archrepo:t_posixUserGroup"
- default="%same" use="optional"/>
- </xs:extension>
- </xs:simpleContent>
- </xs:complexType>
-
- <xs:simpleType name="t_gpgKeyID">
- <xs:restriction base="xs:string">
- <xs:pattern value="(0[Xx])?[0-9A-Fa-f]{40}"/>
- <xs:pattern value="(0[Xx])?[0-9A-Fa-f]{8}"/>
- <xs:pattern value="(0[Xx])?([0-9A-Fa-f]{4} ?){5} *([0-9A-Fa-f]{4}){5}"/>
- <xs:whiteSpace value="collapse"/>
- </xs:restriction>
- </xs:simpleType>
-
- <xs:element name="archrepo">
- <xs:complexType>
- <xs:choice>
- <xs:element name="repo" minOccurs="1" maxOccurs="unbounded">
- <xs:complexType>
- <xs:all minOccurs="1">
- <xs:element name="mirrors" minOccurs="1" maxOccurs="1">
- <xs:complexType>
- <xs:choice minOccurs="1" maxOccurs="unbounded">
- <xs:element name="localMirror"
- maxOccurs="unbounded"
- type="archrepo:t_localMirror"/>
- <xs:element name="remoteMirror"
- maxOccurs="unbounded"
- type="archrepo:t_remoteMirror"/>
- </xs:choice>
- </xs:complexType>
- </xs:element>
- <xs:element name="packages" minOccurs="1">
- <xs:complexType>
- <xs:choice minOccurs="1" maxOccurs="unbounded">
- <xs:element name="aur"
- maxOccurs="unbounded">
- <xs:complexType>
- <xs:simpleContent>
- <xs:extension base="xs:string">
- <xs:attribute name="alwaysBuild" default="true"
- type="xs:boolean" use="optional"/>
- </xs:extension>
- </xs:simpleContent>
- </xs:complexType>
- </xs:element>
- <xs:element name="pkgbuild"
- maxOccurs="unbounded">
- <xs:complexType>
- <xs:simpleContent>
- <xs:extension base="xs:token">
- <xs:attribute name="path" type="archrepo:t_path"
- default="." use="optional"/>
- <xs:attribute name="alwaysBuild" default="true"
- type="xs:boolean" use="optional"/>
- </xs:extension>
- </xs:simpleContent>
- </xs:complexType>
- </xs:element>
- </xs:choice>
- </xs:complexType>
- </xs:element>
- </xs:all>
- <xs:attribute name="name" type="xs:token" use="required"/>
- <xs:attribute name="staging" type="archrepo:t_path" use="optional" default="."/>
- <xs:attribute name="signPkgs" type="xs:boolean" use="optional" default="true"/>
- <xs:attribute name="signDB" type="xs:boolean" use="optional" default="true"/>
- <xs:attribute name="gnupgHome" type="archrepo:t_path" use="optional" default="~/.gnupg"/>
- <xs:attribute name="gpgKeyID" type="archrepo:t_gpgKeyID" use="optional"/>
- </xs:complexType>
- </xs:element>
- </xs:choice>
- </xs:complexType>
- </xs:element>
-</xs:schema> \ No newline at end of file
diff --git a/arch/autorepo/build.py b/arch/autorepo/build.py
deleted file mode 100755
index 5256069..0000000
--- a/arch/autorepo/build.py
+++ /dev/null
@@ -1,191 +0,0 @@
-#!/usr/bin/env python3
-
-# TODO: make as flexible as the <rpms>:/bin/build.py (flesh out args), logging, etc.
-
-import argparse
-import datetime
-import copy
-import io
-import os
-import pathlib
-import re
-import shutil
-import subprocess
-import tarfile
-import tempfile
-import warnings
-##
-import gpg
-import requests
-from lxml import etree
-
-
-# TODO: track which versions are built so we don't need to consistently rebuild ALL packages
-# TODO: should this be a configuration option?
-aurbase = 'https://aur.archlinux.org'
-
-_dflts = {'cfgfile': '~/.config/optools/arch/autorepo.xml'}
-
-
-class Packager(object):
- def __init__(self, cfgfile = _dflts['cfgfile'], *args, **kwargs):
- user_params = kwargs
- self.args = copy.deepcopy(_dflts)
- self.args.update(user_params)
- self.origdir = os.path.abspath(os.path.expanduser(os.getcwd()))
- self.gpg = None
- self.args['destdir'] = os.path.abspath(os.path.expanduser(self.args['destdir']))
- if not self.args['pkgs']:
- self.args['pkgs'] = _dflts['pkgs']
- self._initSigner()
-
- def buildPkgs(self, auronly = None):
- for p in self.args['pkgs']:
- print(p)
- extract_dir = tempfile.mkdtemp(prefix = '.pkgbuilder.{0}-'.format(p))
- sub_extract_dir = os.path.join(extract_dir, p)
- has_pkg = False
- if not auronly:
- has_pkg = self._getLocal(p, extract_dir)
- if not has_pkg:
- has_pkg = self._getAUR(p, extract_dir)
- if not has_pkg:
- warnings.warn('Could not find package {0}; skipping...'.format(p))
- continue
- # We get a list of files to compare.
- prebuild_files = []
- postbuild_files = []
- for root, dirs, files in os.walk(sub_extract_dir):
- for f in files:
- prebuild_files.append(os.path.join(root, f))
- os.chdir(os.path.join(extract_dir, p))
- # customizepkg-scripting in AUR
- try:
- custpkg_out = subprocess.run(['/usr/bin/customizepkg',
- '-m'],
- stdout = subprocess.PIPE,
- stderr = subprocess.PIPE)
- except FileNotFoundError:
- pass # Not installed
- build_out = subprocess.run(['/usr/bin/multilib-build',
- '-c',
- '--',
- '--',
- '--skippgpcheck',
- '--syncdeps',
- '--noconfirm',
- '--log',
- '--holdver',
- '--skipinteg'],
- stdout = subprocess.PIPE,
- stderr = subprocess.PIPE)
- # with open('/tmp/build.log-{0}'.format(p), 'w') as f:
- # f.write(build_out.stdout.decode('utf-8'))
- for root, dirs, files in os.walk(sub_extract_dir):
- for f in files:
- fpath = os.path.join(root, f)
- if fpath in prebuild_files:
- continue
- if fpath.endswith('.log'):
- continue
- postbuild_files.append(fpath)
- postbuild_files = [i for i in postbuild_files if i.endswith('.pkg.tar.xz')]
- if len(postbuild_files) != 1:
- warnings.warn('Could not reliably find a built package for {0}; skipping'.format(p))
- else:
- fdest = os.path.join(self.args['destdir'],
- os.path.basename(postbuild_files[0]))
- if os.path.isfile(fdest):
- os.remove(fdest)
- shutil.move(postbuild_files[0], fdest)
- self._sign(fdest)
- os.chdir(self.origdir)
- shutil.rmtree(extract_dir)
- return()
-
- def _initSigner(self):
- self.gpg = gpg.Context()
- # Just grab the first private key until we flesh this out.
- for k in self.gpg.keylist(secret = True):
- if k.can_sign:
- self.gpg.signers = [k]
- break
- return()
-
- def _getAUR(self, pkgnm, extract_dir):
- dl_url = None
- pkg_srch = requests.get(os.path.join(self.args['aurbase'],
- 'rpc'),
- params = {
- 'v': 5,
- 'type': 'search',
- 'by': 'name',
- 'arg': pkgnm}).json()
- for pkg in pkg_srch['results']:
- dl_url = None
- if pkg['Name'] == pkgnm:
- dl_url = os.path.join(self.args['aurbase'], re.sub('^/+', '', pkg['URLPath']))
- # dl_file = os.path.basename(pkg['URLPath'])
- break
- if not dl_url:
- warnings.warn('Could not find a download path for {0}; skipping'.format(pkgnm))
- return(False)
- with requests.get(dl_url, stream = True) as url:
- with tarfile.open(mode = 'r|*', fileobj = io.BytesIO(url.content)) as tar:
- tar.extractall(extract_dir)
- return(True)
-
- def _getLocal(self, pkgnm, extract_dir):
- curfile = os.path.realpath(os.path.abspath(os.path.expanduser(__file__)))
- localpkg_dir = os.path.abspath(os.path.join(os.path.dirname(curfile),
- '..',
- 'local_pkgs'))
- pkgbuild_dir = os.path.join(localpkg_dir,
- pkgnm)
- if not os.path.isdir(pkgbuild_dir):
- return(False)
- shutil.copytree(pkgbuild_dir, os.path.join(extract_dir, pkgnm))
- return(True)
-
- def _sign(self, pkgfile, passphrase = None):
- sigfile = '{0}.sig'.format(pkgfile)
- with open(pkgfile, 'rb') as pkg:
- with open(sigfile, 'wb') as sig:
- # We want ascii-armoured detached sigs
- sig.write(self.gpg.sign(pkg.read(), mode = gpg.constants.SIG_MODE_DETACH)[0])
- return()
-
- def createRepo(self):
- pkgfiles = []
- for root, dirs, files in os.walk(self.args['destdir']):
- for f in files:
- if f.endswith('.pkg.tar.xz'):
- pkgfiles.append(os.path.join(root, f))
- repo_out = subprocess.run(['/usr/bin/repo-add',
- '-s',
- '-R',
- os.path.join(self.args['destdir'], '{0}.db.tar.xz'.format(self.args['reponame'])),
- *pkgfiles],
- stdout = subprocess.PIPE,
- stderr = subprocess.PIPE)
- return()
-
-
-def parseArgs():
- args = argparse.ArgumentParser(description = 'Build Pacman packages and update a local repository')
- args.add_argument('-c', '--config',
- dest = 'cfgfile',
- default = _dflts['cfgfile'],
- help = ('The path to the configuration file. Default: {0}').format(_dflts['cfgfile']))
- return(args)
-
-def main():
- args = parseArgs().parse_args()
- varargs = vars(args)
- pkgr = Packager(**varargs)
- pkgr.buildPkgs(auronly = varargs['auronly'])
- pkgr.createRepo()
- return()
-
-if __name__ == '__main__':
- main()
diff --git a/arch/autorepo/example.pkgs.xml b/arch/autorepo/example.pkgs.xml
deleted file mode 100644
index 8942485..0000000
--- a/arch/autorepo/example.pkgs.xml
+++ /dev/null
@@ -1,112 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- The parsing supports XInclude (https://www.w3.org/TR/xinclude/).
- You can use external XML snippets if that's easier/cleaner (it usually is).
--->
-<archrepo xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xmlns="http://git.square-r00t.net/OpTools/tree/arch/autorepo/"
- xsi:schemaLocation="http://git.square-r00t.net/OpTools/plain/arch/autorepo/autorepo.xsd">
- <!--
- The repo element contains information for each repository we should build for.
- Attributes:
- name: The name of the repository. This is used for the db name and to generate pacman.conf snippets.
- staging: The path to the staging directory. This is where we will build packages and sync to mirrors from.
- signPkgs: Either "1"/"true" or "0"/"false". Whether or not we should sign packages. See signDB, gnupgHome,
- and gpgKeyID.
- signDB: Either "1"/"true" or "0"/"false". Whether or not we should sign the database files. See signPkgs,
- gnupgHome, and gpgKeyID.
- gnupgHome: The path to use for the GnuPG home (GNUPGHOME environment variable).
- The order of preference follows:
- 1.) gnupgHome attribute (if set)
- 2.) $GNUPGHOME env var (if set)
- 3.) ~/.gnupg
- See signPkgs, signDB, and gpgKeyID.
- gpgKeyID: The key ID to use. It *must* have the signing ("S") capability. If it is a subkey fingerprint,
- that subkey will be used. If a subkey fpr is specified but lacks the signing capability, the
- (parent) key will be used (if it has signing capability). If no key ID/fingerprint/etc. is
- specified, we will use the first key with signing capability found (this should be fine if you
- only have one key with signing capabilities in your gnupgHome). If no suitable key is found but
- signing is enabled, an error will be thrown. See signPkgs, signDB, and gnupgHome.
- -->
- <repo
- name="testrepo"
- staging="/var/tmp/arch/autorepo"
- signPkgs="true"
- signDB="true"
- gnupgHome="~/.gnupg"
- gpgKeyID="0x748231EBCBD808A14F5E85D28C004C2F93481F6B">
- <!--
- The mirrors element contains either localMirror elements or remoteMirror elements (see below).
- There must be at least 1 of either type.
- -->
- <mirrors>
- <!-- localMirror elements contain the path to a local mirror (exists on the same system as you're building
- from). Most users will probably want this if their build box and mirror are the same machine, or if
- you only want a local repository.
- Attributes:
- user: The user to chown the files/directories to (must be running as root user). If not
- specified, the default is the current user (or the user calling sudo, if done via sudo).
- group: The group to chown the files/directories to (must be running as root user). If not
- specified, the default is the primary group for the current user (or the user calling
- sudo, if done via sudo).
- fileMode: The octal permissions to chmod the files to.
- dirMode: The octal permissions to chmod the directories to.
- -->
- <localMirror
- user="foo"
- group="bar"
- fileMode="0600"
- dirMode="0700">/path/to/path</localMirror>
- <localMirror>a/relative/path</localMirror>
- <!--
- The remoteMirror element is for rsyncing packages to a remote mirror/repo server. Rsync must be installed
- locally (it should; it's part of base-devel) *and* the remote server. Obviously, SSH pubkey auth must also
- be set up as well for the user. They must have a valid shell on the server for chmodding/chowning.
- Attributes:
- user: The (remote) user to sync as (e.g. for "ssh foo@bar", user would be "foo").
- server: The server to sync to. Can be an IP address, hostname (if resolvable), or FQDN.
- port: The remote SSH port.
- key: The pubkey to use to connect.
- remoteUser: The (remote) user to chown the files/directories to (must be connecting as root user).
- If not specified, the default is the connecting user ("user" attribute).
- remoteGroup: The (remote) group to chown the files/directories to (must be connecting as root user).
- If not specified, the default is the connecting user's ("user" attribute) primary
- group.
- fileMode: The octal permissions to chmod the remote files to.
- dirMode: The octal permissions to chmod the remote directories to.
- -->
- <remoteMirror
- user="foo"
- server="bar.domain.tld"
- port="22"
- key="~/.ssh/id_rsa"
- remoteUser="foo"
- remoteGroup="bar"
- fileMode="0600"
- dirMode="0700">/path/to/remote/path
- </remoteMirror>
- </mirrors>
- <!--
- The packages element contains actual packages to build into the repository.
- -->
- <packages>
- <!--
- The aur element specifies packages that should be fetched and built from the AUR.
- They contain the name of the package.
- Attributes:
- alwaysBuild: Accepts "1"/"true" or "0"/"false". If true, always build the package even if the same
- version exists already. This only works if you don't delete/empty your staging
- directory, otherwise it will be built.
- -->
- <aur alwaysBuild="true">somepkg</aur>
- <!--
- The pkgbuild element specifies packages that are locally developed/designed.
- They contain the name of the package.
- Attributes:
- path: The path to the package to build.
- -->
- <pkgbuild path="/path/to/pkgnm.snapshot.tar.gz" alwaysBuild="true">pkgnm</pkgbuild>
- <pkgbuild path="/path/to/PKGBUILD" alwaysBuild="false">pkgnm2</pkgbuild>
- </packages>
- </repo>
-</archrepo> \ No newline at end of file
diff --git a/arch/autorepo/readme.txt b/arch/autorepo/readme.txt
deleted file mode 100644
index 09b89ba..0000000
--- a/arch/autorepo/readme.txt
+++ /dev/null
@@ -1,9 +0,0 @@
-This has a lot of work pending. I need to factor in configuration files, etc.
-
-But it does require the following packages to be installed, and the buildbox (not the repo mirror server itself) needs to be Arch:
-
-- pacman (duh)
-- namcap
-- devtools (for https://wiki.archlinux.org/index.php/DeveloperWiki:Building_in_a_clean_chroot)
-
-It is designed to be run as a *non-root* user. Use the regen_sudoers.py script to create a sudoers CMND_ALIAS (https://www.sudo.ws/man/1.7.10/sudoers.man.html) to add for your packaging user.
diff --git a/arch/autorepo/regen_sudoers.py b/arch/autorepo/regen_sudoers.py
deleted file mode 100755
index 8dd8454..0000000
--- a/arch/autorepo/regen_sudoers.py
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env python3
-
-import re
-
-sudo_cmds = []
-
-# All of these commands...
-cmds = ['/usr/bin/extra-x86_64-build',
- '/usr/bin/testing-x86_64-build',
- '/usr/bin/staging-x86_64-build',
- '/usr/bin/multilib-build',
- '/usr/bin/multilib-testing-build',
- '/usr/bin/multilib-staging-build',
- '/usr/bin/makechrootpkg']
-
-# Should allow all of these args.
-args = ['-c',
- '-c -- -- --skippgpcheck --syncdeps --noconfirm --log --holdver --skipinteg',
- '-- -- --skippgpcheck --syncdeps --noconfirm --log --holdver --skipinteg']
-
-for c in cmds:
- for a in args:
- sudo_cmds.append('{0} {1}'.format(c, a))
-
-s = ''
-
-s += 'Cmnd_Alias\tPKGBUILDER = \\\n'
-for c in sudo_cmds:
- s += '\t\t\t\t{0}, \\\n'.format(c)
-
-s = re.sub(r', \\s*$', '', s)
-print(s)
-
diff --git a/arch/autorepo/sync.sh b/arch/autorepo/sync.sh
deleted file mode 100755
index 32a5537..0000000
--- a/arch/autorepo/sync.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/bash
-
-# This obviously will require some tweaking. Will roll into build.py later.
-set -e
-
-server=my_repo.domain.tld
-port=2222
-user=pkgrepo
-src=~/pkgs/built/.
-# You should use rrsync to restrict to a specific directory
-dest='Arch/.'
-
-echo "Syncing..."
-rsync -a --delete -e "ssh -p ${port}" ${src} ${user}@${server}:${dest}
-echo "Done."