adding split-architecture source tarball support... untested and probably not done yet, so not tagging a release

This commit is contained in:
brent s. 2017-03-06 03:21:23 -05:00
parent 47684f989b
commit f26e03fda9
8 changed files with 206 additions and 87 deletions

View File

@ -13,7 +13,11 @@ def genGPG(conf):
bdisk = conf['bdisk']
gpghome = conf['gpg']['mygpghome']
distkey = build['gpgkey']
gpgkeyserver = build['gpgkeyserver']
gpgkeyserver = []
for a in conf['build']['arch']:
keysrv = conf['src'][a]['gpgkeyserver']
if keysrv and (keysrv not in gpgkeyserver):
gpgkeyserver.append(keysrv)
templates_dir = '{0}/extra/templates'.format(build['basedir'])
mykey = False
pkeys = []
@ -31,16 +35,17 @@ def genGPG(conf):
os.environ['GNUPGHOME'] = gpghome
gpg = gpgme.Context()
# do we need to add a keyserver?
if gpgkeyserver != '':
if len(gpgkeyserver) != 0:
dirmgr = '{0}/dirmngr.conf'.format(gpghome)
if os.path.isfile(dirmgr):
with open(dirmgr, 'r+') as f:
findme = any(gpgkeyserver in line for line in f)
if not findme:
f.seek(0, os.SEEK_END)
f.write("\n# Added by {0}.\nkeyserver {1}\n".format(
bdisk['pname'],
gpgkeyserver))
for s in gpgkeyserver:
if os.path.isfile(dirmgr):
with open(dirmgr, 'r+') as f:
findme = any(s in line for line in f)
if not findme:
f.seek(0, os.SEEK_END)
f.write("\n# Added by {0}.\nkeyserver {1}\n".format(
bdisk['pname'],
s))
if mykey:
try:
privkey = gpg.get_key(mykey, True)
@ -62,15 +67,16 @@ def genGPG(conf):
privkey = gpg.get_key(gpg.genkey(tpl_out).fpr, True)
pkeys.append(privkey)
# do we need to add a keyserver? this is for the freshly-generated GNUPGHOME
if build['gpgkeyserver'] != '':
if len(gpgkeyserver) != 0:
dirmgr = '{0}/dirmngr.conf'.format(gpghome)
with open(dirmgr, 'r+') as f:
findme = any(gpgkeyserver in line for line in f)
if not findme:
f.seek(0, os.SEEK_END)
f.write("\n# Added by {0}.\nkeyserver {1}\n".format(
bdisk['pname'],
build['gpgkeyserver']))
for s in gpgkeyserver:
with open(dirmgr, 'r+') as f:
findme = any(s in line for line in f)
if not findme:
f.seek(0, os.SEEK_END)
f.write("\n# Added by {0}.\nkeyserver {1}\n".format(
bdisk['pname'],
s))
gpg.signers = pkeys
# Now we try to find and add the key for the base image.
gpg.keylist_mode = gpgme.KEYLIST_MODE_EXTERN # remote (keyserver)
@ -125,7 +131,7 @@ def killStaleAgent(conf):
psutil.Process(p).terminate()

def signIMG(path, conf):
if conf['build']['gpg']:
if conf['build']['sign']:
# Do we want to kill off any stale gpg-agents? (So we spawn a new one)
# Requires further testing.
#killStaleAgent()

View File

@ -68,9 +68,9 @@ def parseConfig(confs):
config_dict = {s:dict(config.items(s)) for s in config.sections()}
# Convert the booleans to pythonic booleans in the dict...
config_dict['bdisk']['user'] = config['bdisk'].getboolean('user')
config_dict['build']['gpg'] = config['build'].getboolean('gpg')
config_dict['build']['i_am_a_racecar'] = config['build'].getboolean('i_am_a_racecar')
config_dict['build']['ipxe'] = config['build'].getboolean('ipxe')
config_dict['build']['sign'] = config['build'].getboolean('sign')
config_dict['build']['multiarch'] = (config_dict['build']['multiarch']).lower()
config_dict['ipxe']['iso'] = config['ipxe'].getboolean('iso')
config_dict['ipxe']['usb'] = config['ipxe'].getboolean('usb')
@ -126,16 +126,21 @@ def parseConfig(confs):
config_dict['build']['multiarch']))
## VALIDATORS ##
# Validate bootstrap mirror
if (validators.domain(config_dict['build']['mirror']) or validators.ipv4(
config_dict['build']['mirror']) or validatords.ipv6(
config_dict['build']['mirror'])):
try:
getaddrinfo(config_dict['build']['mirror'], None)
except:
exit(('{0}: ERROR: {1} does not resolve and cannot be used as a ' +
'mirror for the bootstrap tarballs. Check your configuration.').format(
datetime.datetime.now(),
config_dict['build']['host']))
config_dict['src'] = {}
for a in config_dict['build']['arch']:
config_dict['src'][a] = config_dict['source_' + a]
if config_dict['src'][a]['enabled']:
if (validators.domain(config_dict['src'][a]['mirror']) or validators.ipv4(
config_dict['src'][a]['mirror']) or validatords.ipv6(
config_dict['src'][a]['mirror'])):
try:
getaddrinfo(config_dict['src'][a]['mirror'], None)
except:
exit(('{0}: ERROR: {1} does not resolve and cannot be used as a ' +
'mirror for the bootstrap tarballs. Check your configuration.').format(
datetime.datetime.now(),
config_dict['src'][a]['host']))
config_dict['src'][a]['gpg'] = config['source_' + a].getboolean('gpg')
# Are we rsyncing? If so, validate the rsync host.
# Works for IP address too. It does NOT check to see if we can
# actually *rsync* to it; that'll come later.

View File

@ -25,27 +25,36 @@ def dirChk(conf):
def downloadTarball(conf):
build = conf['build']
dlpath = build['dlpath']
src = conf['src']
arch = build['arch']
#mirror = 'http://mirrors.kernel.org/archlinux'
mirror = build['mirrorproto'] + '://' + build['mirror']
rlsdir = mirror + build['mirrorpath']
sha_in = urlopen(mirror + build['mirrorchksum'])
# returns path/filename e.g. /some/path/to/file.tar.gz
# we use .gnupg since we'll need it later.
os.makedirs(dlpath + '/.gnupg', exist_ok = True)
tarball_path = {}
for x in arch:
tarball_path[x] = dlpath + '/.latest.' + x + '.tar'
sha1sums = sha_in.read()
sha_in.close()
sha_raw = sha1sums.decode("utf-8")
sha_list = list(filter(None, sha_raw.split('\n')))
sha_dict = {x.split()[1]: x.split()[0] for x in sha_list}
# all that lousy work just to get a sha1 sum. okay. so.
for a in arch:
locsrc = conf['source_' + a]
mirror = locsrc['mirrorproto'] + '://' + locsrc['mirror']
rlsdir = mirror + locsrc['mirrorpath']
if locsrc['mirrorchksum'] != '':
if not locsrc['chksumtype']:
exit(("{0}: source_{1}:chksumtype is unset!".format(
datetime.datetime.now(),
a))
hash_type = locsrc['chksumtype']
hash_in = urlopen(mirror + locsrc['mirrorchksum'])
hashsums = hash_in.read()
hash_in.close()
hash_raw = hashsums.decode("utf-8")
hash_list = list(filter(None, hash_raw.split('\n')))
hash_dict = {x.split()[1]: x.split()[0] for x in hash_list}
# returns path/filename e.g. /some/path/to/file.tar.gz
# we use .gnupg since we'll need it later.
os.makedirs(dlpath + '/.gnupg', exist_ok = True)
tarball_path[a] = dlpath + '/.latest.' + a + '.tar'
pattern = re.compile('^.*' + a + '\.tar(\.(gz|bz2|xz))?$')
tarball = [filename.group(0) for l in list(sha_dict.keys()) for filename in [pattern.search(l)] if filename][0]
sha1 = sha_dict[tarball]
if locsrc['mirrorfile'] != '':
tarball = locsrc['mirrorfile']
else:
tarball = [filename.group(0) for l in list(hash_dict.keys()) for filename in [pattern.search(l)] if filename][0]
if locsrc['mirrorchksum'] != '':
hashsum = hash_dict[tarball]
if os.path.isfile(tarball_path[a]):
pass
else:
@ -53,7 +62,6 @@ def downloadTarball(conf):
print("{0}: [PREP] Fetching tarball ({1} architecture)...".format(
datetime.datetime.now(),
a))
#dl_file = urllib.URLopener()
tarball_dl = urlopen(rlsdir + tarball)
with open(tarball_path[a], 'wb') as f:
f.write(tarball_dl.read())
@ -63,20 +71,32 @@ def downloadTarball(conf):
tarball_path[a],
humanize.naturalsize(
os.path.getsize(tarball_path[a]))))
print("{0}: [PREP] Checking hash checksum {1} against {2}...".format(
datetime.datetime.now(),
sha1,
tarball_path[a]))
tarball_hash = hashlib.sha1(open(tarball_path[a], 'rb').read()).hexdigest()
if tarball_hash != sha1:
exit(("{0}: {1} either did not download correctly\n\t\t\t or a wrong (probably old) version exists on the filesystem.\n\t\t\t " +
"Please delete it and try again.").format(datetime.datetime.now(), tarball))
elif build['mirrorgpgsig'] != '':
# okay, so the sha1 matches. let's verify the signature.
if build['mirrorgpgsig'] == '.sig':
if locsrc['mirrorchksum'] != '':
print("{0}: [PREP] Checking hash checksum {1} against {2}...".format(
datetime.datetime.now(),
hashsum,
tarball_path[a]))
# Calculate the checksum according to type specified.
tarball_hash = False
for i in hashlib.algorithms_available:
if hash_type == i:
hashfunc = getattr(hashlib, i)
tarball_hash = hashfunc(open(tarball_path[a], 'rb').read()).hexdigest()
break
if not tarball_hash:
exit(("{0}: source_{1}:chksumtype '{2}' is not supported on this machine!".format(
datetime.datetime.now(),
a,
hash_type))
if tarball_hash != hashsum:
exit(("{0}: {1} either did not download correctly\n\t\t\t or a wrong (probably old) version exists on the filesystem.\n\t\t\t " +
"Please delete it and try again.").format(datetime.datetime.now(), tarball))
if locsrc['mirrorgpgsig'] != '':
# let's verify the signature.
if locsrc['mirrorgpgsig'] == '.sig':
gpgsig_remote = rlsdir + tarball + '.sig'
else:
gpgsig_remote = build['mirrorgpgsig']
gpgsig_remote = locsrc['mirrorgpgsig']
sig_dl = urlopen(gpgsig_remote)
sig = tarball_path[a] + '.sig'
with open(sig, 'wb+') as f:

View File

@ -9,15 +9,15 @@
-- https://code.google.com/p/byte-unixbench/
-- https://github.com/akopytov/sysbench
-- (http://blog.due.io/2014/linode-digitalocean-and-vultr-comparison/ etc.)
-implement pyalpm to decreate dependency on chroot pacman-ing?
-There *has* to be a better way of handling package installation in the chroots.
--implement pyalpm to decreate dependency on chroot pacman-ing?
--or even maybe https://wiki.archlinux.org/index.php/offline_installation_of_packages in pure python!
-set up automatic exporting to PDF of the user manual server-side. https://pypi.python.org/pypi/unoconv/0.6
-There *has* to be a better way of handling package installation in the chroots.
-maybe remove lxde, firefox, chrome and replace with enlightenment/midori?
-custom repo? https://brainwreckedtech.wordpress.com/2013/01/27/making-your-own-arch-linux-repository/
--https://wiki.archlinux.org/index.php/Building_32-bit_packages_on_a_64-bit_system
--https://wiki.archlinux.org/index.php/Building_32-bit_packages_on_a_64-bit_system # NOTE: arch has dropped i686, now continued as archlinux32
-implement better "additional" packages list. specify for path in build.ini- these should be more easily changed by end users. DON'T TOUCH iso.pkgs.lst since those are necessary for booting.
-shorewall/some other firewall?
-automatic shorewall/some other firewall?
-autodetection/configuration of network. DHCP is currently running by default, but does it need to support IPv6? if so, how would the user configure their network?
-DISABLE NETWORKMANAGER AND "fi.w1.wpa_supplicant1"??? keeps spawning wpa_supplicant (and thusly killing networking proper)
-for netboot, custom user agent (should be defined by build.ini)
@ -27,7 +27,7 @@
-WISH: signing for secureboot releases (PreLoader and loader.efi handle this okay, but require manual intervention)
-does loader.efi support splash backgrounds? can i implement that differently somehow?
--yes, see e.g. https://www.reddit.com/r/archlinux/comments/3bwgf0/where_put_the_splasharchbmp_to_splash_screen_boot/
-strip out/remove unnecessary and orphan packages (e.g. gcc, make, automake, etc.)
-strip out/remove unnecessary and orphan packages (e.g. gcc, make, automake, etc.) before building ISO
-incorporate iPXE tweaks:
--http://ipxe.org/crypto
--http://ipxe.org/cmd/imgtrust
@ -39,6 +39,7 @@
---#imgverify initrd path/to/initrd.sig
---DONE, partially. need to incorporate codesign certs/keys. routines, conf variables
-enable mirror= kernel commandline.
-NOTE: Following should be implemented via AIF-NG (https://git.square-r00t.net/AIF-NG, work pending for fix to BDisk for i686/x86_64 split)
--if mirror_(NAME) is present, use that as repo name.
--if it starts with /, treat as mirrorlist (Include); otherwise use Server =
--if it has mirror_SIG-X, set signature options e.g. _SIG-N would be "SigLevel = Never"
@ -46,6 +47,4 @@
--note that iPXE VESAFB console is not (yet) supported in EFI, so this is on hold. check into this to see if it has changed.
-include WinMTR, build Mac OS X MTR for dist/tools on CD
-include pre-compiled LibreCrypt for opening LUKS parts on Windows (https://github.com/t-d-k/LibreCrypt)
--curl -s https://raw.githubusercontent.com/t-d-k/LibreCrypt/master/README.md | egrep 'InstallLibreCrypt_v[A-Za-z0-9\.]*.exe' | cut -f2 -d'"'


--curl -s https://raw.githubusercontent.com/t-d-k/LibreCrypt/master/README.md | egrep 'InstallLibreCrypt_v[A-Za-z0-9\.]*.exe' | cut -f2 -d'"'

View File

@ -1,6 +1,6 @@
= BDisk User and Developer Manual
Brent Saner <bts@square-r00t.net>
v1.0, 2016-12
v1.1, 2017-03-06
:doctype: book
:data-uri:
:imagesdir: images

View File

@ -24,7 +24,7 @@ We'll go into more detail for each section below.
name = BDISK
uxname = bdisk
pname = BDisk
ver =
ver =
dev = A Developer
email = dev@domain.tld
desc = A rescue/restore live environment.
@ -35,16 +35,27 @@ We'll go into more detail for each section below.
username = ${bdisk:uxname}
name = Default user
password = $$6$$t92Uvm1ETLocDb1D$$BvI0Sa6CSXxzIKBinIaJHb1gLJWheoXp7WzdideAJN46aChFu3hKg07QaIJNk4dfIJ2ry3tEfo3FRvstKWasg/
[build]
[source_x86_64]
mirror = mirror.us.leaseweb.net
mirrorproto = https
mirrorpath = /archlinux/iso/latest/
mirrorfile =
mirrorfile = .sig
mirrorchksum = ${mirrorpath}sha1sums.txt
chksumtype = sha1
mirrorgpgsig =
gpgkey = 7F2D434B9741E8AC
gpgkeyserver =
gpg = no
[source_i686]
mirror = mirror.us.leaseweb.net
mirrorproto = https
mirrorpath = /archlinux/iso/latest/
mirrorfile =
mirrorchksum = ${mirrorpath}sha1sums.txt
chksumtype = sha1
mirrorgpgsig =
gpgkey =
gpgkeyserver =
[build]
dlpath = /var/tmp/${bdisk:uxname}
chrootdir = /var/tmp/chroots
basedir = /opt/dev/bdisk
@ -54,6 +65,7 @@ We'll go into more detail for each section below.
archboot = ${prepdir}/${bdisk:name}
mountpt = /mnt/${bdisk:uxname}
multiarch = yes
sign = yes
ipxe = no
i_am_a_racecar = no
[gpg]
@ -71,7 +83,7 @@ We'll go into more detail for each section below.
[tftp]
path = ${build:dlpath}/tftpboot
user = root
group = root
group = root
[ipxe]
iso = no
uri = https://domain.tld
@ -81,9 +93,9 @@ We'll go into more detail for each section below.
ssl_crt = ${ssldir}/main.crt
ssl_key = ${ssldir}/main.key
[rsync]
host =
user =
path =
host =
user =
path =
iso = no

=== `[bdisk]`
@ -184,15 +196,23 @@ The escaped, salted, hashed string to use for the non-root user.

Please see <<passwords,the section on passwords>> for information on this value. In the <<example,example above>>, the string `$$6$$t92Uvm1ETLocDb1D$$BvI0Sa6CSXxzIKBinIaJHb1gLJWheoXp7WzdideAJN46aChFu3hKg07QaIJNk4dfIJ2ry3tEfo3FRvstKWasg/` is created from the password `test`. I cannot stress this enough, do not use a plaintext password here nor just use a regular `/etc/shadow` file/`crypt(3)` hash here. Read the section. I promise it's short.

=== `[build]`
This section controls some aspects about the host and things like filesystem paths, etc.
=== `[source_<arch>]`
This section controls where to fetch the "base" tarballs.

NOTE: Previously, these settings were *not* architecture-specific, and included in the <<code_build_code,`build`>> section.

It was necessary to create this section per architecture, because https://www.archlinux.org/news/phasing-out-i686-support/[Arch Linux has dropped i686 support^]. However, plenty of other distros also have removed support and other third-party projects have ported. (You can find the Arch Linux 32-bit/i686 port project http://archlinux32.org/[here^].)

The directives here are only covered once, however, since both sections are identical- they just allow you to specify different mirrors. Note that the two settings are `[source_i686]` (for 32-bit) and `[source_x86_64]` (for 64-bit/multilib).

Which section is used (or both) depends on what <<code_multiarch_code, architectures you have enabled>> for the build.

==== `mirror`
A mirror that hosts the bootstrap tarball. It is *highly* recommended you use an Arch Linux https://wiki.archlinux.org/index.php/Install_from_existing_Linux#Method_A:_Using_the_bootstrap_image_.28recommended.29[bootstrap tarball^] as the build process is highly specialized to this (but <<bug_reports_feature_requests,patches/feature requests>> are welcome for other built distros). You can find a list of mirrors at the bottom of Arch's https://www.archlinux.org/download/[download page^].

. No whitespace
. Must be accessible remotely/via a WAN-recognized address
. Must be a domain/FQDN only; no paths (those come later!)
. Must be a domain/FQDN (or IP address) only; no paths (those come later!)

==== `mirrorproto`
What protocol should we use for the <<code_mirror_code,`mirror`>>?
@ -208,14 +228,42 @@ What is the path to the tarball directory on the <<code_mirror_code,`mirror`>>?
. No whitespace

==== `mirrorfile`
What is the filename for the tarball found in the path specified in <<code_mirrorpath_code,`mirrorpath`>> ? If left blank, we will use the sha1 <<code_mirrorchksum_code,checksum>> file to try to guess the most recent file.
What is the filename for the tarball found in the path specified in <<code_mirrorpath_code,`mirrorpath`>> ? If left blank, we will use the hash <<code_mirrorchksum_code,checksum>> file to try to guess the most recent file.

==== `mirrorchksum`
The path to a sha1 checksum file of the bootstrap tarball.
*[optional]* +
*default: (no hash checking done)* +
*requires: <<code_chksumtype_code,`chksumtype`>>*

The path to a checksum file of the bootstrap tarball.

. No whitespace
. Must be the full path
. Don't include the mirror domain or protocol
. Don't include the <<code_mirror_code,mirror domain>> or <<code_mirrorproto_code,protocol>>

==== `chksumtype`
The algorithm that <<code_mirrorchksum_code,`mirrorchksum`>>'s hashes are in.

[options="header"]
|======================
7+^|Accepts one of:
^m|blake2b
^m|blake2s
^m|md5
^m|sha1
^m|sha224
^m|sha256
^m|sha384
^m|sha512
^m|sha3_224
^m|sha3_256
^m|sha3_384
^m|sha3_512
^m|shake_128
^m|shake_256
|======================

TIP: You may have support for additional hashing algorithms, but these are the ones gauranteed to be supported by Python's https://docs.python.org/3/library/hashlib.html[hashlib module^]. To get a full list of algorithms the computer you're building on supports, you can run `python3 -c 'import hashlib;print(hashlib.algorithms_available)'`. Most likely, however, <<code_mirrorchksum_code,`mirrorchksum`>> is going to be hashes of one of the above.

==== `mirrorgpgsig`
*[optional]* +
@ -225,7 +273,7 @@ The path to a sha1 checksum file of the bootstrap tarball.

If the bootstrap tarball file has a GPG signature, we can use it for extra checking. If it's blank, GPG checking will be disabled.

If you specify just `.sig` (or use the default and don't specify a <<code_mirrorfile_code,`mirrorfile`>>), BDisk will try to guess based on the file from the sha1 <<code_mirrorchksum_code,checksum>> file. Note that this must evaluate to a full URL. (e.g. `${mirrorproto}://${mirror}${mirrorpath}somefile.sig`)
If you specify just `.sig` (or use the default and don't specify a <<code_mirrorfile_code,`mirrorfile`>>), BDisk will try to guess based on the file from the hash <<code_mirrorchksum_code,checksum>> file. Note that unless you're using the `.sig` "autodetection", this must evaluate to a full URL. (e.g. `${mirrorproto}://${mirror}${mirrorpath}somefile.sig`)

==== `gpgkey`
*requires: <<optional,_gpg/gnupg_>>*
@ -245,6 +293,18 @@ What is a valid keyserver we should use to fetch <<code_gpgkey_code,`gpgkey`>>?
. The default (blank) is probably fine. If you don't specify a personal GPG config, then you'll most likely want to leave this blank.
. If set, make sure it is a valid keyserver URI (e.g. `hkp://keys.gnupg.net`)

[options="header"]
|======================
2+^|Accepts (case-insensitive) one of:
^m|yes ^m|no
^m|true ^m|false
^m|1 ^m|0
|======================

=== `[build]`
This section controls some aspects about the host and things like filesystem paths, etc.


==== `gpg`
Should we sign our release files? See the <<code_gpg_code_2,`[gpg]`>> section.


View File

@ -28,15 +28,29 @@ username = ${bdisk:uxname}
name = Default user
password =

[build]
[source_x86_64]
mirror = mirror.us.leaseweb.net
mirrorproto = https
mirrorpath = /archlinux/iso/latest/
mirrorfile =
mirrorchksum = ${mirrorpath}sha1sums.txt
chksumtype = sha1
mirrorgpgsig =
gpgkey = 7F2D434B9741E8AC
gpgkeyserver =

[source_i686]
mirror = mirror.us.leaseweb.net
mirrorproto = https
mirrorpath = /archlinux/iso/latest/
mirrorfile =
mirrorchksum = ${mirrorpath}sha1sums.txt
chksumtype = sha1
mirrorgpgsig =
gpgkey = 7F2D434B9741E8AC
gpgkeyserver =

[build]
gpg = no
dlpath = /var/tmp/${bdisk:uxname}
chrootdir = /var/tmp/chroots
@ -47,6 +61,7 @@ prepdir = ${dlpath}/temp
archboot = ${prepdir}/${bdisk:name}
mountpt = /mnt/${bdisk:uxname}
multiarch = yes
sign = yes
ipxe =
i_am_a_racecar = yes


View File

@ -28,15 +28,29 @@ username = ${bdisk:uxname}
name = Default user
password =

[build]
[source_x86_64]
mirror = mirror.us.leaseweb.net
mirrorproto = https
mirrorpath = /archlinux/iso/latest/
mirrorfile =
mirrorfile =
mirrorchksum = ${mirrorpath}sha1sums.txt
chksumtype = sha1
mirrorgpgsig =
gpgkey = 7F2D434B9741E8AC
gpgkeyserver =

[source_i686]
mirror = mirror.us.leaseweb.net
mirrorproto = https
mirrorpath = /archlinux/iso/latest/
mirrorfile =
mirrorchksum = ${mirrorpath}sha1sums.txt
chksumtype = sha1
mirrorgpgsig =
gpgkey = 7F2D434B9741E8AC
gpgkeyserver =

[build]
gpg = no
dlpath = /var/tmp/${bdisk:uxname}
chrootdir = /var/tmp/chroots