diff --git a/bdisk/bGPG.py b/bdisk/bGPG.py index f0ee8e3..2759457 100755 --- a/bdisk/bGPG.py +++ b/bdisk/bGPG.py @@ -13,7 +13,11 @@ def genGPG(conf): bdisk = conf['bdisk'] gpghome = conf['gpg']['mygpghome'] distkey = build['gpgkey'] - gpgkeyserver = build['gpgkeyserver'] + gpgkeyserver = [] + for a in conf['build']['arch']: + keysrv = conf['src'][a]['gpgkeyserver'] + if keysrv and (keysrv not in gpgkeyserver): + gpgkeyserver.append(keysrv) templates_dir = '{0}/extra/templates'.format(build['basedir']) mykey = False pkeys = [] @@ -31,16 +35,17 @@ def genGPG(conf): os.environ['GNUPGHOME'] = gpghome gpg = gpgme.Context() # do we need to add a keyserver? - if gpgkeyserver != '': + if len(gpgkeyserver) != 0: dirmgr = '{0}/dirmngr.conf'.format(gpghome) - if os.path.isfile(dirmgr): - with open(dirmgr, 'r+') as f: - findme = any(gpgkeyserver in line for line in f) - if not findme: - f.seek(0, os.SEEK_END) - f.write("\n# Added by {0}.\nkeyserver {1}\n".format( - bdisk['pname'], - gpgkeyserver)) + for s in gpgkeyserver: + if os.path.isfile(dirmgr): + with open(dirmgr, 'r+') as f: + findme = any(s in line for line in f) + if not findme: + f.seek(0, os.SEEK_END) + f.write("\n# Added by {0}.\nkeyserver {1}\n".format( + bdisk['pname'], + s)) if mykey: try: privkey = gpg.get_key(mykey, True) @@ -62,15 +67,16 @@ def genGPG(conf): privkey = gpg.get_key(gpg.genkey(tpl_out).fpr, True) pkeys.append(privkey) # do we need to add a keyserver? this is for the freshly-generated GNUPGHOME - if build['gpgkeyserver'] != '': + if len(gpgkeyserver) != 0: dirmgr = '{0}/dirmngr.conf'.format(gpghome) - with open(dirmgr, 'r+') as f: - findme = any(gpgkeyserver in line for line in f) - if not findme: - f.seek(0, os.SEEK_END) - f.write("\n# Added by {0}.\nkeyserver {1}\n".format( - bdisk['pname'], - build['gpgkeyserver'])) + for s in gpgkeyserver: + with open(dirmgr, 'r+') as f: + findme = any(s in line for line in f) + if not findme: + f.seek(0, os.SEEK_END) + f.write("\n# Added by {0}.\nkeyserver {1}\n".format( + bdisk['pname'], + s)) gpg.signers = pkeys # Now we try to find and add the key for the base image. gpg.keylist_mode = gpgme.KEYLIST_MODE_EXTERN # remote (keyserver) @@ -125,7 +131,7 @@ def killStaleAgent(conf): psutil.Process(p).terminate() def signIMG(path, conf): - if conf['build']['gpg']: + if conf['build']['sign']: # Do we want to kill off any stale gpg-agents? (So we spawn a new one) # Requires further testing. #killStaleAgent() diff --git a/bdisk/host.py b/bdisk/host.py index 03ac578..39f7403 100755 --- a/bdisk/host.py +++ b/bdisk/host.py @@ -68,9 +68,9 @@ def parseConfig(confs): config_dict = {s:dict(config.items(s)) for s in config.sections()} # Convert the booleans to pythonic booleans in the dict... config_dict['bdisk']['user'] = config['bdisk'].getboolean('user') - config_dict['build']['gpg'] = config['build'].getboolean('gpg') config_dict['build']['i_am_a_racecar'] = config['build'].getboolean('i_am_a_racecar') config_dict['build']['ipxe'] = config['build'].getboolean('ipxe') + config_dict['build']['sign'] = config['build'].getboolean('sign') config_dict['build']['multiarch'] = (config_dict['build']['multiarch']).lower() config_dict['ipxe']['iso'] = config['ipxe'].getboolean('iso') config_dict['ipxe']['usb'] = config['ipxe'].getboolean('usb') @@ -126,16 +126,21 @@ def parseConfig(confs): config_dict['build']['multiarch'])) ## VALIDATORS ## # Validate bootstrap mirror - if (validators.domain(config_dict['build']['mirror']) or validators.ipv4( - config_dict['build']['mirror']) or validatords.ipv6( - config_dict['build']['mirror'])): - try: - getaddrinfo(config_dict['build']['mirror'], None) - except: - exit(('{0}: ERROR: {1} does not resolve and cannot be used as a ' + - 'mirror for the bootstrap tarballs. Check your configuration.').format( - datetime.datetime.now(), - config_dict['build']['host'])) + config_dict['src'] = {} + for a in config_dict['build']['arch']: + config_dict['src'][a] = config_dict['source_' + a] + if config_dict['src'][a]['enabled']: + if (validators.domain(config_dict['src'][a]['mirror']) or validators.ipv4( + config_dict['src'][a]['mirror']) or validatords.ipv6( + config_dict['src'][a]['mirror'])): + try: + getaddrinfo(config_dict['src'][a]['mirror'], None) + except: + exit(('{0}: ERROR: {1} does not resolve and cannot be used as a ' + + 'mirror for the bootstrap tarballs. Check your configuration.').format( + datetime.datetime.now(), + config_dict['src'][a]['host'])) + config_dict['src'][a]['gpg'] = config['source_' + a].getboolean('gpg') # Are we rsyncing? If so, validate the rsync host. # Works for IP address too. It does NOT check to see if we can # actually *rsync* to it; that'll come later. diff --git a/bdisk/prep.py b/bdisk/prep.py index b5a0e25..4551cc8 100755 --- a/bdisk/prep.py +++ b/bdisk/prep.py @@ -25,27 +25,36 @@ def dirChk(conf): def downloadTarball(conf): build = conf['build'] dlpath = build['dlpath'] + src = conf['src'] arch = build['arch'] - #mirror = 'http://mirrors.kernel.org/archlinux' - mirror = build['mirrorproto'] + '://' + build['mirror'] - rlsdir = mirror + build['mirrorpath'] - sha_in = urlopen(mirror + build['mirrorchksum']) - # returns path/filename e.g. /some/path/to/file.tar.gz - # we use .gnupg since we'll need it later. - os.makedirs(dlpath + '/.gnupg', exist_ok = True) tarball_path = {} - for x in arch: - tarball_path[x] = dlpath + '/.latest.' + x + '.tar' - sha1sums = sha_in.read() - sha_in.close() - sha_raw = sha1sums.decode("utf-8") - sha_list = list(filter(None, sha_raw.split('\n'))) - sha_dict = {x.split()[1]: x.split()[0] for x in sha_list} - # all that lousy work just to get a sha1 sum. okay. so. for a in arch: + locsrc = conf['source_' + a] + mirror = locsrc['mirrorproto'] + '://' + locsrc['mirror'] + rlsdir = mirror + locsrc['mirrorpath'] + if locsrc['mirrorchksum'] != '': + if not locsrc['chksumtype']: + exit(("{0}: source_{1}:chksumtype is unset!".format( + datetime.datetime.now(), + a)) + hash_type = locsrc['chksumtype'] + hash_in = urlopen(mirror + locsrc['mirrorchksum']) + hashsums = hash_in.read() + hash_in.close() + hash_raw = hashsums.decode("utf-8") + hash_list = list(filter(None, hash_raw.split('\n'))) + hash_dict = {x.split()[1]: x.split()[0] for x in hash_list} + # returns path/filename e.g. /some/path/to/file.tar.gz + # we use .gnupg since we'll need it later. + os.makedirs(dlpath + '/.gnupg', exist_ok = True) + tarball_path[a] = dlpath + '/.latest.' + a + '.tar' pattern = re.compile('^.*' + a + '\.tar(\.(gz|bz2|xz))?$') - tarball = [filename.group(0) for l in list(sha_dict.keys()) for filename in [pattern.search(l)] if filename][0] - sha1 = sha_dict[tarball] + if locsrc['mirrorfile'] != '': + tarball = locsrc['mirrorfile'] + else: + tarball = [filename.group(0) for l in list(hash_dict.keys()) for filename in [pattern.search(l)] if filename][0] + if locsrc['mirrorchksum'] != '': + hashsum = hash_dict[tarball] if os.path.isfile(tarball_path[a]): pass else: @@ -53,7 +62,6 @@ def downloadTarball(conf): print("{0}: [PREP] Fetching tarball ({1} architecture)...".format( datetime.datetime.now(), a)) - #dl_file = urllib.URLopener() tarball_dl = urlopen(rlsdir + tarball) with open(tarball_path[a], 'wb') as f: f.write(tarball_dl.read()) @@ -63,20 +71,32 @@ def downloadTarball(conf): tarball_path[a], humanize.naturalsize( os.path.getsize(tarball_path[a])))) - print("{0}: [PREP] Checking hash checksum {1} against {2}...".format( - datetime.datetime.now(), - sha1, - tarball_path[a])) - tarball_hash = hashlib.sha1(open(tarball_path[a], 'rb').read()).hexdigest() - if tarball_hash != sha1: - exit(("{0}: {1} either did not download correctly\n\t\t\t or a wrong (probably old) version exists on the filesystem.\n\t\t\t " + - "Please delete it and try again.").format(datetime.datetime.now(), tarball)) - elif build['mirrorgpgsig'] != '': - # okay, so the sha1 matches. let's verify the signature. - if build['mirrorgpgsig'] == '.sig': + if locsrc['mirrorchksum'] != '': + print("{0}: [PREP] Checking hash checksum {1} against {2}...".format( + datetime.datetime.now(), + hashsum, + tarball_path[a])) + # Calculate the checksum according to type specified. + tarball_hash = False + for i in hashlib.algorithms_available: + if hash_type == i: + hashfunc = getattr(hashlib, i) + tarball_hash = hashfunc(open(tarball_path[a], 'rb').read()).hexdigest() + break + if not tarball_hash: + exit(("{0}: source_{1}:chksumtype '{2}' is not supported on this machine!".format( + datetime.datetime.now(), + a, + hash_type)) + if tarball_hash != hashsum: + exit(("{0}: {1} either did not download correctly\n\t\t\t or a wrong (probably old) version exists on the filesystem.\n\t\t\t " + + "Please delete it and try again.").format(datetime.datetime.now(), tarball)) + if locsrc['mirrorgpgsig'] != '': + # let's verify the signature. + if locsrc['mirrorgpgsig'] == '.sig': gpgsig_remote = rlsdir + tarball + '.sig' else: - gpgsig_remote = build['mirrorgpgsig'] + gpgsig_remote = locsrc['mirrorgpgsig'] sig_dl = urlopen(gpgsig_remote) sig = tarball_path[a] + '.sig' with open(sig, 'wb+') as f: diff --git a/docs/TODO b/docs/TODO index 5931a84..bcd4bf5 100644 --- a/docs/TODO +++ b/docs/TODO @@ -9,15 +9,15 @@ -- https://code.google.com/p/byte-unixbench/ -- https://github.com/akopytov/sysbench -- (http://blog.due.io/2014/linode-digitalocean-and-vultr-comparison/ etc.) --implement pyalpm to decreate dependency on chroot pacman-ing? +-There *has* to be a better way of handling package installation in the chroots. +--implement pyalpm to decreate dependency on chroot pacman-ing? --or even maybe https://wiki.archlinux.org/index.php/offline_installation_of_packages in pure python! -set up automatic exporting to PDF of the user manual server-side. https://pypi.python.org/pypi/unoconv/0.6 --There *has* to be a better way of handling package installation in the chroots. -maybe remove lxde, firefox, chrome and replace with enlightenment/midori? -custom repo? https://brainwreckedtech.wordpress.com/2013/01/27/making-your-own-arch-linux-repository/ ---https://wiki.archlinux.org/index.php/Building_32-bit_packages_on_a_64-bit_system +--https://wiki.archlinux.org/index.php/Building_32-bit_packages_on_a_64-bit_system # NOTE: arch has dropped i686, now continued as archlinux32 -implement better "additional" packages list. specify for path in build.ini- these should be more easily changed by end users. DON'T TOUCH iso.pkgs.lst since those are necessary for booting. --shorewall/some other firewall? +-automatic shorewall/some other firewall? -autodetection/configuration of network. DHCP is currently running by default, but does it need to support IPv6? if so, how would the user configure their network? -DISABLE NETWORKMANAGER AND "fi.w1.wpa_supplicant1"??? keeps spawning wpa_supplicant (and thusly killing networking proper) -for netboot, custom user agent (should be defined by build.ini) @@ -27,7 +27,7 @@ -WISH: signing for secureboot releases (PreLoader and loader.efi handle this okay, but require manual intervention) -does loader.efi support splash backgrounds? can i implement that differently somehow? --yes, see e.g. https://www.reddit.com/r/archlinux/comments/3bwgf0/where_put_the_splasharchbmp_to_splash_screen_boot/ --strip out/remove unnecessary and orphan packages (e.g. gcc, make, automake, etc.) +-strip out/remove unnecessary and orphan packages (e.g. gcc, make, automake, etc.) before building ISO -incorporate iPXE tweaks: --http://ipxe.org/crypto --http://ipxe.org/cmd/imgtrust @@ -39,6 +39,7 @@ ---#imgverify initrd path/to/initrd.sig ---DONE, partially. need to incorporate codesign certs/keys. routines, conf variables -enable mirror= kernel commandline. +-NOTE: Following should be implemented via AIF-NG (https://git.square-r00t.net/AIF-NG, work pending for fix to BDisk for i686/x86_64 split) --if mirror_(NAME) is present, use that as repo name. --if it starts with /, treat as mirrorlist (Include); otherwise use Server = --if it has mirror_SIG-X, set signature options e.g. _SIG-N would be "SigLevel = Never" @@ -46,6 +47,4 @@ --note that iPXE VESAFB console is not (yet) supported in EFI, so this is on hold. check into this to see if it has changed. -include WinMTR, build Mac OS X MTR for dist/tools on CD -include pre-compiled LibreCrypt for opening LUKS parts on Windows (https://github.com/t-d-k/LibreCrypt) ---curl -s https://raw.githubusercontent.com/t-d-k/LibreCrypt/master/README.md | egrep 'InstallLibreCrypt_v[A-Za-z0-9\.]*.exe' | cut -f2 -d'"' - - +--curl -s https://raw.githubusercontent.com/t-d-k/LibreCrypt/master/README.md | egrep 'InstallLibreCrypt_v[A-Za-z0-9\.]*.exe' | cut -f2 -d'"' \ No newline at end of file diff --git a/docs/manual/HEAD.adoc b/docs/manual/HEAD.adoc index afa95fe..c3b86f1 100644 --- a/docs/manual/HEAD.adoc +++ b/docs/manual/HEAD.adoc @@ -1,6 +1,6 @@ = BDisk User and Developer Manual Brent Saner -v1.0, 2016-12 +v1.1, 2017-03-06 :doctype: book :data-uri: :imagesdir: images diff --git a/docs/manual/user/BUILDINI.adoc b/docs/manual/user/BUILDINI.adoc index 4e51ad7..ed63ceb 100644 --- a/docs/manual/user/BUILDINI.adoc +++ b/docs/manual/user/BUILDINI.adoc @@ -24,7 +24,7 @@ We'll go into more detail for each section below. name = BDISK uxname = bdisk pname = BDisk - ver = + ver = dev = A Developer email = dev@domain.tld desc = A rescue/restore live environment. @@ -35,16 +35,27 @@ We'll go into more detail for each section below. username = ${bdisk:uxname} name = Default user password = $$6$$t92Uvm1ETLocDb1D$$BvI0Sa6CSXxzIKBinIaJHb1gLJWheoXp7WzdideAJN46aChFu3hKg07QaIJNk4dfIJ2ry3tEfo3FRvstKWasg/ - [build] + [source_x86_64] mirror = mirror.us.leaseweb.net mirrorproto = https mirrorpath = /archlinux/iso/latest/ - mirrorfile = + mirrorfile = .sig mirrorchksum = ${mirrorpath}sha1sums.txt + chksumtype = sha1 mirrorgpgsig = gpgkey = 7F2D434B9741E8AC gpgkeyserver = - gpg = no + [source_i686] + mirror = mirror.us.leaseweb.net + mirrorproto = https + mirrorpath = /archlinux/iso/latest/ + mirrorfile = + mirrorchksum = ${mirrorpath}sha1sums.txt + chksumtype = sha1 + mirrorgpgsig = + gpgkey = + gpgkeyserver = + [build] dlpath = /var/tmp/${bdisk:uxname} chrootdir = /var/tmp/chroots basedir = /opt/dev/bdisk @@ -54,6 +65,7 @@ We'll go into more detail for each section below. archboot = ${prepdir}/${bdisk:name} mountpt = /mnt/${bdisk:uxname} multiarch = yes + sign = yes ipxe = no i_am_a_racecar = no [gpg] @@ -71,7 +83,7 @@ We'll go into more detail for each section below. [tftp] path = ${build:dlpath}/tftpboot user = root - group = root + group = root [ipxe] iso = no uri = https://domain.tld @@ -81,9 +93,9 @@ We'll go into more detail for each section below. ssl_crt = ${ssldir}/main.crt ssl_key = ${ssldir}/main.key [rsync] - host = - user = - path = + host = + user = + path = iso = no === `[bdisk]` @@ -184,15 +196,23 @@ The escaped, salted, hashed string to use for the non-root user. Please see <> for information on this value. In the <>, the string `$$6$$t92Uvm1ETLocDb1D$$BvI0Sa6CSXxzIKBinIaJHb1gLJWheoXp7WzdideAJN46aChFu3hKg07QaIJNk4dfIJ2ry3tEfo3FRvstKWasg/` is created from the password `test`. I cannot stress this enough, do not use a plaintext password here nor just use a regular `/etc/shadow` file/`crypt(3)` hash here. Read the section. I promise it's short. -=== `[build]` -This section controls some aspects about the host and things like filesystem paths, etc. +=== `[source_]` +This section controls where to fetch the "base" tarballs. + +NOTE: Previously, these settings were *not* architecture-specific, and included in the <> section. + +It was necessary to create this section per architecture, because https://www.archlinux.org/news/phasing-out-i686-support/[Arch Linux has dropped i686 support^]. However, plenty of other distros also have removed support and other third-party projects have ported. (You can find the Arch Linux 32-bit/i686 port project http://archlinux32.org/[here^].) + +The directives here are only covered once, however, since both sections are identical- they just allow you to specify different mirrors. Note that the two settings are `[source_i686]` (for 32-bit) and `[source_x86_64]` (for 64-bit/multilib). + +Which section is used (or both) depends on what <> for the build. ==== `mirror` A mirror that hosts the bootstrap tarball. It is *highly* recommended you use an Arch Linux https://wiki.archlinux.org/index.php/Install_from_existing_Linux#Method_A:_Using_the_bootstrap_image_.28recommended.29[bootstrap tarball^] as the build process is highly specialized to this (but <> are welcome for other built distros). You can find a list of mirrors at the bottom of Arch's https://www.archlinux.org/download/[download page^]. . No whitespace . Must be accessible remotely/via a WAN-recognized address -. Must be a domain/FQDN only; no paths (those come later!) +. Must be a domain/FQDN (or IP address) only; no paths (those come later!) ==== `mirrorproto` What protocol should we use for the <>? @@ -208,14 +228,42 @@ What is the path to the tarball directory on the <>? . No whitespace ==== `mirrorfile` -What is the filename for the tarball found in the path specified in <> ? If left blank, we will use the sha1 <> file to try to guess the most recent file. +What is the filename for the tarball found in the path specified in <> ? If left blank, we will use the hash <> file to try to guess the most recent file. ==== `mirrorchksum` -The path to a sha1 checksum file of the bootstrap tarball. +*[optional]* + +*default: (no hash checking done)* + +*requires: <>* + +The path to a checksum file of the bootstrap tarball. . No whitespace . Must be the full path -. Don't include the mirror domain or protocol +. Don't include the <> or <> + +==== `chksumtype` +The algorithm that <>'s hashes are in. + +[options="header"] +|====================== +7+^|Accepts one of: +^m|blake2b +^m|blake2s +^m|md5 +^m|sha1 +^m|sha224 +^m|sha256 +^m|sha384 +^m|sha512 +^m|sha3_224 +^m|sha3_256 +^m|sha3_384 +^m|sha3_512 +^m|shake_128 +^m|shake_256 +|====================== + +TIP: You may have support for additional hashing algorithms, but these are the ones gauranteed to be supported by Python's https://docs.python.org/3/library/hashlib.html[hashlib module^]. To get a full list of algorithms the computer you're building on supports, you can run `python3 -c 'import hashlib;print(hashlib.algorithms_available)'`. Most likely, however, <> is going to be hashes of one of the above. ==== `mirrorgpgsig` *[optional]* + @@ -225,7 +273,7 @@ The path to a sha1 checksum file of the bootstrap tarball. If the bootstrap tarball file has a GPG signature, we can use it for extra checking. If it's blank, GPG checking will be disabled. -If you specify just `.sig` (or use the default and don't specify a <>), BDisk will try to guess based on the file from the sha1 <> file. Note that this must evaluate to a full URL. (e.g. `${mirrorproto}://${mirror}${mirrorpath}somefile.sig`) +If you specify just `.sig` (or use the default and don't specify a <>), BDisk will try to guess based on the file from the hash <> file. Note that unless you're using the `.sig` "autodetection", this must evaluate to a full URL. (e.g. `${mirrorproto}://${mirror}${mirrorpath}somefile.sig`) ==== `gpgkey` *requires: <>* @@ -245,6 +293,18 @@ What is a valid keyserver we should use to fetch <>? . The default (blank) is probably fine. If you don't specify a personal GPG config, then you'll most likely want to leave this blank. . If set, make sure it is a valid keyserver URI (e.g. `hkp://keys.gnupg.net`) +[options="header"] +|====================== +2+^|Accepts (case-insensitive) one of: +^m|yes ^m|no +^m|true ^m|false +^m|1 ^m|0 +|====================== + +=== `[build]` +This section controls some aspects about the host and things like filesystem paths, etc. + + ==== `gpg` Should we sign our release files? See the <> section. diff --git a/extra/dist.build.ini b/extra/dist.build.ini index ea4bb98..15f5fa1 100644 --- a/extra/dist.build.ini +++ b/extra/dist.build.ini @@ -28,15 +28,29 @@ username = ${bdisk:uxname} name = Default user password = -[build] +[source_x86_64] mirror = mirror.us.leaseweb.net mirrorproto = https mirrorpath = /archlinux/iso/latest/ mirrorfile = mirrorchksum = ${mirrorpath}sha1sums.txt +chksumtype = sha1 mirrorgpgsig = gpgkey = 7F2D434B9741E8AC gpgkeyserver = + +[source_i686] +mirror = mirror.us.leaseweb.net +mirrorproto = https +mirrorpath = /archlinux/iso/latest/ +mirrorfile = +mirrorchksum = ${mirrorpath}sha1sums.txt +chksumtype = sha1 +mirrorgpgsig = +gpgkey = 7F2D434B9741E8AC +gpgkeyserver = + +[build] gpg = no dlpath = /var/tmp/${bdisk:uxname} chrootdir = /var/tmp/chroots @@ -47,6 +61,7 @@ prepdir = ${dlpath}/temp archboot = ${prepdir}/${bdisk:name} mountpt = /mnt/${bdisk:uxname} multiarch = yes +sign = yes ipxe = i_am_a_racecar = yes diff --git a/extra/pkg.build.ini b/extra/pkg.build.ini index 2b396f4..5a14829 100644 --- a/extra/pkg.build.ini +++ b/extra/pkg.build.ini @@ -28,15 +28,29 @@ username = ${bdisk:uxname} name = Default user password = -[build] +[source_x86_64] mirror = mirror.us.leaseweb.net mirrorproto = https mirrorpath = /archlinux/iso/latest/ -mirrorfile = +mirrorfile = mirrorchksum = ${mirrorpath}sha1sums.txt +chksumtype = sha1 mirrorgpgsig = gpgkey = 7F2D434B9741E8AC gpgkeyserver = + +[source_i686] +mirror = mirror.us.leaseweb.net +mirrorproto = https +mirrorpath = /archlinux/iso/latest/ +mirrorfile = +mirrorchksum = ${mirrorpath}sha1sums.txt +chksumtype = sha1 +mirrorgpgsig = +gpgkey = 7F2D434B9741E8AC +gpgkeyserver = + +[build] gpg = no dlpath = /var/tmp/${bdisk:uxname} chrootdir = /var/tmp/chroots