diff --git a/bdisk/bdisk.py b/bdisk/bdisk.py index a8427ed..6765876 100755 --- a/bdisk/bdisk.py +++ b/bdisk/bdisk.py @@ -4,6 +4,8 @@ import prep import bchroot import build import datetime +import bSSL +import ipxe # we need to: # we also need to figure out how to implement "mentos" (old bdisk) like functionality, letting us reuse an existing chroot install if possible to save time for future builds. @@ -27,4 +29,6 @@ if __name__ == '__main__': build.displayStats(fulliso) if conf['build']['ipxe']: bSSL.sslPKI(conf) + iso = ipxe.buildIPXE(conf) + build.displayStats(iso) print('{0}: Finish.'.format(datetime.datetime.now())) diff --git a/bdisk/build.py b/bdisk/build.py index 1324ced..85b2df4 100755 --- a/bdisk/build.py +++ b/bdisk/build.py @@ -160,7 +160,7 @@ def genUEFI(build, bdisk): # This is more important than it looks. #sizetotal = 33553920 # The spec'd EFI binary size (32MB). It's okay to go over this though (and we do) # because xorriso sees it as a filesystem image and adjusts the ISO automagically. - sizetotal = 786432 # we start with 768KB and add to it for wiggle room + sizetotal = 2097152 # we start with 2MB and add to it for wiggle room sizefiles = ['/boot/' + bdisk['uxname'] + '.64.img', '/boot/' + bdisk['uxname'] + '.64.kern', '/EFI/boot/bootx64.efi', @@ -366,6 +366,7 @@ def genISO(conf): # Get size of ISO iso = {} iso['name'] = ['Main'] + iso['Main'] = {} iso['Main']['sha'] = hashlib.sha256() with open(isopath, 'rb') as f: while True: @@ -373,7 +374,7 @@ def genISO(conf): if not stream: break iso['Main']['sha'].update(stream) - iso['Main']['sha'] = iso['sha'].hexdigest() + iso['Main']['sha'] = iso['Main']['sha'].hexdigest() iso['Main']['file'] = isopath iso['Main']['size'] = humanize.naturalsize(os.path.getsize(isopath)) iso['Main']['type'] = 'Full' @@ -382,10 +383,10 @@ def genISO(conf): def displayStats(iso): for i in iso['name']: - print("{0}:\n== {1} {2} ==".format(datetime.datetime.now(), iso[i]['type'], iso[i]['fmt'])) - print('Size: {0}'.format(iso[i]['size'])) - print('SHA256: {0}'.format(iso[i]['sha'])) - print('Location: {0}\n'.format(iso[i]['file'])) + print("{0}: == {1} {2} ==".format(datetime.datetime.now(), iso[i]['type'], iso[i]['fmt'])) + print('\t\t\t = Size: {0}'.format(iso[i]['size'])) + print('\t\t\t = SHA256: {0}'.format(iso[i]['sha'])) + print('\t\t\t = Location: {0}\n'.format(iso[i]['file'])) def cleanUp(): # TODO: clear out all of tempdir? diff --git a/bdisk/ipxe.py b/bdisk/ipxe.py index 2bd1097..40d97af 100755 --- a/bdisk/ipxe.py +++ b/bdisk/ipxe.py @@ -6,54 +6,32 @@ import jinja2 import git import patch import datetime -import bSSL +import humanize +import hashlib -def sslIPXE(conf): - try: - ca = conf['ipxe']['ssl_ca'] - except: - ca = None - try: - cakey = conf['ipxe']['ssl_cakey'] - except: - cakey = None - try: - crt = conf['ipxe']['ssl_crt'] - except: - crt = None - try: - key = conf['ipxe']['ssl_key'] - except: - key = None - # http://www.pyopenssl.org/en/stable/api/crypto.html#pkey-objects - # http://docs.ganeti.org/ganeti/2.14/html/design-x509-ca.html - if not cakey: - cakey = bSSL.sslCAKey - - pass - def buildIPXE(conf): build = conf['build'] bdisk = conf['bdisk'] ipxe = conf['ipxe'] + mini = ipxe['iso'] + usb = ipxe['usb'] tempdir = conf['build']['tempdir'] templates_dir = build['basedir'] + '/extra/templates' ipxe_tpl = templates_dir + '/iPXE' patches_dir = tempdir + '/patches' srcdir = build['srcdir'] - embedscript = tempdir + '/EMBED' + embedscript = build['dlpath'] + '/EMBED' ipxe_src = srcdir + '/ipxe' img_path = build['isodir'] + '/' ipxe_usb = '{0}-{1}.usb.img'.format(bdisk['uxname'], bdisk['ver']) ipxe_mini = '{0}-{1}.mini.iso'.format(bdisk['uxname'], bdisk['ver']) usb_file = '{0}/{1}'.format(img_path, ipxe_usb) - mini_file = '{0}/{1}'.format(img_path, ipxe_mini) + mini_file = '{0}{1}'.format(img_path, ipxe_mini) ipxe_git_uri = 'git://git.ipxe.org/ipxe.git' patches_git_uri = 'https://github.com/eworm-de/ipxe.git' - print('{0}: Building iPXE in {1}. Please wait...'.format( - datetime.datetime.now(), - ipxe_src)) + print('{0}: Preparing and fetching sources for iPXE. Please wait...'.format( + datetime.datetime.now())) # Get the source and apply some cherrypicks if os.path.isdir(ipxe_src): shutil.rmtree(ipxe_src) @@ -88,72 +66,91 @@ def buildIPXE(conf): # Feature enabling # In config/general.h with open('{0}/src/config/general.h'.format(ipxe_src), 'r') as f: - generalconf = f.readlines() + generalconf = f.read() # And in config/console.h with open('{0}/src/config/console.h'.format(ipxe_src), 'r') as f: - consoleconf = f.readlines() - gendict = {'^#undef(\s*NET_PROTO_IPV6)':'#define\g<1>', # enable IPv6 - '^#undef(\s*DOWNLOAD_PROTO_HTTPS)':'#define\g<1>', # enable HTTPS - '^//(#define\s*IMAGE_TRUST_CMD)':'\g<1>', # moar HTTPS - '^#undef(\s*DOWNLOAD_PROTO_FTP)':'#define\g<1>'} # FTP - #'^//(#define\s*CONSOLE_CMD)':'\g<1>', # BROKEN in EFI? TODO. if enable, replace } with , above etc. - #'^//(#define\s*IMAGE_PNG':'\g<1>'} # SAME, broken in EFI? TODO. - consdict = {}#'^//(#define\s*CONSOLE_VESAFB)':'\g<1>'} # BROKEN in EFI? TODO. + consoleconf = f.read() + patterns = (('^#undef(\s*NET_PROTO_IPV6.*)$','#define\g<1>'), # enable IPv6 + ('^#undef(\s*DOWNLOAD_PROTO_HTTPS)','#define\g<1>'), # enable HTTPS + ('^//(#define\s*IMAGE_TRUST_CMD)','\g<1>'), # moar HTTPS + ('^#undef(\s*DOWNLOAD_PROTO_FTP)','#define\g<1>')) # enable FTP + #('^//(#define\s*CONSOLE_CMD)','\g<1>'), # BROKEN in EFI? TODO. if enable, replace } with , above etc. + #('^//(#define\s*IMAGE_PNG','\g<1>'), # SAME, broken in EFI? TODO. + #console = ('^//(#define\s*CONSOLE_VESAFB)','\g<1>') # BROKEN in EFI? TODO. # https://stackoverflow.com/a/4427835 # https://emilics.com/notebook/enblog/p869.html - sedlike = re.compile('|'.join(gendict.keys())) + # The above methods don't seem to work. it craps out on the pattern matchings + # so we use tuples instead. + for x in patterns: + generalconf = re.sub(x[0], x[1], generalconf, flags=re.MULTILINE) with open('{0}/src/config/general.h'.format(ipxe_src), 'w') as f: - f.write(re.sub(lambda m: gendict[m.group(0)], generalconf)) + f.write(generalconf) # Uncomment when we want to test the above consdict etc. - #sedlike = re.compile('|'.join(consdict.keys())) + #for x in patterns: + # generalconf = re.sub(x[0], x[1], generalconf, flags=re.MULTILINE) #with open('{0}/src/config/console.h'.format(ipxe_src), 'w') as f: - # f.write(re.sub(lambda m: consdict[m.group(0)], consoleconf)) + # f.write(console) # Now we make! cwd = os.getcwd() os.chdir(ipxe_src + '/src') # TODO: split this into logic to only create the selected images. # Command to build the .efi file - build_efi = ['/usr/bin/make', + modenv = os.environ.copy() + modenv['EMBED'] = embedscript + #modenv['TRUST'] = ipxe_ssl_ca # TODO: test these + #modenv['CERT'] = '{0},{1}'.format(ipxe_ssl_ca, ipxe_ssl_crt) # TODO: test these + #modenv['PRIVKEY'] = ipxe_ssl_ckey # TODO: test these + build_cmd = {} + build_cmd['efi'] = ['/usr/bin/make', 'bin-i386-efi/ipxe.efi', - 'bin-x86_64-efi/ipxe.efi', - 'EMBED="{0}"'.format(embedscript) - #'TRUST="{0}"'.format(ipxe_ssl_ca), # finish work on ipxe SSL. make sure you throw a comma at the end above. - #'CERT="{0},{1}"'.format(ipxe_ssl_ca, ipxe_ssl_crt), # finish work on ipxe SSL - #'PRIVKEY="{0}"'.format(ipxe_ssl_ckey) - ] + 'bin-x86_64-efi/ipxe.efi'] # Command to build the actual USB and Mini images - build_boots = ['/usr/bin/make', - 'bin/ipxe.eiso', - 'bin/ipxe.usb', - 'EMBED="{0}"'.format(embedscript) - #'TRUST="{0}"'.format(ipxe_ssl_ca), # finish work on ipxe SSL. make sure you throw a comma at the end above. - #'CERT="{0},{1}"'.format(ipxe_ssl_ca, ipxe_ssl_crt), # finish work on ipxe SSL - #'PRIVKEY="{0}"'.format(ipxe_ssl_ckey) - ] + build_cmd['img'] = ['/usr/bin/make'] # Now we call the commands. - for n in ('build_efi', 'build_boots'): - subprocess.call(n, stdout = DEVNULL, stderr = subprocess.STDOUT) # TODO: log the make output to a file? + DEVNULL = open(os.devnull, 'w') + if os.path.isfile(build['dlpath'] + '/ipxe.log'): + os.remove(build['dlpath'] + '/ipxe.log') + print(('{0}: Building iPXE in {1}. Please wait...\n\t\t\t You can view progress' + + ' via:\n\t\t\t tail -f {2}/ipxe.log').format( + datetime.datetime.now(), + ipxe_src, + build['dlpath'])) + if mini and not usb: + build_cmd['img'].insert(1, 'bin/ipxe.eiso') + elif usb and not mini: + build_cmd['img'].insert(1, 'bin/ipxe.usb') + elif usb and mini: + build_cmd['img'].insert(1, 'bin/ipxe.eiso') + build_cmd['img'].insert(2, 'bin/ipxe.usb') + with open('{0}/ipxe.log'.format(build['dlpath']), 'a') as f: + subprocess.call(build_cmd['efi'], stdout = f, stderr = subprocess.STDOUT, env=modenv) + subprocess.call(build_cmd['img'], stdout = f, stderr = subprocess.STDOUT, env=modenv) + print('{0}: Built iPXE image(s) successfully.'.format(datetime.datetime.now())) os.chdir(cwd) # move the files to the results dir os.rename('{0}/src/bin/ipxe.usb'.format(ipxe_src), usb_file) os.rename('{0}/src/bin/ipxe.eiso'.format(ipxe_src), mini_file) # Get size etc. of build results iso = {} + stream = {} iso['name'] = [] for t in ('USB', 'Mini'): # TODO: do this programmatically based on config iso['name'].append(t) - iso[t]['sha'] = hashlib.sha256() + iso[t] = {} + shasum = False + shasum = hashlib.sha256() if t == 'USB': isopath = usb_file elif t == 'Mini': isopath = mini_file + stream = False with open(isopath, 'rb') as f: while True: stream = f.read(65536) # 64kb chunks if not stream: break - iso[t]['sha'].update(stream) - iso[t]['sha'] = iso['sha'].hexdigest() + shasum.update(stream) + iso[t]['sha'] = shasum.hexdigest() iso[t]['file'] = isopath iso[t]['size'] = humanize.naturalsize(os.path.getsize(isopath)) iso[t]['type'] = 'iPXE {0}'.format(t) diff --git a/bdisk/prep.py b/bdisk/prep.py index 4217504..ad0fdd3 100755 --- a/bdisk/prep.py +++ b/bdisk/prep.py @@ -49,7 +49,7 @@ def downloadTarball(build): # python-gnupg 0.3.9 spits this error in Arch. it's harmless, but ugly af. # TODO: remove this when the error doesn't happen anymore. print("\t\t\t If you see a \"ValueError: Unknown status message: 'KEY_CONSIDERED'\" error,\n\t\t\t it can be safely ignored.") - print("\t\t\t If this is taking a VERY LONG time, try installing haveged and starting it.\n\t\t\t This can be" + + print("\t\t\t If this is taking a VERY LONG time, try installing haveged and starting it.\n\t\t\t This can be " + "done safely in parallel with the build process.\n") input_data = gpg.gen_key_input(name_email = 'tempuser@nodomain.tld', passphrase = 'placeholder_passphrase') key = gpg.gen_key(input_data) diff --git a/docs/TODO b/docs/TODO index 7847196..92c0bd9 100644 --- a/docs/TODO +++ b/docs/TODO @@ -15,6 +15,8 @@ -implement pyalpm to decreate dependency on chroot pacman-ing? --or even maybe https://wiki.archlinux.org/index.php/offline_installation_of_packages in pure python! -set up automatic exporting to PDF of the user manual server-side. https://pypi.python.org/pypi/unoconv/0.6 +-There *has* to be a better way of handling package installation in the chroots. +-maybe remove lxde, firefox, chrome and replace with enlightenment/midori? ## NETWORKING ## diff --git a/extra/dist.build.ini b/extra/dist.build.ini index 7ec0786..47e2cc6 100644 --- a/extra/dist.build.ini +++ b/extra/dist.build.ini @@ -220,7 +220,7 @@ isodir = ${dlpath}/iso ; source code we download (if we need it). ; 0.) No whitespace ; 1.) Will be created if it doesn't exist, and is needed -srcdir = ${basedir}/extrasrc +srcdir = ${dlpath}/src ; What directory should we use for staging? ; 0.) No whitespace diff --git a/extra/pre-build.d/root/pre-build.sh b/extra/pre-build.d/root/pre-build.sh index 20d97c3..d6d4155 100755 --- a/extra/pre-build.d/root/pre-build.sh +++ b/extra/pre-build.d/root/pre-build.sh @@ -44,7 +44,7 @@ pacman -Syy # Just in case. cleanPacorigs # Install some prereqs -pacman -S sed +pacman -S --noconfirm --needed sed pacman -S --noconfirm --needed base syslinux wget rsync unzip jshon sudo abs xmlto bc docbook-xsl git locale-gen # And get rid of files it wants to replace @@ -69,7 +69,7 @@ cleanPacorigs # Install multilib-devel if we're in an x86_64 chroot. if $(egrep -q '^\[multilib' /etc/pacman.conf); then - pacman --noconfirm -Rdd gcc gcc-libs libtool + yes 'y' | pacman -S --needed gcc-multilib lib32-fakeroot lib32-libltdl pacman --noconfirm -S --needed multilib-devel cleanPacorigs TGT_ARCH='x86_64'