ipxe building, ssl pki builds. untested with ssl-integrated iPXE though.

bdisk/bdisk.py

@@ -4,6 +4,8 @@ import prep
 import bchroot
 import build
 import datetime
+import bSSL
+import ipxe
 
 # we need to:
 # we also need to figure out how to implement "mentos" (old bdisk) like functionality, letting us reuse an existing chroot install if possible to save time for future builds.
@@ -27,4 +29,6 @@ if __name__ == '__main__':
     build.displayStats(fulliso)
     if conf['build']['ipxe']:
         bSSL.sslPKI(conf)
+        iso = ipxe.buildIPXE(conf)
+        build.displayStats(iso)
     print('{0}: Finish.'.format(datetime.datetime.now()))

bdisk/build.py

@@ -160,7 +160,7 @@ def genUEFI(build, bdisk):
         # This is more important than it looks.
         #sizetotal = 33553920  # The spec'd EFI binary size (32MB). It's okay to go over this though (and we do)
         # because xorriso sees it as a filesystem image and adjusts the ISO automagically.
-        sizetotal = 786432  # we start with 768KB and add to it for wiggle room
+        sizetotal = 2097152  # we start with 2MB and add to it for wiggle room
         sizefiles = ['/boot/' + bdisk['uxname'] + '.64.img',
                     '/boot/' + bdisk['uxname'] + '.64.kern',
                     '/EFI/boot/bootx64.efi',
@@ -366,6 +366,7 @@ def genISO(conf):
     # Get size of ISO
     iso = {}
     iso['name'] = ['Main']
+    iso['Main'] = {}
     iso['Main']['sha'] = hashlib.sha256()
     with open(isopath, 'rb') as f:
         while True:
@@ -373,7 +374,7 @@ def genISO(conf):
             if not stream:
                 break
             iso['Main']['sha'].update(stream)
-    iso['Main']['sha'] = iso['sha'].hexdigest()
+    iso['Main']['sha'] = iso['Main']['sha'].hexdigest()
     iso['Main']['file'] = isopath
     iso['Main']['size'] = humanize.naturalsize(os.path.getsize(isopath))
     iso['Main']['type'] = 'Full'
@@ -382,10 +383,10 @@ def genISO(conf):
 
 def displayStats(iso):
     for i in iso['name']:
-        print("{0}:\n== {1} {2} ==".format(datetime.datetime.now(), iso[i]['type'], iso[i]['fmt']))
+        print("{0}: == {1} {2} ==".format(datetime.datetime.now(), iso[i]['type'], iso[i]['fmt']))
-        print('Size: {0}'.format(iso[i]['size']))
+        print('\t\t\t    = Size: {0}'.format(iso[i]['size']))
-        print('SHA256: {0}'.format(iso[i]['sha']))
+        print('\t\t\t    = SHA256: {0}'.format(iso[i]['sha']))
-        print('Location: {0}\n'.format(iso[i]['file']))
+        print('\t\t\t    = Location: {0}\n'.format(iso[i]['file']))
 
 def cleanUp():
     # TODO: clear out all of tempdir?

bdisk/ipxe.py

@@ -6,54 +6,32 @@ import jinja2
 import git
 import patch
 import datetime
-import bSSL
+import humanize
+import hashlib
 
 
-def sslIPXE(conf):
-    try:
-        ca = conf['ipxe']['ssl_ca']
-    except:
-        ca = None
-    try:
-        cakey = conf['ipxe']['ssl_cakey']
-    except:
-        cakey = None
-    try:
-        crt = conf['ipxe']['ssl_crt']
-    except:
-        crt = None
-    try:
-        key = conf['ipxe']['ssl_key']
-    except:
-        key = None
-    # http://www.pyopenssl.org/en/stable/api/crypto.html#pkey-objects
-    # http://docs.ganeti.org/ganeti/2.14/html/design-x509-ca.html
-    if not cakey:
-        cakey = bSSL.sslCAKey
-
-    pass
-
 def buildIPXE(conf):
     build = conf['build']
     bdisk = conf['bdisk']
     ipxe = conf['ipxe']
+    mini = ipxe['iso']
+    usb = ipxe['usb']
     tempdir = conf['build']['tempdir']
     templates_dir = build['basedir'] + '/extra/templates'
     ipxe_tpl = templates_dir + '/iPXE'
     patches_dir = tempdir + '/patches'
     srcdir = build['srcdir']
-    embedscript = tempdir + '/EMBED'
+    embedscript = build['dlpath'] + '/EMBED'
     ipxe_src = srcdir + '/ipxe'
     img_path = build['isodir'] + '/'
     ipxe_usb = '{0}-{1}.usb.img'.format(bdisk['uxname'], bdisk['ver'])
     ipxe_mini = '{0}-{1}.mini.iso'.format(bdisk['uxname'], bdisk['ver'])
     usb_file = '{0}/{1}'.format(img_path, ipxe_usb)
-    mini_file = '{0}/{1}'.format(img_path, ipxe_mini)
+    mini_file = '{0}{1}'.format(img_path, ipxe_mini)
     ipxe_git_uri = 'git://git.ipxe.org/ipxe.git'
     patches_git_uri = 'https://github.com/eworm-de/ipxe.git'
-    print('{0}: Building iPXE in {1}. Please wait...'.format(
+    print('{0}: Preparing and fetching sources for iPXE. Please wait...'.format(
-                                            datetime.datetime.now(),
+                                        datetime.datetime.now()))
-                                            ipxe_src))
     # Get the source and apply some cherrypicks
     if os.path.isdir(ipxe_src):
         shutil.rmtree(ipxe_src)
@@ -88,72 +66,91 @@ def buildIPXE(conf):
     # Feature enabling
     # In config/general.h
     with open('{0}/src/config/general.h'.format(ipxe_src), 'r') as f:
-        generalconf = f.readlines()
+        generalconf = f.read()
     # And in config/console.h
     with open('{0}/src/config/console.h'.format(ipxe_src), 'r') as f:
-        consoleconf = f.readlines()
+        consoleconf = f.read()
-    gendict = {'^#undef(\s*NET_PROTO_IPV6)':'#define\g<1>',  # enable IPv6
+    patterns = (('^#undef(\s*NET_PROTO_IPV6.*)$','#define\g<1>'),  # enable IPv6
-                '^#undef(\s*DOWNLOAD_PROTO_HTTPS)':'#define\g<1>',  # enable HTTPS
+                ('^#undef(\s*DOWNLOAD_PROTO_HTTPS)','#define\g<1>'),  # enable HTTPS
-                '^//(#define\s*IMAGE_TRUST_CMD)':'\g<1>',  # moar HTTPS
+                ('^//(#define\s*IMAGE_TRUST_CMD)','\g<1>'),  # moar HTTPS
-                '^#undef(\s*DOWNLOAD_PROTO_FTP)':'#define\g<1>'}  # FTP
+                ('^#undef(\s*DOWNLOAD_PROTO_FTP)','#define\g<1>'))  # enable FTP 
-                #'^//(#define\s*CONSOLE_CMD)':'\g<1>',  # BROKEN in EFI? TODO. if enable, replace } with , above etc.
+                #('^//(#define\s*CONSOLE_CMD)','\g<1>'),  # BROKEN in EFI? TODO. if enable, replace } with , above etc.
-                #'^//(#define\s*IMAGE_PNG':'\g<1>'}  # SAME, broken in EFI? TODO.
+                #('^//(#define\s*IMAGE_PNG','\g<1>'),  # SAME, broken in EFI? TODO.
-    consdict = {}#'^//(#define\s*CONSOLE_VESAFB)':'\g<1>'}  # BROKEN in EFI? TODO.
+    #console = ('^//(#define\s*CONSOLE_VESAFB)','\g<1>')  # BROKEN in EFI? TODO.
     # https://stackoverflow.com/a/4427835
     # https://emilics.com/notebook/enblog/p869.html
-    sedlike = re.compile('|'.join(gendict.keys()))
+    # The above methods don't seem to work. it craps out on the pattern matchings
+    # so we use tuples instead.
+    for x in patterns:
+        generalconf = re.sub(x[0], x[1], generalconf, flags=re.MULTILINE)
     with open('{0}/src/config/general.h'.format(ipxe_src), 'w') as f:
-        f.write(re.sub(lambda m: gendict[m.group(0)], generalconf))
+        f.write(generalconf)
     # Uncomment when we want to test the above consdict etc.
-    #sedlike = re.compile('|'.join(consdict.keys()))
+    #for x in patterns:
+    #    generalconf = re.sub(x[0], x[1], generalconf, flags=re.MULTILINE)
     #with open('{0}/src/config/console.h'.format(ipxe_src), 'w') as f:
-    #    f.write(re.sub(lambda m: consdict[m.group(0)], consoleconf))
+    #    f.write(console)
     # Now we make!
     cwd = os.getcwd()
     os.chdir(ipxe_src + '/src')
     # TODO: split this into logic to only create the selected images.
     # Command to build the .efi file
-    build_efi = ['/usr/bin/make',
+    modenv = os.environ.copy()
+    modenv['EMBED'] = embedscript
+    #modenv['TRUST'] = ipxe_ssl_ca  # TODO: test these
+    #modenv['CERT'] = '{0},{1}'.format(ipxe_ssl_ca, ipxe_ssl_crt)  # TODO: test these
+    #modenv['PRIVKEY'] = ipxe_ssl_ckey  # TODO: test these
+    build_cmd = {}
+    build_cmd['efi'] = ['/usr/bin/make',
                 'bin-i386-efi/ipxe.efi',
-                'bin-x86_64-efi/ipxe.efi',
+                'bin-x86_64-efi/ipxe.efi']
-                'EMBED="{0}"'.format(embedscript)
-                #'TRUST="{0}"'.format(ipxe_ssl_ca),  # finish work on ipxe SSL. make sure you throw a comma at the end above.
-                #'CERT="{0},{1}"'.format(ipxe_ssl_ca, ipxe_ssl_crt),  # finish work on ipxe SSL
-                #'PRIVKEY="{0}"'.format(ipxe_ssl_ckey)
-                ]
     # Command to build the actual USB and Mini images
-    build_boots = ['/usr/bin/make',
+    build_cmd['img'] = ['/usr/bin/make']
-                'bin/ipxe.eiso',
-                'bin/ipxe.usb',
-                'EMBED="{0}"'.format(embedscript)
-                #'TRUST="{0}"'.format(ipxe_ssl_ca),  # finish work on ipxe SSL. make sure you throw a comma at the end above.
-                #'CERT="{0},{1}"'.format(ipxe_ssl_ca, ipxe_ssl_crt),  # finish work on ipxe SSL
-                #'PRIVKEY="{0}"'.format(ipxe_ssl_ckey)
-                ]
     # Now we call the commands.
-    for n in ('build_efi', 'build_boots'):
+    DEVNULL = open(os.devnull, 'w')
-        subprocess.call(n, stdout = DEVNULL, stderr = subprocess.STDOUT)  # TODO: log the make output to a file?
+    if os.path.isfile(build['dlpath'] + '/ipxe.log'):
+        os.remove(build['dlpath'] + '/ipxe.log')
+    print(('{0}: Building iPXE in {1}. Please wait...\n\t\t\t    You can view progress' +
+            ' via:\n\t\t\t    tail -f {2}/ipxe.log').format(
+                                            datetime.datetime.now(),
+                                            ipxe_src,
+                                            build['dlpath']))
+    if mini and not usb:
+        build_cmd['img'].insert(1, 'bin/ipxe.eiso')
+    elif usb and not mini:
+        build_cmd['img'].insert(1, 'bin/ipxe.usb')
+    elif usb and mini:
+        build_cmd['img'].insert(1, 'bin/ipxe.eiso')
+        build_cmd['img'].insert(2, 'bin/ipxe.usb')
+    with open('{0}/ipxe.log'.format(build['dlpath']), 'a') as f:
+        subprocess.call(build_cmd['efi'], stdout = f, stderr = subprocess.STDOUT, env=modenv)
+        subprocess.call(build_cmd['img'], stdout = f, stderr = subprocess.STDOUT, env=modenv)
+    print('{0}: Built iPXE image(s) successfully.'.format(datetime.datetime.now()))
     os.chdir(cwd)
     # move the files to the results dir
     os.rename('{0}/src/bin/ipxe.usb'.format(ipxe_src), usb_file)
     os.rename('{0}/src/bin/ipxe.eiso'.format(ipxe_src), mini_file)
     # Get size etc. of build results
     iso = {}
+    stream = {}
     iso['name'] = []
     for t in ('USB', 'Mini'):  # TODO: do this programmatically based on config
         iso['name'].append(t)
-        iso[t]['sha'] = hashlib.sha256()
+        iso[t] = {}
+        shasum = False
+        shasum = hashlib.sha256()
         if t == 'USB':
             isopath = usb_file
         elif t == 'Mini':
             isopath = mini_file
+        stream = False
         with open(isopath, 'rb') as f:
             while True:
                 stream = f.read(65536)  # 64kb chunks
                 if not stream:
                     break
-            iso[t]['sha'].update(stream)
+                shasum.update(stream)
-        iso[t]['sha'] = iso['sha'].hexdigest()
+        iso[t]['sha'] = shasum.hexdigest()
         iso[t]['file'] = isopath
         iso[t]['size'] = humanize.naturalsize(os.path.getsize(isopath))
         iso[t]['type'] = 'iPXE {0}'.format(t)

bdisk/prep.py

@@ -49,7 +49,7 @@ def downloadTarball(build):
         # python-gnupg 0.3.9 spits this error in Arch. it's harmless, but ugly af.
         # TODO: remove this when the error doesn't happen anymore.
         print("\t\t\t    If you see a \"ValueError: Unknown status message: 'KEY_CONSIDERED'\" error,\n\t\t\t    it can be safely ignored.")
-        print("\t\t\t    If this is taking a VERY LONG time, try installing haveged and starting it.\n\t\t\t    This can be" +
+        print("\t\t\t    If this is taking a VERY LONG time, try installing haveged and starting it.\n\t\t\t    This can be " +
                         "done safely in parallel with the build process.\n")
         input_data = gpg.gen_key_input(name_email = 'tempuser@nodomain.tld', passphrase = 'placeholder_passphrase')
         key = gpg.gen_key(input_data)

docs/TODO

@@ -15,6 +15,8 @@
 -implement pyalpm to decreate dependency on chroot pacman-ing?
 --or even maybe https://wiki.archlinux.org/index.php/offline_installation_of_packages in pure python!
 -set up automatic exporting to PDF of the user manual server-side. https://pypi.python.org/pypi/unoconv/0.6
+-There *has* to be a better way of handling package installation in the chroots.
+-maybe remove lxde, firefox, chrome and replace with enlightenment/midori?
 
 
 ## NETWORKING ##

extra/dist.build.ini

@@ -220,7 +220,7 @@ isodir = ${dlpath}/iso
 ; source code we download (if we need it).
 ; 0.) No whitespace
 ; 1.) Will be created if it doesn't exist, and is needed
-srcdir = ${basedir}/extrasrc
+srcdir = ${dlpath}/src
 
 ; What directory should we use for staging?
 ; 0.) No whitespace

extra/pre-build.d/root/pre-build.sh

@@ -44,7 +44,7 @@ pacman -Syy
 # Just in case.
 cleanPacorigs
 # Install some prereqs
-pacman -S sed
+pacman -S --noconfirm --needed sed
 pacman -S --noconfirm --needed base syslinux wget rsync unzip jshon sudo abs xmlto bc docbook-xsl git
 locale-gen
 # And get rid of files it wants to replace
@@ -69,7 +69,7 @@ cleanPacorigs
 # Install multilib-devel if we're in an x86_64 chroot.
 if $(egrep -q '^\[multilib' /etc/pacman.conf);
 then
-	pacman --noconfirm -Rdd gcc gcc-libs libtool
+	yes 'y' | pacman -S --needed gcc-multilib lib32-fakeroot lib32-libltdl
 	pacman --noconfirm -S --needed multilib-devel
 	cleanPacorigs
 	TGT_ARCH='x86_64'