fingers crossed... testing build

bdisk/ipxe.py

@@ -23,7 +23,7 @@ def buildIPXE(conf):
     embedscript = build['dlpath'] + '/EMBED'
     ipxe_src = srcdir + '/ipxe'
     ipxe_git_uri = 'git://git.ipxe.org/ipxe.git'
-    patches_git_uri = 'https://github.com/eworm-de/ipxe.git'
+    #patches_git_uri = 'https://github.com/eworm-de/ipxe.git'  # DO WE EVEN NEED THIS ANYMORE THO
     print('{0}: [IPXE] Prep/fetch sources...'.format(
                                         datetime.datetime.now()))
     # Get the source and apply some cherrypicks
@@ -31,35 +31,35 @@ def buildIPXE(conf):
         shutil.rmtree(ipxe_src)
     ipxe_repo = git.Repo.clone_from(ipxe_git_uri, ipxe_src)
     # Generate patches
-    os.makedirs(patches_dir, exist_ok = True)
+    #os.makedirs(patches_dir, exist_ok = True)  # needed?
     os.makedirs(img_path, exist_ok = True)
     tpl_loader = jinja2.FileSystemLoader(ipxe_tpl)
     env = jinja2.Environment(loader = tpl_loader)
-    patches = ipxe_repo.create_remote('eworm', patches_git_uri)
+    #patches = ipxe_repo.create_remote('eworm', patches_git_uri)  # needed?
-    patches.fetch()
+    #patches.fetch()  # needed?
     # TODO: per http://ipxe.org/download#uefi, it builds efi *binaries* now.
     # we can probably skip the commit patching from eworm and the iso/eiso
     # (and even usb) generation, and instead use the same method we use in genISO
-    eiso_commit = '189652b03032305a2db860e76fb58e81e3420c4d'
+    #eiso_commit = '189652b03032305a2db860e76fb58e81e3420c4d'  # needed?
-    nopie_commit = '58557055e51b2587ad3843af58075de916e5399b'
+    #nopie_commit = '58557055e51b2587ad3843af58075de916e5399b'  # needed?
-    # patch files
+    # patch files needed?
-    for p in ('01.git-version.patch', '02.banner.patch'):
+   # for p in ('01.git-version.patch', '02.banner.patch'):
-        try:
+   #     try:
-            tpl = env.get_template('patches/{0}.j2'.format(p))
+   #         tpl = env.get_template('patches/{0}.j2'.format(p))
-            tpl_out = tpl.render(bdisk = bdisk)
+   #         tpl_out = tpl.render(bdisk = bdisk)
-            with open('{0}/{1}'.format(patches_dir, p), 'w+') as f:
+   #         with open('{0}/{1}'.format(patches_dir, p), 'w+') as f:
-                f.write(tpl_out)
+   #             f.write(tpl_out)
-            patchfile = patch.fromfile(patches_dir + '/' + p)
+   #         patchfile = patch.fromfile(patches_dir + '/' + p)
-            patchfile.apply(strip = 2, root = ipxe_src + '/src')
+   #         patchfile.apply(strip = 2, root = ipxe_src + '/src')
-        except:
+   #     except:
-            pass
+   #         pass
     tpl = env.get_template('EMBED.j2')
     tpl_out = tpl.render(ipxe = ipxe)
     with open(embedscript, 'w+') as f:
         f.write(tpl_out)
-    # Patch using the files before applying the cherrypicks
+    # Patch using the files before applying the cherrypicks needed?
-    ipxe_repo.git.cherry_pick('-n', eiso_commit)
+   # ipxe_repo.git.cherry_pick('-n', eiso_commit)
-    ipxe_repo.git.cherry_pick('-n', nopie_commit)
+   # ipxe_repo.git.cherry_pick('-n', nopie_commit)
     # Feature enabling
     # In config/general.h
     with open('{0}/src/config/general.h'.format(ipxe_src), 'r') as f:
@@ -110,11 +110,11 @@ def buildIPXE(conf):
                             'bin-i386-efi/ipxe.efi',
                             'bin-x86_64-efi/ipxe.efi',
                             'EMBED={0}'.format(embedscript)]
-    # Command to build the actual mini image
+    # Command to build the actual mini image needed?
-    build_cmd['iso'] = ['/usr/bin/make',
+   # build_cmd['iso'] = ['/usr/bin/make',
-                            'bin/ipxe.liso',
+   #                         'bin/ipxe.liso',
-                            'bin/ipxe.eiso',
+   #                         'bin/ipxe.eiso',
-                            'EMBED={0}'.format(embedscript)]
+   #                         'EMBED={0}'.format(embedscript)]
     # Now we call the commands.
     DEVNULL = open(os.devnull, 'w')
     if os.path.isfile(build['dlpath'] + '/ipxe.log'):
@@ -127,15 +127,15 @@ def buildIPXE(conf):
         subprocess.call(build_cmd['base'], stdout = f, stderr = subprocess.STDOUT, env=modenv)
         subprocess.call(build_cmd['undi'], stdout = f, stderr = subprocess.STDOUT, env=modenv)
         subprocess.call(build_cmd['efi'], stdout = f, stderr = subprocess.STDOUT, env=modenv)
-        if mini:
+        #if mini:
-            subprocess.call(build_cmd['iso'], stdout = f, stderr = subprocess.STDOUT, env=modenv)
+        #    subprocess.call(build_cmd['iso'], stdout = f, stderr = subprocess.STDOUT, env=modenv)
     print('{0}: [IPXE] Built iPXE image(s) successfully.'.format(datetime.datetime.now()))
     os.chdir(cwd)
     # move the files to the results dir
     # TODO: grab ipxe.pxe here too.
-    if mini:
+   # if mini: # needed?
-        os.rename('{0}/src/bin/ipxe.eiso'.format(ipxe_src), emini_file)
+   #     os.rename('{0}/src/bin/ipxe.eiso'.format(ipxe_src), emini_file)
-        os.rename('{0}/src/bin/ipxe.iso'.format(ipxe_src), mini_file)
+   #     os.rename('{0}/src/bin/ipxe.iso'.format(ipxe_src), mini_file)
     # Get size etc. of build results
     iso = {}
     stream = {}
@@ -167,19 +167,150 @@ def buildIPXE(conf):
                 iso[t]['fmt'] = 'ISO'
     return(iso)
 
-def genISO(conf, files):
+def genISO(conf):
     build = conf['build']
     bdisk = conf['bdisk']
     ipxe = conf['ipxe']
+    arch = build['arch']
+    ver = build['ver']
+    isofile = '{0}-{1}-{2}.mini.iso'.format(bdisk['uxname'], bdisk['ver'], build['buildnum'])
+    isopath = '{0}/{1}'.format(isodir, isofile)
+    tempdir = build['tempdir']
+    chrootdir = build['chrootdir']
     mini = ipxe['iso']
-    usb = ipxe['usb']
     iso = {}
     srcdir = build['srcdir']
     ipxe_src = srcdir + '/ipxe'
+    mountpt = build['mountpt']
+    templates_dir = build['basedir'] + '/extra/templates/iPXE/'
+    tpl_loader = jinja2.FileSystemLoader(templates_dir)
+    env = jinja2.Environment(loader = tpl_loader)
+    bootdir = tempdir + '/ipxe_mini'
+    efiboot_img = bootdir + '/efiboot.efi'
+    innerefi64 = '{0}/src/bin-x86_64-efi/ipxe.efi'.format(ipxe_src)
+    efi = False
+    # this shouldn't be necessary... if it is, we can revisit this in the future. see "Inner dir" below.
+    #innerefi32 = '{0}/src/bin-i386-efi/ipxe.efi'.format(ipxe_src)
+    # We only need to do EFI prep if we have UEFI/x86_64 support. See above, but IA64 is dead, Zed.
+    if mini and (('x86_64') in arch):
+        efi = True
+        # EFI prep/building
+        print('{0}: [IPXE] UEFI support for Mini ISO...'.format(datetime.datetime.now()))
+        if os.path.isdir(bootdir):
+            shutil.rmtree(bootdir)
+        # Inner dir (efiboot.img file)
+        sizetotal = 65536  # 64K wiggle room. increase this if we add IA64.
+        sizetotal += os.path.getsize(innerefi64)
+        print("{0}: [IPXE] Creating EFI ESP image {1} ({2})...".format(
+                                                                datetime.datetime.now(),
+                                                                efiboot_img,
+                                                                humanize.naturalsize(sizetotal)))
+        if os.path.isfile(efiboot_img):
+            os.remove(efiboot_img)
+        with open(efiboot_img, 'wb+') as f:
+            f.truncate(sizetotal)
+        DEVNULL = open(os.devnull, 'w')
+        cmd = ['/sbin/mkfs.vfat', '-F', '32', '-n', 'iPXE_EFI', efiboot_img]
+        subprocess.call(cmd, stdout = DEVNULL, stderr = subprocess.STDOUT)
+        cmd = ['/bin/mount', efiboot_img, build['mountpt']]
+        subprocess.call(cmd)
+        os.makedirs(mountpt + '/EFI/BOOT')
+        shutil.copy2(innerefi64,'{0}/EFI/BOOT/BOOTX64.EFI'.format(mountpt))
+        cmd = ['/bin/umount', mountpt]
+        subprocess.call(cmd)
+        # Outer dir
+        os.makedirs('{0}/boot'.format(bootdir), exist_ok = True)  # kernel(s)
+        os.makedirs('{0}/EFI/BOOT'.format(bootdir), exist_ok = True)  # EFI
+        os.makedirs('{0}/loader/entries'.format(bootdir), exist_ok = True)  # EFI
+        os.makedirs('{0}/isolinux'.format(bootdir), exist_ok = True)  # BIOS
+        # we reuse the preloader.efi from full ISO build
+        shutil.copy2('{0}/EFI/boot/bootx64.efi'.format(tempdir),
+                        '{0}/EFI/BOOT/BOOTX64.EFI'.format(bootdir))
+        # and we create the loader entries
+        for t in ('loader','base'):
+            if t == 'base':
+                name = bdisk['uxname']
+                tplpath = '{0}/loader/entries'.format(bootdir)
+            else:
+                name = t
+                tplpath = '{0}/loader'.format(bootdir)
+            tpl = env.get_template('EFI/{0}.conf.j2'.format(t))
+            tpl_out = tpl.render(build = build, bdisk = bdisk)
+            with open('{0}/{1}.conf'.format(tplpath, name), "w+") as f:
+                f.write(tpl_out)
     if mini:
-        # EFI prep
+        # BIOS prepping
-
+        shutil.copy2('{0}/src/bin/ipxe.lkrn'.format(ipxe_src), '{0}/boot/ipxe.krn'.format(bootdir))
-        pass  # create ISO here. hybrid, efi support
+        isolinux_filelst = ['isolinux.bin',
+                            'ldlinux.c32']
+        for f in isolinux_filelst:
+            shutil.copy2('{0}/root.{1}/usr/lib/syslinux/bios/{2}'.format(chrootdir, arch[0], f), '{0}/{1}'.format(bootdir, f))
+        tpl = env.get_template('BIOS/isolinux.cfg.j2')
+        tpl_out = tpl.render(build = build, bdisk = bdisk)
+        with open('{0}/isolinux.cfg'.format(bootdir), "w+") as f:
+            f.write(tpl_out)
+        print("{0}: [IPXE] Building Mini ISO ({1})...".format(datetime.datetime.now(), isopath))
+        if efi:
+            cmd = ['/usr/bin/xorriso',
+                    '-as', 'mkisofs',
+                    '-iso-level', '3',
+                    '-full-iso9660-filenames',
+                    '-volid', bdisk['name'] + '_MINI',
+                    '-appid', bdisk['desc'],
+                    '-publisher', bdisk['dev'],
+                    '-preparer', 'prepared by ' + bdisk['dev'],
+                    '-eltorito-boot', 'isolinux.bin',
+                    '-eltorito-catalog', 'boot.cat',
+                    '-no-emul-boot',
+                    '-boot-load-size', '4',
+                    '-boot-info-table',
+                    '-isohybrid-mbr', '{0}/root.{1}/usr/lib/syslinux/bios/isohdpfx.bin'.format(chrootdir, arch[0]),
+                    '-eltorito-alt-boot',
+                    '-e', 'efiboot.img',
+                    '-no-emul-boot',
+                    '-isohybrid-gpt-basdat',
+                    '-output', isopath,
+                    bootdir]
+        else:
+            # UNTESTED. TODO.
+            # I think i want to also get rid of: -boot-load-size 4,
+            # -boot-info-table, ... possiblyyy -isohybrid-gpt-basedat...
+            cmd = ['/usr/bin/xorriso',
+                    '-as', 'mkisofs',
+                    '-iso-level', '3',
+                    '-full-iso9660-filenames',
+                    '-volid', bdisk['name'] + '_MINI',
+                    '-appid', bdisk['desc'],
+                    '-publisher', bdisk['dev'],
+                    '-preparer', 'prepared by ' + bdisk['dev'],
+                    '-eltorito-boot', 'isolinux/isolinux.bin',
+                    '-eltorito-catalog', 'isolinux/boot.cat',
+                    '-no-emul-boot',
+                    '-boot-load-size', '4',
+                    '-boot-info-table',
+                    '-isohybrid-mbr', '{0}/root.{1}/usr/lib/syslinux/bios/isohdpfx.bin'.format(chrootdir, arch[0]),
+                    '-no-emul-boot',
+                    '-isohybrid-gpt-basdat',
+                    '-output', isopath,
+                    bootdir]
+        DEVNULL = open(os.devnull, 'w')
+        subprocess.call(cmd, stdout = DEVNULL, stderr = subprocess.STDOUT)
+        # Get size of ISO
+        iso['name'] = ['Mini']
+        iso['Mini'] = {}
+        iso['Mini']['sha'] = hashlib.sha256()
+        with open(isopath, 'rb') as f:
+            while True:
+                stream = f.read(65536)  # 64kb chunks
+                if not stream:
+                    break
+                iso['Mini']['sha'].update(stream)
+        iso['Mini']['sha'] = iso['Mini']['sha'].hexdigest()
+        iso['Mini']['file'] = isopath
+        iso['Mini']['size'] = humanize.naturalsize(os.path.getsize(isopath))
+        iso['Mini']['type'] = 'Mini'
+        iso['Mini']['fmt'] = 'Hybrid ISO'
+        return(iso)
 
 def tftpbootEnv(conf):
     build = conf['build']

extra/templates/iPXE/EFI/base.conf.j2

@@ -0,0 +1,3 @@
+title   {{ bdisk['pname'] }} iPXE (netboot)
+efi   /efiboot.img
+

extra/templates/iPXE/EFI/loader.conf.j2

@@ -0,0 +1,2 @@
+default {{ bdisk['uxname'] }}
+

extra/templates/iPXE/isolinux.cfg.j2

@@ -0,0 +1,7 @@
+DEFAULT ipxe
+PROMPT 0
+TIMEOUT 10
+
+LABEL ipxe
+  KERNEL ipxe.lkrn
+