checking in some work

TODO

@@ -9,6 +9,15 @@
 - for docs, 3.x (as of 3.10) was 2.4M.
 - GUI? at least for generating config...
 
+- SSL key gen:
+import OpenSSL
+k = OpenSSL.crypto.PKey()
+k.generate_key(OpenSSL.crypto.TYPE_RSA, 4096)
+x = OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM,
+                                   k,
+                                   cipher = 'aes256',
+                                   passphrase = 'test')
+
 - need to package:
   python-hashid (https://psypanda.github.io/hashID/,
                  https://github.com/psypanda/hashID,

bdisk/GPG.py

@@ -3,6 +3,9 @@ import os
 import psutil
 import gpg.errors
 
+# http://files.au.adversary.org/crypto/GPGMEpythonHOWTOen.html
+# https://www.gnupg.org/documentation/manuals/gpgme.pdf
+
 class GPGHandler(object):
     def __init__(self, gnupg_homedir = None, key_id = None, keyservers = None):
         self.home = gnupg_homedir

bdisk/SSL.py

@@ -1 +1,5 @@
 import OpenSSL
+# https://cryptography.io/en/latest/x509/reference/#cryptography.x509.CertificateBuilder.sign
+# migrate old functions of bSSL to use cryptography
+# but still waiting on their recpipes.
+# https://cryptography.io/en/latest/x509/tutorial/

bdisk/confgen.py

@@ -1,5 +1,8 @@
 #!/usr/bin/env python3.6
 
+# Ironically enough, I think building a GUI for this would be *cleaner*.
+# Go figure.
+
 import confparse
 import crypt
 import getpass
@@ -147,6 +150,9 @@ class ConfGenerator(object):
             self.get_accounts()
             self.get_sources()
             self.get_build()
+            self.get_iso()
+            self.get_ipxe()
+            self.get_pki()
         except KeyboardInterrupt:
             exit('\n\nCaught KeyboardInterrupt; quitting...')
         return()
@@ -290,6 +296,8 @@ class ConfGenerator(object):
                 '\nWhat is YOUR name?\nName: ')).strip()
             meta_items['dev']['email'] = (input('\nWhat is your email address?'
                                                 '\nemail: ')).strip()
+            # TODO: this always returns invalid?? and doesn't seem to trigger
+            # the redo
             if not valid.email(meta_items['dev']['email']):
                 print('Invalid; skipping...')
                 meta_items['dev']['email'] = None
@@ -395,7 +403,8 @@ class ConfGenerator(object):
                              'x86_64': ('(Also referred to by distros as '
                                         '"64-bit")')}
         while more_sources:
-            if len(_arches) == len(_supported_arches):
+            # this doesn't trigger? maybe?
+            if len(_arches) >= len(_supported_arches):
                 # All supported arches have been added. We currently don't
                 # support mirror-balancing. TODO?
                 print('\nCannot add more sources; all supported architectures '
@@ -403,7 +412,7 @@ class ConfGenerator(object):
                 more_sources = False
                 break
             if len(_arches) > 0:
-                print('\n(Currently added arches: {0})'.format(
+                print('\n\t(Currently added arches: {0})'.format(
                                                         ', '.join(_arches)))
             _print_arches = '\n\t'.join(
                 ['{0}:\t{1}'.format(*i) for i in _supported_arches.items()])
@@ -531,7 +540,7 @@ class ConfGenerator(object):
                         continue
                     sig.attrib['keys'] = sigkeys
                 else:
-                    sigkeys = detect.gpgkeyID_from_url(gpgsig)
+                    sigkeys = detect.gpgkeyID_from_url(gpgsig['full_url'])
                     if not isinstance(sigkeys, list):
                         print('Could not properly parse any keys in the '
                               'signature file. Restarting.')
@@ -553,7 +562,8 @@ class ConfGenerator(object):
                             print('\t\t{0}'.format(_uid['Name']))
                             for k in _uid:
                                 if k != 'Name':
-                                    print('\t\t\t{0}:\t{1}'.format(k, _uid[k]))
+                                    print('\t\t\t{0:<9} {1}'.format(
+                                            '{0}:'.format(k), _uid[k]))
                     _key_chk = prompt.confirm_or_no(prompt = (
                         '\n{0} look correct?\n').format(_s))
                     if not _key_chk:
@@ -633,23 +643,158 @@ class ConfGenerator(object):
                 distro_path = self.profile.xpath('//paths/distros/text()')[0]
             except IndexError:
                 distro_path = 'your "distros" path'
-            distro = (input('\nWhich distro plugin/distro base are you using? '
+            distro = (input(('\nWhich distro plugin/distro base are you '
-                            'See the manual for more information. A matching '
+                             'using? See the manual for more information. A '
-                            'plugin MUST exist in {0} for a build to '
+                             'matching plugin MUST exist in {0} for a build '
-                            'complete successfully! The default (Arch Linux, '
+                             'to complete successfully! The default (Arch '
-                            '"archlinux") will be used if left blank.'
+                             'Linux, "archlinux") will be used if left blank.'
-                            '\nDistro base: ').format(
+                             '\nDistro base: ').format(distro_path))).strip()
-                                    distro_path)).strip()
             if distro == '':
                 distro = 'archlinux'
             if not valid.plugin_name(distro):
                 print('That is not a valid name. See the manual for examples '
                       'and shipped plugins. Retrying.')
                 continue
+            else:
+                has_distro = True
             distro_elem = lxml.etree.SubElement(build, 'distro')
             distro_elem.text = distro
         return()
     
+    def get_iso(self):
+        print('\n++ ISO ++')
+        # We don't need to ask if it's multiarch if we only have one arch.
+        iso = lxml.etree.Element('iso')
+        _arches = []
+        for _source in self.profile.xpath('sources/source'):
+            _arches.append(_source.attrib['arch'])
+        if len(_arches) < 2:
+            iso.attrib['multi_arch'] = _arches[0]
+            self.profile.append(iso)
+        # We have more than one arch, so we need to ask how they want to handle
+        # it.
+        _ma_strings = {'yes': ('a multi-arch ISO (both architectures on one '
+                               'ISO)'),
+                       'no': ('separate image files for '
+                              '{0}').format(' and '.join(_arches))}
+        for a in _arches:
+            _ma_strings[a] = 'only build an image file for {0}'.format(a)
+        if len(_arches) > 1:
+            _multi_arch_input = None
+            while not _multi_arch_input:
+                print('\n++ ISO || MULTI-ARCH ++')
+                _multi_arch = (input((
+                    '\nYou have defined mutliple architecture sources. BDisk '
+                    'allows you to build an ISO image (USB image, etc.) that '
+                    'will support both architectures using the same file. '
+                    'Please consult the manual if you need further '
+                    'information.\nPossible values:\n'
+                    '\n\t{0}\n\nMulti-arch: ').format(
+                    '\n\t'.join(
+                    ['{0}:\t{1}'.format(k, v) for k, v in _ma_strings.items()]
+                        )))).strip().lower()
+                if _multi_arch not in _ma_strings.keys():
+                    print('Invalid selection; retrying.')
+                    continue
+                else:
+                    _multi_arch_input = _multi_arch
+            iso.attrib['multi_arch'] = _multi_arch_input
+        _gpg_sign = None
+        while not _gpg_sign:
+            print('\n++ ISO || SIGNING ++')
+            _gpg_input = prompt.confirm_or_no(prompt = (
+                '\nWe can sign ISO image files using GPG (we\'ll give the '
+                'option to configure it a bit later).\nWould you like to sign '
+                'the ISO/USB image files with GPG?\n'), usage = (
+                                '{0} for yes, {1} for no...\n'))
+            _gpg_sign = ('yes' if _gpg_input else 'no')
+        iso.attrib['sign'] = _gpg_sign
+        self.profile.append(iso)
+        return()
+    
+    def get_ipxe(self):
+        print('\n++ iPXE ++')
+        ipxe = lxml.etree.Element('ipxe')
+        _ipxe = None
+        while not _ipxe:
+            _ipxe = prompt.confirm_or_no(prompt = (
+                '\nBDisk has built-in support for iPXE (https://ipxe.org/, '
+                'see the manual for more information). Would you like to '
+                'build iPXE support?\n'), usage = (
+                                '{0} for yes, {1} for no...\n'))
+            _ipxe = ('yes' if _ipxe else 'no')
+        if _ipxe == 'yes':
+            print('\n++ iPXE || MINI-ISO ++')
+            _iso = prompt.confirm_or_no(prompt = (
+                '\nWould you like to build a "mini-ISO" (see the manual) for '
+                'bootstrapping iPXE booting from USB or optical media?\n'),
+                                    usage = ('{0} for yes, {1} for no...\n'))
+            ipxe.attrib['iso'] = ('yes' if _iso else 'no')
+            print('\n++ iPXE || SIGNING ++')
+            _sign = prompt.confirm_or_no(prompt = (
+                '\nBDisk can sign the mini-ISO and other relevant files for '
+                'iPXE builds using GPG. Would you like to sign the iPXE build '
+                'distributables? (You\'ll have the chance to configure GPG '
+                'later).\n'), usage = ('{0} for yes, {1} for no...\n'))
+            ipxe.attrib['sign'] = ('yes' if _sign else 'no')
+            _uri = None
+            while not _uri:
+                print('\n++ iPXE || URL ++')
+                _uri = (input(
+                    '\niPXE uses a remote URI to boot. What URI should this '
+                    'profile\'s iPXE use? (Consult the manual for more '
+                    'information.)\niPXE Boot URL: ')).strip()
+                if not valid.url(_uri):
+                    print('Invalid URL, retrying...')
+                    _uri = None
+                    continue
+                else:
+                    uri = lxml.etree.SubElement(ipxe, 'uri')
+                    uri.text = _uri
+        if _ipxe == 'yes':
+            self.profile.append(ipxe)
+        return()
+    
+    def get_pki(self):
+        print('\n++ SSL/TLS PKI ++')
+        pki = lxml.etree.Element('pki')
+        _pki = None
+        while not _pki:
+            _pki = prompt.confirm_or_no(prompt = (
+                '\nWould you like to support SSL/TLS transport for various '
+                'functions? Currently this is only used for iPXE, but future '
+                'applications may be possible.\n'),
+                                    usage = ('{0} for yes, {1} for no...\n'))
+        if _pki:
+            _pki_url_chk = self.profile.xpath('ipxe/uri/text()')
+            _pki_url = (_pki_url_chk[0] if _pki_url_chk else None)
+            print('\n++ SSL/TLS PKI || OVERWRITE ++')
+            _overwrite = prompt.confirm_or_no(prompt = (
+                '\nYou\'ll have the opportunity in a moment to configure '
+                'paths for the various files, but do you want BDisk to '
+                're-generate (and thus overwrite) any of the files it finds? '
+                'If you use these files for anything OTHER than BDisk (or '
+                'wish to keep persistent keys and certs), you should '
+                'DEFINITELY answer no here.\n'),
+                    usage = ('{0} for yes, {1} for no...\n'))
+            pki.attrib['overwrite'] = ('yes' if _overwrite else 'no')
+            for x in ('ca', 'client'):
+                print('\n++ SSL/TLS PKI || {0} ++'.format(x.upper()))
+                _x = None
+                while not _x:
+                    _x = prompt.ssl_object(x, _pki_url)
+                elem = lxml.etree.SubElement(pki, x)
+                _elems = {}
+                for e in _x['paths']:
+                    _elems[e] = lxml.etree.SubElement(elem, e)
+                    _elems[e].text = _x['paths'][e]
+                for e in _x['attribs']:
+                    for a in _x['attribs'][e]:
+                        _elems[e].attrib[a] = _x['attribs'][e][a]
+        if _pki:
+            self.profile.append(pki)
+        return()
+
 def main():
     cg = ConfGenerator()
     cg.main()

bdisk/utils.py

@@ -3,11 +3,11 @@ import crypt
 import GPG
 import hashid
 import hashlib
+import iso3166
 import os
 import pprint
 import re
 import string
-import textwrap
 import uuid
 import validators
 import zlib
@@ -17,6 +17,7 @@ from dns import resolver
 from email.utils import parseaddr as emailparse
 from passlib.context import CryptContext as cryptctx
 from urllib.parse import urlparse
+from urllib.request import urlopen
 
 # Supported by all versions of GNU/Linux shadow
 passlib_schemes = ['des_crypt', 'md5_crypt', 'sha256_crypt', 'sha512_crypt']
@@ -26,19 +27,21 @@ digest_schemes = list(hashlib.algorithms_available)
 # Provided by zlib
 digest_schemes.append('adler32')
 digest_schemes.append('crc32')
-#clean_digest_schemes = sorted(list(set(digest_schemes)))
 
 crypt_map = {'sha512': crypt.METHOD_SHA512,
              'sha256': crypt.METHOD_SHA256,
              'md5': crypt.METHOD_MD5,
              'des': crypt.METHOD_CRYPT}
 
-class XPathFmt(string.Formatter):
+# These are *key* ciphers, for encrypting exported keys.
-    def __init__(self):
+openssl_ciphers = ['aes128', 'aes192', 'aes256', 'bf', 'blowfish',
-        print('foo')
+                   'camellia128', 'camellia192', 'camellia256', 'cast', 'des',
+                   'des3', 'idea', 'rc2', 'seed']
+openssl_digests = ['blake2b512', 'blake2s256', 'gost', 'md4', 'md5', 'mdc2',
+                   'rmd160', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512']
 
+class XPathFmt(string.Formatter):
     def get_field(self, field_name, args, kwargs):
-        # custom arg to specify if it's a regex pattern or not
         vals = self.get_value(field_name, args, kwargs), field_name
         if not vals[0]:
             vals = ('{{{0}}}'.format(vals[1]), vals[1])
@@ -51,7 +54,7 @@ class detect(object):
     def any_hash(self, hash_str):
         h = hashid.HashID()
         hashes = []
-        for i in h.IdentifyHash(hash_str):
+        for i in h.identifyHash(hash_str):
             if i.extended:
                 continue
             x = i.name
@@ -74,7 +77,7 @@ class detect(object):
         return()
 
     def gpgkeyID_from_url(self, url):
-        with urlparse(url) as u:
+        with urlopen(url) as u:
             data = u.read()
         g = GPG.GPGHandler()
         key_ids = g.get_sigs(data)
@@ -85,7 +88,7 @@ class detect(object):
         def _get_key():
             key = None
             try:
-                key = g.get_key(keyID, secret = secret)
+                key = g.ctx.get_key(keyID, secret = secret)
             except GPG.gpg.errors.KeyNotFound:
                 return(None)
             except Exception:
@@ -93,16 +96,16 @@ class detect(object):
             return(key)
         uids = {}
         g = GPG.GPGHandler()
-        _orig_kl_mode = g.get_keylist_mode()
+        _orig_kl_mode = g.ctx.get_keylist_mode()
         if _orig_kl_mode != GPG.gpg.constants.KEYLIST_MODE_EXTERN:
             _key = _get_key()
             if not _key:
-                g.set_keylist_mode(GPG.gpg.constants.KEYLIST_MODE_EXTERN)
+                g.ctx.set_keylist_mode(GPG.gpg.constants.KEYLIST_MODE_EXTERN)
                 _key = _get_key()
         else:
             _key = _get_key()
         if not _key:
-            g.set_keylist_mode(_orig_kl_mode)
+            g.ctx.set_keylist_mode(_orig_kl_mode)
             del(g)
             return(None)
         else:
@@ -119,7 +122,7 @@ class detect(object):
                 _u['Invalid'] = (True if _uid.invalid else False)
                 _u['Revoked'] = (True if _uid.revoked else False)
                 uids['User IDs'].append(_u)
-        g.set_keylist_mode(_orig_kl_mode)
+        g.ctx.set_keylist_mode(_orig_kl_mode)
         del(g)
         return(uids)
 
@@ -163,7 +166,7 @@ class prompts(object):
 
     def confirm_or_no(self, prompt = '', invert = False,
                       usage = '{0} to confirm, otherwise {1}...\n'):
-        # A simplified version of multiline_input, really.
+        # A simplified version of multiline_input(), really.
         # By default, Enter confirms (and returns True) and CTRL-d returns
         # False unless - you guessed it - invert is True.
         # usage is a string appended to prompt that explains which keys to use.
@@ -224,12 +227,136 @@ class prompts(object):
                 print(end_str)
         return('\n'.join(_lines))
 
-    def path(self, path_desc):
+    def path(self, path_desc, empty_passthru = False):
         path = input(('\nWhere would you like to put {0}?\n'
                       'Path: ').format(path_desc))
+        if empty_passthru:
+            if path.strip() == '':
+                return('')
         path = transform().full_path(path)
         return(path)
     
+    def ssl_object(self, pki_role, cn_url):
+        ssl_vals = {'paths': {},
+                    'attribs': {},
+                    'subject': {}}
+        # pki_role should be 'ca' or 'client'
+        if pki_role not in ('ca', 'client'):
+            raise ValueError('pki_role must be either "ca" or "client"')
+        _attribs = {'cert': {'hash_algo': {'text': ('What hashing algorithm '
+                                'do you want to use? (Default is sha512.)'),
+                                           'prompt': 'Hashing algorithm: ',
+                                           'options': openssl_digests,
+                                           'default': 'sha512'}},
+                    'key': {'cipher': {'text': ('What encryption algorithm/'
+                                'cipher do you want to use? (Default is '
+                                                'aes256.)'),
+                                      'prompt': 'Cipher: ',
+                                      'options': openssl_ciphers,
+                                       'default': 'aes256'},
+                            # This can actually theoretically be anywhere from
+                            # 512 to... who knows how high. I couldn't find the
+                            # upper bound. So we just set it to sensible
+                            # defaults. If they want something higher, they can
+                            # edit the XML when they're done.
+                            'keysize': {'text': ('What keysize/length (in '
+                                'bits) do you want the key to be? (Default is '
+                                '4096; much higher values are possible but '
+                                'are untested and thus not supported by this '
+                                'tool; feel free to edit the generated '
+                                'configuration by hand.)'),
+                                        'prompt': 'Keysize: ',
+                                        'options': ['1024', '2048', '4096'],
+                                        'default': '4096'}}}
+        _paths = {'cert': '(or read from) the certificate',
+                  'key': '(or read from) the key',
+                  'csr': ('(or read from) the certificate signing request (if '
+                          'blank, we won\'t write to disk - the operation '
+                          'will occur entirely in memory assuming we need to '
+                          'generate/sign)')}
+        if pki_role == 'ca':
+            _paths['index'] = ('(or read from) the CA DB index file (if left '
+                              'blank, one will not be used)')
+            _paths['serial'] = ('(or read from) the CA DB serial file (if '
+                                'left blank, one will not be used)')
+        for a in _attribs:
+            ssl_vals['attribs'][a] = {}
+            for x in _attribs[a]:
+                ssl_vals['attribs'][a][x] = None
+        for p in _paths:
+            if p == 'csr':
+                _allow_empty = True
+            else:
+                _allow_empty = False
+            ssl_vals['paths'][p] = self.path(_paths[p],
+                                             empty_passthru = _allow_empty)
+            print()
+            if ssl_vals['paths'][p] == '':
+                ssl_vals['paths'][p] = None
+            if p in _attribs:
+                for x in _attribs[p]:
+                    while not ssl_vals['attribs'][p][x]:
+                        ssl_vals['attribs'][p][x] = (input(
+                            ('\n{0}\n\n\t{1}\n\n{2}').format(
+                                    _attribs[p][x]['text'],
+                                    '\n\t'.join(_attribs[p][x]['options']),
+                                    _attribs[p][x]['prompt'])
+                                )).strip().lower()
+                        if ssl_vals['attribs'][p][x] not in \
+                                                    _attribs[p][x]['options']:
+                            print(('\nInvalid selection; setting default '
+                                   '({0}).').format(_attribs[p][x]['default']))
+                            ssl_vals['attribs'][p][x] = \
+                                                    _attribs[p][x]['default']
+        _subject = {'countryName': {'text': ('the 2-letter country '
+                                             'abbreviation (must conform to '
+                                             'ISO3166 ALPHA-2)?\nCountry '
+                                             'code: '),
+                                    'check': 'func',
+                                    'func': valid().country_abbrev},
+                    'localityName': {'text': ('the city/town/borough/locality '
+                                              'name?\nLocality: '),
+                                     'check': None},
+                    'stateOrProvinceName': {'text': ('the state/region '
+                                                     'name (full string)?'
+                                                     '\nRegion: '),
+                                            'check': None},
+                    'organization': {'text': ('your organization\'s name?'
+                                              '\nOrganization: '),
+                                     'check': None},
+                    'organizationalUnitName': {'text': ('your department/role/'
+                                                        'team/department name?'
+                                                        '\nOrganizational '
+                                                        'Unit: '),
+                                               'check': None},
+                    'emailAddress': {'text': ('the email address to be '
+                                              'associated with this '
+                                              'certificate/PKI object?\n'
+                                              'Email: '),
+                                     'check': 'func',
+                                     'func': valid().email}}
+        for s in _subject:
+            ssl_vals['subject'][s] = None
+        for s in _subject:
+            while not ssl_vals['subject'][s]:
+                _input = (input(
+                            ('\nWhat is {0}').format(_subject[s]['text'])
+                        )).strip()
+                _chk = _subject[s]['check']
+                if _chk:
+                    if _chk == 'func':
+                        _chk = _subject[s]['func'](_input)
+                        if not _chk:
+                            print('Invalid value; retrying.')
+                            continue
+                print()
+                ssl_vals['subject'][s] = _input
+        _url = transform().url_to_dict(cn_url, no_None = True)
+        ssl_vals['subject']['commonName'] = _url['host']
+        if pki_role == 'client':
+            ssl_vals['subject']['commonName'] += ' (Client)'
+        return(ssl_vals)
+
 class transform(object):
     def __init__(self):
         pass
@@ -280,6 +407,7 @@ class transform(object):
         text_out = re.sub('[^\w]', '', text_out)
         return(text_out)
 
+    # noinspection PyDictCreation
     def url_to_dict(self, orig_url, no_None = False):
         def _getuserinfo(uinfo_str):
             if len(uinfo_str) == 0:
@@ -332,25 +460,25 @@ class transform(object):
                     return(None)
                 else:
                     return('')
-            params = {}
+            _params = {}
             for i in in_str.split(split_char):
                 p = [x.strip() for x in i.split('=')]
-                params[p[0]] = p[1]
+                _params[p[0]] = p[1]
-            if not params:
+            if not _params:
                 if not no_None:
                     return(None)
                 else:
                     return('')
-            if not params and not no_None:
+            if not _params and not no_None:
                 return(None)
-            return(params)
+            return(_params)
         _dflt_ports = _getdfltport()
         scheme = None
-        _scheme_re = re.compile('^([\w+\.-]+)(://.*)', re.IGNORECASE)
+        _scheme_re = re.compile('^([\w+.-]+)(://.*)', re.IGNORECASE)
         if not _scheme_re.search(orig_url):
             # They probably didn't prefix a URI signifier (RFC3986 § 3.1).
             # We'll add one for them.
-            url = 'http://' + url
+            url = 'http://' + orig_url
             scheme = 'http'
         else:
             # urlparse's .scheme? Total trash.
@@ -407,7 +535,7 @@ class transform(object):
                'url': orig_url}
         url['full_url'] = '{0}://'.format(scheme)
         if userinfo not in (None, ''):
-            url['full_url'] += '{user}:{password}@'.format(userinfo)
+            url['full_url'] += '{user}:{password}@'.format(**userinfo)
         url['full_url'] += host
         if port not in (None, ''):
             url['full_url'] += ':{0}'.format(port)
@@ -424,145 +552,15 @@ class transform(object):
             url['full_url'] += '#{0}'.format('#'.join(_f))
         return(url)
 
-class xml_supplicant(object):
-    def __init__(self, cfg, profile = None, max_recurse = 5):
-        raw = self._detect_cfg(cfg)
-        xmlroot = lxml.etree.fromstring(raw)
-        self.btags = {'xpath': {},
-                      'regex': {}}
-        self.fmt = XPathFmt()
-        self.max_recurse = max_recurse
-        #self.ptrn = re.compile('(?<=(?<!\{)\{)[^{}]*(?=\}(?!\}))')
-        # I don't have permission to credit them, but to the person who helped
-        # me with this regex - thank you. You know who you are.
-        self.ptrn = re.compile(('(?<=(?<!\{)\{)(?:[^{}]+'
-                                '|{{[^{}]*}})*(?=\}(?!\}))'))
-        self.root = lxml.etree.ElementTree(xmlroot)
-        if not profile:
-            self.profile = xmlroot.xpath('/bdisk/profile[1]')[0]
-        else:
-            self.profile = xmlroot.xpath(profile)[0]
-        
-    def _detect_cfg(self, cfg):
-        if isinstance(cfg, str):
-            try:
-                lxml.etree.fromstring(cfg.encode('utf-8'))
-                return(cfg.encode('utf-8'))
-            except lxml.etree.XMLSyntaxError:
-                path = os.path.abspath(os.path.expanduser(cfg))
-                try:
-                    with open(path, 'rb') as f:
-                        cfg = f.read()
-                except FileNotFoundError:
-                    raise ValueError('Could not open {0}'.format(path))
-        elif isinstance(cfg, _io.TextIOWrapper):
-            _cfg = cfg.read().encode('utf-8')
-            cfg.close()
-            cfg = _cfg
-        elif isinstance(cfg,  _io.BufferedReader):
-            _cfg = cfg.read()
-            cfg.close()
-            cfg = _cfg
-        elif isinstance(cfg, lxml.etree._Element):
-            return(lxml.etree.tostring(cfg))
-        elif isinstance(cfg, bytes):
-            return(cfg)
-        else:
-            raise TypeError('Could not determine the object type.')
-        return(cfg)
-
-    def get_path(self, element):
-        path = element
-        try:
-            path = self.root.getpath(element)
-        except ValueError:
-            raise ValueError(
-                (
-                    'Could not find a path for the expression {0}'
-                ).format(element.text))
-        return(path)
-
-    def substitute(self, element, recurse_count = 0):
-        if recurse_count >= self.max_recurse:
-            return(element)
-        if isinstance(element, lxml.etree._Element):
-            if isinstance(element, lxml.etree._Comment):
-                return(element)
-#            if len(element) == 0:
-#                print(element.text)
-            if element.text:
-                _dictmap = self.xpath_to_dict(element.text)
-                while _dictmap:
-                    for elem in _dictmap:
-                        if isinstance(_dictmap[elem], str):
-                            try:
-                                newpath = element.xpath(_dictmap[elem])
-                            except (AttributeError, IndexError, TypeError):
-                                newpath = element
-                            except lxml.etree.XPathEvalError:
-                                return(element)
-                            try:
-                                self.btags['xpath'][elem] = self.substitute(
-                                            newpath, (recurse_count + 1))[0]
-                            except (IndexError, TypeError):
-                                raise ValueError(
-                                    ('Encountered an error while trying to '
-                                     'substitute {0} at {1}').format(
-                                        elem, self.get_path(element)
-                                    ))
-                            print(element.text)
-                            element.text = self.fmt.format(
-                                                element.text,
-                                                {**self.btags['xpath'],
-                                                 **self.btags['regex']})
-#                            element.text = self.fmt.vformat(
-#                                                element.text,
-#                                                [],
-#                                                {**self.btags['xpath'],
-#                                                 **self.btags['regex']})
-#                            element.text = (element.text).format(
-#                                                    {**self.btags['xpath'],
-#                                                     **self.btags['regex']})
-                            _dictmap = self.xpath_to_dict(element.text)
-        return(element)
-
-    def xpath_selector(self, selectors,
-                       selector_ids = ('id', 'name', 'uuid')):
-        # selectors is a dict of {attrib:value}
-        xpath = ''
-        for i in selectors.items():
-            if i[1] and i[0] in selector_ids:
-                xpath += '[@{0}="{1}"]'.format(*i)
-        return(xpath)
-
-    def xpath_to_dict(self, text_in):
-        d = None
-        ptrn_id = self.ptrn.findall(text_in)
-        if len(ptrn_id) >= 1:
-            for item in ptrn_id:
-                if not isinstance(d, dict):
-                    d = {}
-                try:
-                    _, xpath_expr = item.split('%', 1)
-                    if _ not in self.btags:
-                        continue
-                    if item not in self.btags[_]:
-                        self.btags[_][item] = None
-                        if _ == 'regex':
-                            _re = re.sub('^regex%', '', item)
-                            _re = re.sub('{{(.*)}}', '\g<1>', _re)
-                            # We use a native python object
-                            self.btags['regex'][item] = re.compile(_re)
-                    d[item] = xpath_expr
-                except ValueError:
-                    return(None)
-        return(d)
-
-
 class valid(object):
     def __init__(self):
         pass
 
+    def country_abbrev(self, country_code):
+        if country_code not in iso3166.countries_by_alpha2:
+            return(False)
+        return(True)
+
     def dns(self, addr):
         pass
 
@@ -572,7 +570,7 @@ class valid(object):
 
     def email(self, addr):
         return(
-            isinstance(validators.email(emailparse(addr)[1]),
+            not isinstance(validators.email(emailparse(addr)[1]),
                       validators.utils.ValidationFailure))
 
     def gpgkeyID(self, key_id):
@@ -626,9 +624,10 @@ class valid(object):
 
     def salt_hash(self, salthash):
         _idents = ''.join([i.ident for i in crypt_map if i.ident])
-        _regex = re.compile('^(\$[{0}]\$)?[./0-9A-Za-z]{0,16}\$?'.format(
+        # noinspection PyStringFormat
+        _regex = re.compile('^(\$[{0}]\$)?[./0-9A-Za-z]{{0,16}}\$?'.format(
                                                                     _idents))
-        if not regex.search(salthash):
+        if not _regex.search(salthash):
             return(False)
         return(True)
 
@@ -650,7 +649,7 @@ class valid(object):
         return(True)
 
     def url(self, url):
-        if not re.search('^[\w+\.-]+://', url):
+        if not re.search('^[\w+.-]+://', url):
             # They probably didn't prefix a URI signifier (RFC3986 § 3.1).
             # We'll add one for them.
             url = 'http://' + url
@@ -670,9 +669,152 @@ class valid(object):
     def uuid(self, uuid_str):
         is_uuid = True
         try:
-            u = uuid.UUID(uuid_in)
+            u = uuid.UUID(uuid_str)
         except ValueError:
             return(False)
-        if not uuid_in == str(u):
+        if not uuid_str == str(u):
             return(False)
         return(is_uuid)
+
+class xml_supplicant(object):
+    def __init__(self, cfg, profile = None, max_recurse = 5):
+        raw = self._detect_cfg(cfg)
+        xmlroot = lxml.etree.fromstring(raw)
+        self.btags = {'xpath': {},
+                      'regex': {},
+                      'variable': {}}
+        self.fmt = XPathFmt()
+        self.max_recurse = max_recurse
+        # I don't have permission to credit them, but to the person who helped
+        # me with this regex - thank you. You know who you are.
+        self.ptrn = re.compile(('(?<=(?<!\{)\{)(?:[^{}]+'
+                                '|{{[^{}]*}})*(?=\}(?!\}))'))
+        self.root = lxml.etree.ElementTree(xmlroot)
+        if not profile:
+            self.profile = xmlroot.xpath('/bdisk/profile[1]')[0]
+        else:
+            self.profile = xmlroot.xpath(profile)[0]
+        self._parse_regexes()
+        self._parse_variables()
+        
+    def _detect_cfg(self, cfg):
+        if isinstance(cfg, str):
+            try:
+                lxml.etree.fromstring(cfg.encode('utf-8'))
+                return(cfg.encode('utf-8'))
+            except lxml.etree.XMLSyntaxError:
+                path = os.path.abspath(os.path.expanduser(cfg))
+                try:
+                    with open(path, 'rb') as f:
+                        cfg = f.read()
+                except FileNotFoundError:
+                    raise ValueError('Could not open {0}'.format(path))
+        elif isinstance(cfg, _io.TextIOWrapper):
+            _cfg = cfg.read().encode('utf-8')
+            cfg.close()
+            cfg = _cfg
+        elif isinstance(cfg,  _io.BufferedReader):
+            _cfg = cfg.read()
+            cfg.close()
+            cfg = _cfg
+        elif isinstance(cfg, lxml.etree._Element):
+            return(lxml.etree.tostring(cfg))
+        elif isinstance(cfg, bytes):
+            return(cfg)
+        else:
+            raise TypeError('Could not determine the object type.')
+        return(cfg)
+    
+    def _parse_regexes(self):
+        for regex in self.profile.xpath('//meta/regexes/pattern'):
+            self.btags['regex'][regex.attrib['id']] = re.compile(regex.text)
+        return()
+
+    def _parse_variables(self):
+        for variable in self.profile.xpath('//meta/variables/variable'):
+            self.btags['variable'][
+                                'variable%{0}'.format(variable.attrib['id'])
+                                    ] = variable.text
+        return()
+
+    def get_path(self, element):
+        path = element
+        try:
+            path = self.root.getpath(element)
+        except ValueError:
+            raise ValueError(
+                (
+                    'Could not find a path for the expression {0}'
+                ).format(element.text))
+        return(path)
+
+    def substitute(self, element, recurse_count = 0):
+        if recurse_count >= self.max_recurse:
+            return(element)
+        if isinstance(element, lxml.etree._Element):
+            if element.tag == 'regex':
+                return(element)
+            if isinstance(element, lxml.etree._Comment):
+                return(element)
+            if element.text:
+                _dictmap = self.btags_to_dict(element.text)
+                for elem in _dictmap:
+                    # This is needed because _dictmap gets replaced below
+                    if not _dictmap:
+                        return(element)
+                    _btag, _value = _dictmap[elem]
+                    if isinstance(_value, str):
+                        if _btag == 'xpath':
+                            try:
+                                newpath = element.xpath(_dictmap[elem][1])
+                            except (AttributeError, IndexError, TypeError):
+                                newpath = element
+                            except lxml.etree.XPathEvalError:
+                                return(element)
+                            try:
+                                self.btags['xpath'][elem] = self.substitute(
+                                            newpath, (recurse_count + 1))[0]
+                            except (IndexError, TypeError):
+                                raise ValueError(
+                                    ('Encountered an error while trying to '
+                                     'substitute {0} at {1}').format(
+                                        elem, self.get_path(element)
+                                    ))
+                        element.text = self.fmt.vformat(
+                                            element.text,   
+                                            [],
+                                            {**self.btags['xpath'],
+                                             **self.btags['variable']})
+                        _dictmap = self.btags_to_dict(element.text)
+        return(element)
+
+    def xpath_selector(self, selectors,
+                       selector_ids = ('id', 'name', 'uuid')):
+        # selectors is a dict of {attrib:value}
+        xpath = ''
+        for i in selectors.items():
+            if i[1] and i[0] in selector_ids:
+                xpath += '[@{0}="{1}"]'.format(*i)
+        return(xpath)
+
+    def btags_to_dict(self, text_in):
+        d = {}
+        ptrn_id = self.ptrn.findall(text_in)
+        if len(ptrn_id) >= 1:
+            for item in ptrn_id:
+                try:
+                    btag, expr = item.split('%', 1)
+                    if btag not in self.btags:
+                        continue
+                    if item not in self.btags[btag]:
+                        self.btags[btag][item] = None
+                    #self.btags[btag][item] = expr # remove me?
+                    if btag == 'xpath':
+                        d[item] = (btag, expr)
+                    elif btag == 'variable':
+                        d[item] = (btag, self.btags['variable'][item])
+                except ValueError:
+                    return(d)
+        return(d)
+
+

docs/examples/multi_profile.xml

@@ -23,13 +23,25 @@
       <!-- This is the VERY FIRST value parsed, and is required. It controls how many levels of {xpath%...} to recurse. -->
       <!-- If the maximum level is reached, the substitution will evaluate as blank. -->
       <max_recurse>5</max_recurse>
+      <!-- You need to store regex patterns here and reference them in a special way later, and it's only valid for certain
+                 items. See the manual for more information. -->
+      <regexes>
+        <pattern id="tarball_x86_64">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz$</pattern>
+        <pattern id="sig_x86_64">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz\.sig$</pattern>
+        <pattern id="tarball_i686">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-i686\.tar\.gz$</pattern>
+        <pattern id="sig_i686">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-i686\.tar\.gz\.sig$</pattern>
+      </regexes>
+      <!-- You can also define variables. NO xpath or regex btags, and they can't be used within other btags! -->
+      <variables>
+        <variable id="bdisk_root">/var/tmp/BDisk</variable>
+      </variables>
     </meta>
     <accounts>
       <!-- Salted/hashed password is "test" -->
       <rootpass hashed="yes">$6$7KfIdtHTcXwVrZAC$LZGNeMNz7v5o/cYuA48FAxtZynpIwO5B1CPGXnOW5kCTVpXVt4SypRqfM.AoKkFt/O7MZZ8ySXJmxpELKmdlF1</rootpass>
       <user sudo="yes">
         <username>{xpath%//meta/names/uxname/text()}</username>
-                <!-- You can also use substitution from different profiles: -->
+        <!-- You can also use substitution from different profiles in this same configuration: -->
         <!-- <username>{xpath%//profile[@name='another_profile']/meta/names/uxname"}</username> -->
         <comment>{xpath%//meta/dev/author/text()}</comment>
         <password hashed="no" hash_algo="sha512" salt="auto">testpassword</password>
@@ -43,43 +55,43 @@
     <sources>
       <source arch="x86_64">
         <mirror>http://archlinux.mirror.domain.tld</mirror>
-                <webroot>/iso/latest</webroot>
+        <rootpath>/iso/latest</rootpath>
-                <tarball flags="regex,latest">{xpath%../mirror/text()}{xpath%../webroot/text()}/{regex%archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz}</tarball>
+        <tarball flags="regex,latest">{regex%tarball_x86_64}</tarball>
-                <checksum hash_algo="sha1" flags="none">{xpath%../mirror/text()}{xpath%../webroot/text()}/sha1sums.txt</checksum>
+        <checksum hash_algo="sha1" flags="none">sha1sums.txt</checksum>
-                <sig keys="7F2D434B9741E8AC" keyserver="hkp://pool.sks-keyservers.net" flags="latest">{xpath%../tarball/text()}.sig</sig>
+        <sig keys="7F2D434B9741E8AC" keyserver="hkp://pool.sks-keyservers.net" flags="regex,latest">{regex%sig_x86_64}</sig>
       </source>
       <source arch="i686">
         <mirror>http://archlinux32.mirror.domain.tld</mirror>
-                <webroot>/iso/latest</webroot>
+        <rootpath>/iso/latest</rootpath>
-                <tarball flag="regex,latest">{xpath%../mirror/text()}/{xpath%../webroot/text()}/{regex%archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-i686\.tar\.gz}</tarball>
+        <tarball flags="regex,latest">{regex%tarball_i686}</tarball>
         <checksum hash_algo="sha512" explicit="yes">cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e</checksum>
-                <sig keys="248BF41F9BDD61D41D060AE774EDA3C6B06D0506" keyserver="hkp://pool.sks-keyservers.net">{xpath%../tarball/text()}.sig</sig>
+        <sig keys="248BF41F9BDD61D41D060AE774EDA3C6B06D0506" keyserver="hkp://pool.sks-keyservers.net" flags="regex,latest">{regex%sig_i686}</sig>
       </source>
     </sources>
     <build its_full_of_stars="yes">
       <paths>
-                <cache>/var/tmp/{xpath%//meta/names/uxname/text()}</cache>
+        <cache>{variable%bdisk_root}/cache</cache>
-                <chroot>/var/tmp/chroots/{xpath%//meta/names/uxname/text()}</chroot>
+        <chroot>{variable%bdisk_root}/chroots</chroot>
-                <overlay>{xpath%../cache/text()}/overlay</overlay>
+        <overlay>{variable%bdisk_root}/overlay</overlay>
-                <templates>~/{xpath%//meta/names/uxname/text()}/templates</templates>
+        <templates>{variable%bdisk_root}/templates</templates>
         <mount>/mnt/{xpath%//meta/names/uxname/text()}</mount>
-                <distros>~/{xpath%//meta/names/uxname/text()}/distros</distros>
+        <distros>{variable%bdisk_root}/distros</distros>
-                <dest>~/{xpath%//meta/names/uxname/text()}/results</dest>
+        <dest>{variable%bdisk_root}/results</dest>
-                <iso>{xpath%../dest/text()}/iso</iso>
+        <iso>{variable%bdisk_root}/iso_overlay</iso>
-                <http>{xpath%../dest/text()}/http</http>
+        <http>{variable%bdisk_root}/http</http>
-                <tftp>{xpath%../dest/text()}/tftp</tftp>
+        <tftp>{variable%bdisk_root}/tftp</tftp>
-                <pki>{xpath%../dest/text()}/pki</pki>
+        <pki>{variable%bdisk_root}/pki</pki>
       </paths>
       <basedistro>archlinux</basedistro>
     </build>
-        <iso sign="yes" multiarch="yes"/>
+    <iso sign="yes" multi_arch="yes"/>
     <ipxe sign="yes" iso="yes">
       <uri>{xpath%//meta/dev/website/text()}/ipxe</uri>
     </ipxe>
     <pki overwrite="no">
       <!-- http://ipxe.org/crypto -->
       <ca>
-                <cert>{xpath%../../../build/paths/pki/text()}/ca.crt</cert>
+        <cert hash_algo="sha512">{xpath%../../../build/paths/pki/text()}/ca.crt</cert>
         <!-- If csr is self-enclosed (<csr />), we'll just generate and use a CSR in-memory.
                      Assuming we need to generate a certificate, anyways.
                      If you want to write it out to disk (for debugging, etc.) OR use one already generated,
@@ -87,7 +99,16 @@
                      e.g.:
                         <csr>{xpath%build/paths/ssl/text()}/ca.csr</csr> -->
         <csr/>
-                <key des="no" passphrase="none">{xpath%../../../build/paths/pki/text()}/ca.key</key>
+        <!-- If you use an index file (or want to) to serialize client certificates, specify it here. -->
+        <!-- It must conform to CADB spec (https://pki-tutorial.readthedocs.io/en/latest/cadb.html). -->
+        <!-- You should probably also specify a serial file if so. -->
+        <!-- Both of these are entirely optional if you aren't using an existing PKI. -->
+        <index>{xpath%../../../build/paths/pki/text()}/index.txt</index>
+        <serial>{xpath%../../../build/paths/pki/text()}/serial</serial>
+        <!-- If you specify a cipher, the key will be encrypted to the passphrase provided by the passphrase attribute.
+                     If the key is encrypted (either a pre-existing or a created one) but passphrase is not provided, you will
+                     be (securely) prompted for the passphrase to unlock it/add a passphrase to it. -->
+        <key cipher="none" passphrase="none" keysize="4096">{xpath%../../../build/paths/pki/text()}/ca.key</key>
         <subject>
           <commonName>domain.tld</commonName>
           <countryName>XX</countryName>
@@ -99,9 +120,9 @@
         </subject>
       </ca>
       <client>
-                <cert>{xpath%../../../build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.crt</cert>
+        <cert hash_algo="sha512">{xpath%../../../build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.crt</cert>
         <csr/>
-                <key des="no" passphrase="none">{xpath%//build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.key</key>
+        <key cipher="none" passphrase="none" keysize="4096">{xpath%//build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.key</key>
         <subject>
           <commonName>some client name</commonName>
           <countryName>XX</countryName>
@@ -126,7 +147,7 @@
       </rsync>
     </sync>
   </profile>
-<profile name="alternate" id="2" uuid="2ed07c19-2071-4d66-8569-da40475ba716">
+  <profile name="alternate" id="2" uuid="2ed07c19-2071-4d66-8569-da40475ba716">
     <meta>
       <names>
         <name>AnotherCD</name>
@@ -136,12 +157,22 @@
       <desc>Another rescue/restore live environment.</desc>
       <dev>
         <author>Another Dev Eloper</author>
+        <!-- You can reference other profiles within the same configuration. -->
         <email>{xpath%//profile[@name="default"]/meta/dev/email/text()}</email>
         <website>{xpath%//profile[@name="default"]/meta/dev/website/text()}</website>
       </dev>
       <uri>https://domain.tld/projname</uri>
       <ver>0.0.1</ver>
       <max_recurse>5</max_recurse>
+      <regexes>
+        <pattern id="tarball_x86_64">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz$</pattern>
+        <pattern id="sig_x86_64">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz\.sig$</pattern>
+        <pattern id="tarball_i686">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-i686\.tar\.gz$</pattern>
+        <pattern id="sig_i686">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-i686\.tar\.gz\.sig$</pattern>
+      </regexes>
+      <variables>
+        <variable id="bdisk_root">/var/tmp/BDisk</variable>
+      </variables>
     </meta>
     <accounts>
       <rootpass hashed="no">atotallyinsecurepassword</rootpass>
@@ -154,44 +185,46 @@
     <sources>
       <source arch="x86_64">
         <mirror>http://archlinux.mirror.domain.tld</mirror>
-                <webroot>/iso/latest</webroot>
+        <rootpath>/iso/latest</rootpath>
-                <tarball flags="regex,latest">{xpath%../mirror/text()}{xpath%../webroot/text()}/{regex%archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz}</tarball>
+        <tarball flags="regex,latest">{regex%tarball_x86_64}</tarball>
-                <checksum hash_algo="sha1" flags="none">{xpath%../mirror/text()}{xpath%../webroot/text()}/sha1sums.txt</checksum>
+        <checksum hash_algo="sha1" flags="none">sha1sums.txt</checksum>
-                <sig keys="7F2D434B9741E8AC" keyserver="hkp://pool.sks-keyservers.net" flags="latest">{xpath%../tarball/text()}.sig</sig>
+        <sig keys="7F2D434B9741E8AC" keyserver="hkp://pool.sks-keyservers.net" flags="regex,latest">{regex%sig_x86_64}</sig>
       </source>
       <source arch="i686">
         <mirror>http://archlinux32.mirror.domain.tld</mirror>
-                <webroot>/iso/latest</webroot>
+        <rootpath>/iso/latest</rootpath>
-                <tarball flag="regex,latest">{xpath%../mirror/text()}/{xpath%../webroot/text()}/{regex%archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-i686\.tar\.gz}</tarball>
+        <tarball flags="regex,latest">{regex%tarball_i686}</tarball>
         <checksum hash_algo="sha512" explicit="yes">cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e</checksum>
-                <sig keys="248BF41F9BDD61D41D060AE774EDA3C6B06D0506" keyserver="hkp://pool.sks-keyservers.net">{xpath%../tarball/text()}.sig</sig>
+        <sig keys="248BF41F9BDD61D41D060AE774EDA3C6B06D0506" keyserver="hkp://pool.sks-keyservers.net" flags="regex,latest">{regex%sig_i686}</sig>
       </source>
     </sources>
     <build its_full_of_stars="yes">
       <paths>
-                <cache>/var/tmp/{xpath%//meta/names/uxname/text()}</cache>
+        <cache>{variable%bdisk_root}/cache</cache>
-                <chroot>/var/tmp/chroots/{xpath%//meta/names/uxname/text()}</chroot>
+        <chroot>{variable%bdisk_root}/chroots</chroot>
-                <overlay>{xpath%../cache/text()}/overlay</overlay>
+        <overlay>{variable%bdisk_root}/overlay</overlay>
-                <templates>~/{xpath%//meta/names/uxname/text()}/templates</templates>
+        <templates>{variable%bdisk_root}/templates</templates>
         <mount>/mnt/{xpath%//meta/names/uxname/text()}</mount>
-                <distros>~/{xpath%//meta/names/uxname/text()}/distros</distros>
+        <distros>{variable%bdisk_root}/distros</distros>
-                <dest>~/{xpath%//meta/names/uxname/text()}/results</dest>
+        <dest>{variable%bdisk_root}/results</dest>
-                <iso>{xpath%../dest/text()}/iso</iso>
+        <iso>{variable%bdisk_root}/iso_overlay</iso>
-                <http>{xpath%../dest/text()}/http</http>
+        <http>{variable%bdisk_root}/http</http>
-                <tftp>{xpath%../dest/text()}/tftp</tftp>
+        <tftp>{variable%bdisk_root}/tftp</tftp>
-                <pki>{xpath%../dest/text()}/pki</pki>
+        <pki>{variable%bdisk_root}/pki</pki>
       </paths>
       <basedistro>archlinux</basedistro>
     </build>
-        <iso sign="yes" multiarch="yes"/>
+    <iso sign="yes" multi_arch="yes"/>
     <ipxe sign="yes" iso="yes">
       <uri>{xpath%//meta/dev/website/text()}/ipxe</uri>
     </ipxe>
     <pki overwrite="no">
       <ca>
-                <cert>{xpath%../../../build/paths/pki/text()}/ca.crt</cert>
+        <cert hash_algo="sha512">{xpath%../../../build/paths/pki/text()}/ca.crt</cert>
         <csr/>
-                <key des="no" passphrase="none">{xpath%../../../build/paths/pki/text()}/ca.key</key>
+        <index>{xpath%../../../build/paths/pki/text()}/index.txt</index>
+        <serial>{xpath%../../../build/paths/pki/text()}/serial</serial>
+        <key cipher="none" passphrase="none" keysize="4096">{xpath%../../../build/paths/pki/text()}/ca.key</key>
         <subject>
           <commonName>domain.tld</commonName>
           <countryName>XX</countryName>
@@ -203,9 +236,9 @@
         </subject>
       </ca>
       <client>
-                <cert>{xpath%../../../build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.crt</cert>
+        <cert hash_algo="sha512">{xpath%../../../build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.crt</cert>
         <csr/>
-                <key des="no" passphrase="none">{xpath%//build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.key</key>
+        <key cipher="none" passphrase="none" keysize="4096">{xpath%//build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.key</key>
         <subject>
           <commonName>some client name</commonName>
           <countryName>XX</countryName>

docs/examples/regen_multi.py

@@ -3,8 +3,10 @@
 import copy
 from lxml import etree
 
+parser = etree.XMLParser(remove_blank_text = True)
+
 with open('single_profile.xml', 'rb') as f:
-    xml = etree.fromstring(f.read())
+    xml = etree.fromstring(f.read(), parser)
 
 single_profile = xml.xpath('/bdisk/profile[1]')[0]
 alt_profile = copy.deepcopy(single_profile)
@@ -45,6 +47,9 @@ for e in accounts.iter():
         e.attrib['sudo'] = 'no'
 # Delete the second user
 accounts.remove(accounts[2])
+author = alt_profile.xpath('/profile/meta/dev/author')[0]
+author.addnext(etree.Comment(
+    ' You can reference other profiles within the same configuration. '))
 xml.append(alt_profile)
 
 with open('multi_profile.xml', 'wb') as f:

docs/examples/single_profile.xml

@@ -23,13 +23,25 @@
             <!-- This is the VERY FIRST value parsed, and is required. It controls how many levels of {xpath%...} to recurse. -->
             <!-- If the maximum level is reached, the substitution will evaluate as blank. -->
             <max_recurse>5</max_recurse>
+            <!-- You need to store regex patterns here and reference them in a special way later, and it's only valid for certain
+                 items. See the manual for more information. -->
+            <regexes>
+                <pattern id="tarball_x86_64">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz$</pattern>
+                <pattern id="sig_x86_64">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz\.sig$</pattern>
+                <pattern id="tarball_i686">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-i686\.tar\.gz$</pattern>
+                <pattern id="sig_i686">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-i686\.tar\.gz\.sig$</pattern>
+            </regexes>
+            <!-- You can also define variables. NO xpath or regex btags, and they can't be used within other btags! -->
+            <variables>
+                <variable id="bdisk_root">/var/tmp/BDisk</variable>
+            </variables>
         </meta>
         <accounts>
             <!-- Salted/hashed password is "test" -->
             <rootpass hashed="yes">$6$7KfIdtHTcXwVrZAC$LZGNeMNz7v5o/cYuA48FAxtZynpIwO5B1CPGXnOW5kCTVpXVt4SypRqfM.AoKkFt/O7MZZ8ySXJmxpELKmdlF1</rootpass>
             <user sudo="yes">
                 <username>{xpath%//meta/names/uxname/text()}</username>
-                <!-- You can also use substitution from different profiles: -->
+                <!-- You can also use substitution from different profiles in this same configuration: -->
                 <!-- <username>{xpath%//profile[@name='another_profile']/meta/names/uxname"}</username> -->
                 <comment>{xpath%//meta/dev/author/text()}</comment>
                 <password hashed="no"
@@ -47,48 +59,47 @@
         <sources>
             <source arch="x86_64">
                 <mirror>http://archlinux.mirror.domain.tld</mirror>
-                <webroot>/iso/latest</webroot>
+                <rootpath>/iso/latest</rootpath>
-                <tarball flags="regex,latest">{xpath%../mirror/text()}{xpath%../webroot/text()}/{regex%archlinux-bootstrap-[0-9]{{4}}\.[0-9]{{2}}\.[0-9]{{2}}-x86_64\.tar\.gz}</tarball>
+                <tarball flags="regex,latest">{regex%tarball_x86_64}</tarball>
-                <!-- <tarball flags="regex,latest">{xpath%../mirror/text()}{xpath%../webroot/text()}/{regex%archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz}</tarball> -->
+                <checksum hash_algo="sha1" flags="none">sha1sums.txt</checksum>
-                <checksum hash_algo="sha1" flags="none" >{xpath%../mirror/text()}{xpath%../webroot/text()}/sha1sums.txt</checksum>
                 <sig keys="7F2D434B9741E8AC"    
                      keyserver="hkp://pool.sks-keyservers.net"
-                     flags="latest">{xpath%../tarball/text()}.sig</sig>
+                     flags="regex,latest">{regex%sig_x86_64}</sig>
             </source>
             <source arch="i686">
                 <mirror>http://archlinux32.mirror.domain.tld</mirror>
-                <webroot>/iso/latest</webroot>
+                <rootpath>/iso/latest</rootpath>
-                <tarball flag="regex,latest">{xpath%../mirror/text()}/{xpath%../webroot/text()}/{regex%archlinux-bootstrap-[0-9]{{4}}\.[0-9]{{2}}\.[0-9]{{2}}-i686\.tar\.gz}</tarball>
+                <tarball flags="regex,latest">{regex%tarball_i686}</tarball>
-                <!-- <tarball flag="regex,latest">{xpath%../mirror/text()}/{xpath%../webroot/text()}/{regex%archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-i686\.tar\.gz}</tarball> -->
                 <checksum hash_algo="sha512" explicit="yes">cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e</checksum>
                 <sig keys="248BF41F9BDD61D41D060AE774EDA3C6B06D0506"
-                     keyserver="hkp://pool.sks-keyservers.net">{xpath%../tarball/text()}.sig</sig>
+                     keyserver="hkp://pool.sks-keyservers.net"
+                     flags="regex,latest">{regex%sig_i686}</sig>
             </source>
         </sources>
         <build its_full_of_stars="yes">
             <paths>
-                <cache>/var/tmp/{xpath%//meta/names/uxname/text()}</cache>
+                <cache>{variable%bdisk_root}/cache</cache>
-                <chroot>/var/tmp/chroots/{xpath%//meta/names/uxname/text()}</chroot>
+                <chroot>{variable%bdisk_root}/chroots</chroot>
-                <overlay>{xpath%../cache/text()}/overlay</overlay>
+                <overlay>{variable%bdisk_root}/overlay</overlay>
-                <templates>~/{xpath%//meta/names/uxname/text()}/templates</templates>
+                <templates>{variable%bdisk_root}/templates</templates>
                 <mount>/mnt/{xpath%//meta/names/uxname/text()}</mount>
-                <distros>~/{xpath%//meta/names/uxname/text()}/distros</distros>
+                <distros>{variable%bdisk_root}/distros</distros>
-                <dest>~/{xpath%//meta/names/uxname/text()}/results</dest>
+                <dest>{variable%bdisk_root}/results</dest>
-                <iso>{xpath%../dest/text()}/iso</iso>
+                <iso>{variable%bdisk_root}/iso_overlay</iso>
-                <http>{xpath%../dest/text()}/http</http>
+                <http>{variable%bdisk_root}/http</http>
-                <tftp>{xpath%../dest/text()}/tftp</tftp>
+                <tftp>{variable%bdisk_root}/tftp</tftp>
-                <pki>{xpath%../dest/text()}/pki</pki>
+                <pki>{variable%bdisk_root}/pki</pki>
             </paths>
             <basedistro>archlinux</basedistro>
         </build>
-        <iso sign="yes" multiarch="yes" />
+        <iso sign="yes" multi_arch="yes" />
         <ipxe sign="yes" iso="yes">
             <uri>{xpath%//meta/dev/website/text()}/ipxe</uri>
         </ipxe>
         <pki overwrite="no">
             <!-- http://ipxe.org/crypto -->
             <ca>
-                <cert>{xpath%../../../build/paths/pki/text()}/ca.crt</cert>
+                <cert hash_algo="sha512">{xpath%../../../build/paths/pki/text()}/ca.crt</cert>
                 <!-- If csr is self-enclosed (<csr />), we'll just generate and use a CSR in-memory.
                      Assuming we need to generate a certificate, anyways.
                      If you want to write it out to disk (for debugging, etc.) OR use one already generated,
@@ -96,7 +107,18 @@
                      e.g.:
                         <csr>{xpath%build/paths/ssl/text()}/ca.csr</csr> -->
                 <csr />
-                <key des="no" passphrase="none">{xpath%../../../build/paths/pki/text()}/ca.key</key>
+                <!-- If you use an index file (or want to) to serialize client certificates, specify it here. -->
+                <!-- It must conform to CADB spec (https://pki-tutorial.readthedocs.io/en/latest/cadb.html). -->
+                <!-- You should probably also specify a serial file if so. -->
+                <!-- Both of these are entirely optional if you aren't using an existing PKI. -->
+                <index>{xpath%../../../build/paths/pki/text()}/index.txt</index>
+                <serial>{xpath%../../../build/paths/pki/text()}/serial</serial>
+                <!-- If you specify a cipher, the key will be encrypted to the passphrase provided by the passphrase attribute.
+                     If the key is encrypted (either a pre-existing or a created one) but passphrase is not provided, you will
+                     be (securely) prompted for the passphrase to unlock it/add a passphrase to it. -->
+                <key cipher="none"
+                     passphrase="none"
+                     keysize="4096">{xpath%../../../build/paths/pki/text()}/ca.key</key>
                 <subject>
                     <commonName>domain.tld</commonName>
                     <countryName>XX</countryName>
@@ -108,9 +130,11 @@
                 </subject>
             </ca>
             <client>
-                <cert>{xpath%../../../build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.crt</cert>
+                <cert hash_algo="sha512">{xpath%../../../build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.crt</cert>
                 <csr />
-                <key des="no" passphrase="none">{xpath%//build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.key</key>
+                <key cipher="none"
+                     passphrase="none"
+                     keysize="4096">{xpath%//build/paths/pki/text()}/{xpath%../../../meta/names/uxname/text()}.key</key>
                 <subject>
                     <commonName>some client name</commonName>
                     <countryName>XX</countryName>