checking in...

TODO

@@ -1,24 +1,23 @@
 - write classes/functions
 - XML-based config
+-x XML syntax
+--- regex btags - case-insensitive? this can be represented in-pattern:
+    https://stackoverflow.com/a/9655186/733214
+-x configuration generator
+--- print end result xml config to stderr for easier redirection? or print prompts to stderr and xml to stdout?
+-- XSD for validation
+-- Flask app for generating config?
+-- TKinter (or pygame?) GUI?
+--- https://docs.python.org/3/faq/gui.html
+--- https://www.pygame.org/wiki/gui
 - ensure we use docstrings in a Sphinx-compatible manner?
   https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html
   at the very least document all the functions and such so pydoc's happy.
-- better prompt display. i might include them as constants in a separate file
-  and then import it for e.g. confgen. or maybe a Flask website/app?
+
 - locking
 - for docs, 3.x (as of 3.10) was 2.4M.
-- GUI? at least for generating config...
 - Need ability to write/parse mtree specs (or a similar equivalent) for applying ownerships/permissions to overlay files
 
-- SSL key gen:
-import OpenSSL
-k = OpenSSL.crypto.PKey()
-k.generate_key(OpenSSL.crypto.TYPE_RSA, 4096)
-x = OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM,
-                                   k,
-                                   cipher = 'aes256',
-                                   passphrase = 'test')
-
 - need to package:
   python-hashid (https://psypanda.github.io/hashID/,
                  https://github.com/psypanda/hashID,
@@ -41,4 +40,4 @@ BUGS.SQUARE-R00T.NET bugs/tasks:
 #36: Allow parsing pkg lists with inline comments
 #39: Fix UEFI
 #40: ISO overlay (to add e.g. memtest86+ to final ISO)
-#43: Support resuming partial tarball downloads (Accet-Ranges: bytes)
+#43: Support resuming partial tarball downloads (Accept-Ranges: bytes)

bdisk/SSL.py

@@ -3,3 +3,10 @@ import OpenSSL
 # migrate old functions of bSSL to use cryptography
 # but still waiting on their recpipes.
 # https://cryptography.io/en/latest/x509/tutorial/
+#import OpenSSL
+#k = OpenSSL.crypto.PKey()
+#k.generate_key(OpenSSL.crypto.TYPE_RSA, 4096)
+#x = OpenSSL.crypto.dump_privatekey(OpenSSL.crypto.FILETYPE_PEM,
+#                                   k,
+#                                   cipher = 'aes256',
+#                                   passphrase = 'test')

bdisk/confgen.py

@@ -442,6 +442,8 @@ class ConfGenerator(object):
             tarball_elem.attrib['flags'] = 'latest'
             tarball_elem.text = tarball['full_url']
             print('\n++ SOURCES || {0} || CHECKSUM ++'.format(arch.upper()))
+            # TODO: explicit not being set for explicitly-set sums,
+            # and checksum string not actually added to those. investigate.
             chksum = lxml.etree.SubElement(source, 'checksum')
             _chksum_chk = prompt.confirm_or_no(prompt = (
                 '\nWould you like to add a checksum for the tarball? (BDisk '
@@ -502,7 +504,7 @@ class ConfGenerator(object):
                             print('Invalid selection. Starting over.')
                             continue
                     else:
-                        checksum_type == checksum_type[0]
+                        checksum_type = checksum_type[0]
                         chksum.attrib['explicit'] = "yes"
                         chksum.text = checksum
                     chksum.attrib['hash_algo'] = checksum_type

bdisk/confparse.py

@@ -1,15 +1,18 @@
-import copy
-import re
 import os
+import pprint
+import re
 import utils
-import validators
 import lxml.etree
 from urllib.parse import urlparse
 
 etree = lxml.etree
+detect = utils.detect()
+generate = utils.generate()
+transform = utils.transform()
+valid = utils.valid()
 
 class Conf(object):
-    def __init__(self, cfg, profile = None):
+    def __init__(self, cfg, profile = None, validate = False):
         """
         A configuration object.
 
@@ -37,27 +40,70 @@ class Conf(object):
                         You can provide any combination of these
                         (e.g. "profile={'id': 2, 'name' = 'some_profile'}").
         """
-        #self.raw = _detect_cfg(cfg)  # no longer needed; in utils
         self.xml_suppl = utils.xml_supplicant(cfg, profile = profile)
-        self.profile = self.xml_suppl
-        self.xml = None
-        self.profile = None
-        # Mad props to https://stackoverflow.com/a/12728199/733214
-        self.xpath_re = re.compile('(?<=(?<!\{)\{)[^{}]*(?=\}(?!\}))')
-        self.substitutions = {}
-        self.xpaths = ['xpath']
-        try:
-            self.xml = etree.fromstring(self.raw)
-        except lxml.etree.XMLSyntaxError:
-            raise ValueError('The configuration provided does not seem to be '
-                             'valid')
+        self.xml = self.xml_suppl.xml
+        for e in self.xml_suppl.xml.iter():
+            self.xml_suppl.substitute(e)
+        self.xml_suppl.get_profile(profile = self.xml_suppl.orig_profile)
+        with open('/tmp/parsed.xml', 'wb') as f:
+            f.write(lxml.etree.tostring(self.xml_suppl.xml))
+        self.profile = self.xml_suppl.profile
         self.xsd = None
-        #if not self.validate():  # Need to write the XSD
-        #    raise ValueError('The configuration did not pass XSD/schema '
-        #                     'validation')
-        self.get_profile()
-        self.max_recurse = int(self.profile.xpath('//meta/'
-                                                  'max_recurse')[0].text)
+        self.cfg = {}
+        #if validate:
+            #if not self.validate():  # Need to write the XSD
+            #    raise ValueError('The configuration did not pass XSD/schema '
+            #                     'validation')
+
+    def get_source(self, source, item, _source):
+        _source_item = {'flags': [],
+                         'fname': None}
+        elem = source.xpath('./{0}'.format(item))[0]
+        if item == 'checksum':
+            if elem.get('explicit', False):
+                _explicit = transform.xml2py(
+                        elem.attrib['explicit'])
+                _source_item['explicit'] = _explicit
+                if _explicit:
+                    del(_source_item['fname'])
+                    _source_item['value'] = elem.text
+                    return(_source_item)
+            else:
+                _source_item['explicit'] = False
+            if elem.get('hash_algo', False):
+                _source_item['hash_algo'] = elem.attrib['hash_algo']
+            else:
+                _source_item['hash_algo'] = None
+        if item == 'sig':
+            if elem.get('keys', False):
+                _keys = [i.strip() for i in elem.attrib['keys'].split(',')]
+                _source_item['keys'] = _keys
+            else:
+                _source_item['keys'] = []
+            if elem.get('keyserver', False):
+                _source_item['keyserver'] = elem.attrib['keyserver']
+            else:
+                _source_item['keyserver'] = None
+        _item = elem.text
+        _flags = elem.get('flags', [])
+        if _flags:
+            for f in _flags.split(','):
+                if f.strip().lower() == 'none':
+                    continue
+                _source_item['flags'].append(f.strip().lower())
+        if _source_item['flags']:
+            if 'regex' in _source_item['flags']:
+                ptrn = _item.format(**self.xml_suppl.btags['regex'])
+            else:
+                ptrn = None
+            _source_item['fname'] = detect.remote_files(
+                    '/'.join((_source['mirror'],
+                              _source['rootpath'])),
+                    ptrn = ptrn,
+                    flags = _source_item['flags'])
+        else:
+            _source_item['fname'] = _item
+        return(_source_item)
 
     def get_xsd(self):
         path = os.path.join(os.path.dirname(__file__),
@@ -66,62 +112,116 @@ class Conf(object):
             xsd = f.read()
         return(xsd)
 
+    def parse_accounts(self):
+        ## PROFILE/ACCOUNTS
+        self.cfg['users'] = []
+        # First we handle the root user, since it's a "special" case.
+        _root = self.profile.xpath('./accounts/rootpass')
+        self.cfg['root'] = transform.user(_root)
+        for user in self.profile.xpath('./accounts/user'):
+            _user = {'username': user.xpath('./username/text()')[0],
+                     'sudo': transform.xml2py(user.attrib['sudo']),
+                     'comment': None}
+            _comment = user.xpath('./comment/text()')
+            if len(_comment):
+                _user['comment'] = _comment[0]
+            _password = user.xpath('./password')
+            _user.update(transform.user(_password))
+            self.cfg['users'].append(_user)
+        return()
+
+    def parse_all(self):
+        self.parse_profile()
+        self.parse_meta()
+        self.parse_accounts()
+        self.parse_sources()
+        self.parse_buildpaths()
+        self.parse_pki()
+        return()
+
+    def parse_buildpaths(self):
+        ## PROFILE/BUILD(/PATHS)
+        self.cfg['build'] = {'paths': {}}
+        build = self.profile.xpath('./build')[0]
+        _optimize = build.get('its_full_of_stars', 'no')
+        self.cfg['build']['optimize'] = transform.xml2py(_optimize)
+        for path in build.xpath('./paths/*'):
+            self.cfg['build']['paths'][path.tag] = path.text
+        self.cfg['build']['basedistro'] = build.get('basedistro', 'archlinux')
+        # iso and ipxe are their own basic profile elements, but we group them
+        # in here because 1.) they're related, and 2.) they're simple to
+        # import. This may change in the future if they become more complex.
+        ## PROFILE/ISO
+        self.cfg['iso'] = {'sign': None,
+                           'multi_arch': None}
+        self.cfg['ipxe'] = {'sign': None,
+                            'iso': None}
+        for x in ('iso', 'ipxe'):
+            # We enable all features by default.
+            elem = self.profile.xpath('./{0}'.format(x))[0]
+            for a in self.cfg[x]:
+                self.cfg[x][a] = transform.xml2py(elem.get(a, 'yes'))
+            if x == 'ipxe':
+                self.cfg[x]['uri'] = elem.xpath('./uri/text()')[0]
+        return()
+
+    def parse_meta(self):
+        ## PROFILE/META
+        # Get the various meta strings. We skip regexes (we handle those
+        # separately since they're unique'd per id attrib) and variables (they
+        # are already substituted by self.xml_suppl.substitute(x)).
+        _meta_iters = ('dev', 'names')
+        for t in _meta_iters:
+            self.cfg[t] = {}
+            _xpath = './meta/{0}'.format(t)
+            for e in self.profile.xpath(_xpath):
+                for se in e:
+                    if not isinstance(se, lxml.etree._Comment):
+                        self.cfg[t][se.tag] = se.text
+        for e in ('desc', 'uri', 'ver', 'max_recurse'):
+            _xpath = './meta/{0}/text()'.format(e)
+            self.cfg[e] = self.profile.xpath(_xpath)[0]
+        # HERE is where we would handle regex patterns.
+        # But we don't, because they're in self.xml_suppl.btags['regex'].
+        #self.cfg['regexes'] = {}
+        #_regexes = self.profile.xpath('./meta/regexes/pattern')
+        #if len(_regexes):
+        #    for ptrn in _regexes:
+        #        self.cfg['regexes'][ptrn.attrib['id']] = re.compile(ptrn.text)
+        return()
+
+    def parse_pki(self):
+        self.cfg['pki'] = {'ca': {},
+                           'client': []}
+        elem = self.profile.xpath('./pki')[0]
+        self.cfg['pki']['overwrite'] =
+
+    def parse_profile(self):
+        ## PROFILE
+        # The following are attributes of profiles that serve as identifiers.
+        self.cfg['profile'] = {'id': None,
+                               'name': None,
+                               'uuid': None}
+        for a in self.cfg['profile']:
+            if a in self.profile.attrib:
+                self.cfg['profile'][a] = self.profile.attrib[a]
+        return()
+
+    def parse_sources(self):
+        ## PROFILE/SOURCES
+        self.cfg['sources'] = []
+        for source in self.profile.xpath('./sources/source'):
+            _source = {}
+            _source['arch'] = source.attrib['arch']
+            _source['mirror'] = source.xpath('./mirror/text()')[0]
+            _source['rootpath'] = source.xpath('./rootpath/text()')[0]
+            # The tarball, checksum, and sig components requires some...
+            # special care.
+            for e in ('tarball', 'checksum', 'sig'):
+                _source[e] = self.get_source(source, e, _source)
+            self.cfg['sources'].append(_source)
+        return()
+
     def validate(self):
         self.xsd = etree.XMLSchema(self.get_xsd())
         return(self.xsd.validate(self.xml))
-
-    def get_profile(self):
-        """Get a configuration profile.
-
-        Get a configuration profile from the XML object and set that as a
-        profile object. If a profile is specified, attempt to find it. If not,
-        follow the default rules as specified in __init__.
-        """
-        if self.profile:
-            # A profile identifier was provided
-            if isinstance(self.profile, str):
-                _profile_name = self.profile
-                self.profile = {}
-                for i in _profile_specifiers:
-                    self.profile[i] = None
-                self.profile['name'] = _profile_name
-            elif isinstance(self.profile, dict):
-                for k in _profile_specifiers:
-                    if k not in self.profile.keys():
-                        self.profile[k] = None
-            else:
-                raise TypeError('profile must be a string (name of profile), '
-                                'a dictionary, or None')
-            xpath = ('/bdisk/'
-                     'profile{0}').format(_profile_xpath_gen(self.profile))
-            self.profile = self.xml.xpath(xpath)
-            if not self.profile:
-                raise RuntimeError('Could not find the profile specified in '
-                                   'the given configuration')
-        else:
-            # We need to find the default.
-            profiles = []
-            for p in self.xml.xpath('/bdisk/profile'):
-                profiles.append(p)
-            # Look for one named "default" or "DEFAULT" etc.
-            for idx, value in enumerate([e.attrib['name'].lower() \
-                                         for e in profiles]):
-                if value == 'default':
-                    self.profile = copy.deepcopy(profiles[idx])
-                    break
-            # We couldn't find a profile with a default name. Try to grab the
-            # first profile.
-            if self.profile is None:
-                # Grab the first profile.
-                if profiles:
-                    self.profile = profile[0]
-                else:
-                    # No profiles found.
-                    raise RuntimeError('Could not find any usable '
-                                       'configuration profiles')
-        return()
-
-    def parse_profile(self):
-        pass
-
-

bdisk/utils.py

@@ -15,6 +15,7 @@ import uuid
 import validators
 import zlib
 import lxml.etree
+from bs4 import BeautifulSoup
 from collections import OrderedDict
 from dns import resolver
 from email.utils import parseaddr as emailparse
@@ -28,6 +29,7 @@ passlib_schemes = ['des_crypt', 'md5_crypt', 'sha256_crypt', 'sha512_crypt']
 # Build various hash digest name lists
 digest_schemes = list(hashlib.algorithms_available)
 # Provided by zlib
+# TODO
 digest_schemes.append('adler32')
 digest_schemes.append('crc32')
 
@@ -36,8 +38,6 @@ crypt_map = {'sha512': crypt.METHOD_SHA512,
              'md5': crypt.METHOD_MD5,
              'des': crypt.METHOD_CRYPT}
 
-
-
 class XPathFmt(string.Formatter):
     def get_field(self, field_name, args, kwargs):
         vals = self.get_value(field_name, args, kwargs), field_name
@@ -74,6 +74,41 @@ class detect(object):
             return(None)
         return()
 
+    def password_hash_salt(self, salted_hash):
+        _hash_list = salted_hash.split('$')
+        salt = _hash_list[2]
+        return(salt)
+
+    def remote_files(self, url_base, ptrn = None, flags = []):
+        with urlopen(url_base) as u:
+            soup = BeautifulSoup(u.read(), 'lxml')
+        urls = []
+        if 'regex' in flags:
+            if not isinstance(ptrn, str):
+                raise ValueError('"ptrn" must be a regex pattern to match '
+                                 'against')
+            else:
+                ptrn = re.compile(ptrn)
+        for u in soup.find_all('a'):
+            if not u.has_attr('href'):
+                continue
+            if 'regex' in flags:
+                if not ptrn.search(u.attrs['href']):
+                    continue
+            if u.has_attr('href'):
+                urls.append(u.attrs['href'])
+        if not urls:
+            return(None)
+        # We certainly can't intelligently parse the printed timestamp since it
+        # varies so much and that'd be a nightmare to get consistent...
+        # But we CAN sort by filename.
+        if 'latest' in flags:
+            urls = sorted(list(set(urls)))
+            urls = urls[-1]
+        else:
+            urls = urls[0]
+        return(urls)
+
     def gpgkeyID_from_url(self, url):
         with urlopen(url) as u:
             data = u.read()
@@ -144,6 +179,9 @@ class generate(object):
             _salt = crypt.mksalt(algo)
         else:
             _salt = salt
+        if not password:
+            # Intentionally empty password.
+            return('')
         return(crypt.crypt(password, _salt))
 
     def hashlib_names(self):
@@ -154,7 +192,9 @@ class generate(object):
                 hashes.append(h)
         return(hashes)
 
-    def salt(self, algo = 'sha512'):
+    def salt(self, algo = None):
+        if not algo:
+            algo = 'sha512'
         algo = crypt_map[algo]
         return(crypt.mksalt(algo))
 
@@ -574,6 +614,66 @@ class transform(object):
             url['full_url'] += '#{0}'.format('#'.join(_f))
         return(url)
 
+    def user(self, user_elem):
+        _attribs = ('hashed', 'hash_algo', 'salt')
+        acct = {}
+        for a in _attribs:
+            acct[a] = None
+        if len(user_elem):
+            elem = user_elem[0]
+            for a in _attribs:
+                if a in elem.attrib:
+                    acct[a] = self.xml2py(elem.attrib[a], attrib = True)
+            if acct['hashed']:
+                if not acct['hash_algo']:
+                    _hash = detect().password_hash(elem.text)
+                    if _hash:
+                        acct['hash_algo'] = _hash
+                    else:
+                        raise ValueError(
+                                'Invalid salted password hash: {0}'.format(
+                                        elem.text)
+                                )
+                acct['salt_hash'] = elem.text
+                acct['passphrase'] = None
+            else:
+                if not acct['hash_algo']:
+                    acct['hash_algo'] = 'sha512'
+                acct['passphrase'] = elem.text
+            _saltre = re.compile('^\s*(auto|none|)\s*$', re.IGNORECASE)
+            if acct['salt']:
+                if _saltre.search(acct['salt']):
+                    _salt = generate.salt(acct['hash_algo'])
+                    acct['salt'] = _salt
+            else:
+                if not acct['hashed']:
+                    acct['salt_hash'] = generate().hash_password(
+                                        acct['passphrase'],
+                                        algo = crypt_map[acct['hash_algo']])
+                acct['salt'] = detect().password_hash_salt(acct['salt_hash'])
+        if 'salt_hash' not in acct:
+            acct['salt_hash'] = generate().hash_password(
+                                        acct['passphrase'],
+                                        salt = acct['salt'],
+                                        algo = crypt_map[acct['hash_algo']])
+        return(acct)
+
+    def xml2py(self, value, attrib = True):
+        yes = re.compile('^\s*(y(es)?|true|1)\s*$', re.IGNORECASE)
+        no = re.compile('^\s*(no?|false|0)\s*$', re.IGNORECASE)
+        if no.search(value):
+            if attrib:
+                return(False)
+            else:
+                return(None)
+        elif yes.search(value):
+            # We handle the False case above.
+            return(True)
+        elif value.strip() == '':
+            return(None)
+        else:
+            return(value)
+
 class valid(object):
     def __init__(self):
         pass
@@ -705,22 +805,32 @@ class valid(object):
 
 class xml_supplicant(object):
     def __init__(self, cfg, profile = None, max_recurse = 5):
-        raw = self._detect_cfg(cfg)
-        xmlroot = lxml.etree.fromstring(raw)
+        self.selector_ids = ('id', 'name', 'uuid')
         self.btags = {'xpath': {},
                       'regex': {},
                       'variable': {}}
+        raw = self._detect_cfg(cfg)
+        # This is changed in just a moment.
+        self.profile = profile
+        # This is retained so we can "refresh" the profile if needed.
+        self.orig_profile = profile
+        try:
+            self.xml = lxml.etree.fromstring(raw)
+        except lxml.etree.XMLSyntaxError:
+            raise ValueError('The configuration provided does not seem to be '
+                             'valid')
+        self.get_profile(profile = profile)
+        self.xml = lxml.etree.fromstring(raw)
         self.fmt = XPathFmt()
-        self.max_recurse = max_recurse
+        self.max_recurse = int(self.profile.xpath(
+                                        '//meta/max_recurse/text()')[0])
         # I don't have permission to credit them, but to the person who helped
         # me with this regex - thank you. You know who you are.
+        # Originally this pattern was the one from:
+        # https://stackoverflow.com/a/12728199/733214
         self.ptrn = re.compile(('(?<=(?<!\{)\{)(?:[^{}]+'
                                 '|{{[^{}]*}})*(?=\}(?!\}))'))
-        self.root = lxml.etree.ElementTree(xmlroot)
-        if not profile:
-            self.profile = xmlroot.xpath('/bdisk/profile[1]')[0]
-        else:
-            self.profile = xmlroot.xpath(profile)[0]
+        self.root = lxml.etree.ElementTree(self.xml)
         self._parse_regexes()
         self._parse_variables()
         
@@ -754,7 +864,8 @@ class xml_supplicant(object):
 
     def _parse_regexes(self):
         for regex in self.profile.xpath('//meta/regexes/pattern'):
-            self.btags['regex'][regex.attrib['id']] = re.compile(regex.text)
+            _key = 'regex%{0}'.format(regex.attrib['id'])
+            self.btags['regex'][_key] = regex.text
         return()
 
     def _parse_variables(self):
@@ -764,6 +875,85 @@ class xml_supplicant(object):
                                     ] = variable.text
         return()
 
+    def btags_to_dict(self, text_in):
+        d = {}
+        ptrn_id = self.ptrn.findall(text_in)
+        if len(ptrn_id) >= 1:
+            for item in ptrn_id:
+                try:
+                    btag, expr = item.split('%', 1)
+                    if btag not in self.btags:
+                        continue
+                    if item not in self.btags[btag]:
+                        self.btags[btag][item] = None
+                    #self.btags[btag][item] = expr # remove me?
+                    if btag == 'xpath':
+                        d[item] = (btag, expr)
+                    elif btag == 'variable':
+                        d[item] = (btag, self.btags['variable'][item])
+                except ValueError:
+                    return(d)
+        return(d)
+
+    def get_profile(self, profile = None):
+        """Get a configuration profile.
+
+        Get a configuration profile from the XML object and set that as a
+        profile object. If a profile is specified, attempt to find it. If not,
+        follow the default rules as specified in __init__.
+        """
+        if profile:
+            # A profile identifier was provided
+            if isinstance(profile, str):
+                _profile_name = profile
+                profile = {}
+                for i in self.selector_ids:
+                    profile[i] = None
+                profile['name'] = _profile_name
+            elif isinstance(profile, dict):
+                for k in self.selector_ids:
+                    if k not in profile.keys():
+                        profile[k] = None
+            else:
+                raise TypeError('profile must be a string (name of profile), '
+                                'a dictionary, or None')
+            xpath = '/bdisk/profile{0}'.format(self.xpath_selector(profile))
+            self.profile = self.xml.xpath(xpath)
+            if len(self.profile) != 1:
+                raise ValueError('Could not determine a valid, unique '
+                                 'profile; please check your profile '
+                                 'specifier(s)')
+            else:
+                # We need the actual *profile*, not a list of matching
+                # profile(s).
+                self.profile = self.profile[0]
+            if not len(self.profile):
+                raise RuntimeError('Could not find the profile specified in '
+                                   'the given configuration')
+        else:
+            # We need to find the default.
+            profiles = []
+            for p in self.xml.xpath('/bdisk/profile'):
+                profiles.append(p)
+            # Look for one named "default" or "DEFAULT" etc.
+            for idx, value in enumerate([e.attrib['name'].lower() \
+                                         for e in profiles]):
+                if value == 'default':
+                    #self.profile = copy.deepcopy(profiles[idx])
+                    self.profile = profiles[idx]
+                    break
+            # We couldn't find a profile with a default name. Try to grab the
+            # first profile.
+            if self.profile is None:
+                # Grab the first profile.
+                if profiles:
+                    self.profile = profiles[0]
+                else:
+                    # No profiles found.
+                    raise RuntimeError('Could not find any usable '
+                                       'configuration profiles')
+        return()
+
     def get_path(self, element):
         path = element
         try:
@@ -815,33 +1005,10 @@ class xml_supplicant(object):
                         _dictmap = self.btags_to_dict(element.text)
         return(element)
 
-    def xpath_selector(self, selectors,
-                       selector_ids = ('id', 'name', 'uuid')):
+    def xpath_selector(self, selectors):
         # selectors is a dict of {attrib:value}
         xpath = ''
         for i in selectors.items():
-            if i[1] and i[0] in selector_ids:
+            if i[1] and i[0] in self.selector_ids:
                 xpath += '[@{0}="{1}"]'.format(*i)
         return(xpath)
-
-    def btags_to_dict(self, text_in):
-        d = {}
-        ptrn_id = self.ptrn.findall(text_in)
-        if len(ptrn_id) >= 1:
-            for item in ptrn_id:
-                try:
-                    btag, expr = item.split('%', 1)
-                    if btag not in self.btags:
-                        continue
-                    if item not in self.btags[btag]:
-                        self.btags[btag][item] = None
-                    #self.btags[btag][item] = expr # remove me?
-                    if btag == 'xpath':
-                        d[item] = (btag, expr)
-                    elif btag == 'variable':
-                        d[item] = (btag, self.btags['variable'][item])
-                except ValueError:
-                    return(d)
-        return(d)
-
-

docs/examples/multi_profile.xml

@@ -9,7 +9,7 @@
                      but now with the neat benefits of XPath! Everything you could do in build.ini's and more.
                      See https://www.w3schools.com/xml/xpath_syntax.asp
                      If you need a literal curly brace, double them (e.g. for "{foo}", use "{{foo}}"),
-                     UNLESS it's in a {regex%...} placeholder/filter (as part of the expression). -->
+                     UNLESS it's in a <regexes><pattern> as part of the expression. Those are taken as literal strings.  -->
         <pname>{xpath%../name/text()}</pname>
       </names>
       <desc>A rescue/restore live environment.</desc>
@@ -24,7 +24,7 @@
       <!-- If the maximum level is reached, the substitution will evaluate as blank. -->
       <max_recurse>5</max_recurse>
       <!-- You need to store regex patterns here and reference them in a special way later, and it's only valid for certain
-                 items. See the manual for more information. -->
+                 items. See the manual for more information. NO btags within the patterns is allowed. -->
       <regexes>
         <pattern id="tarball_x86_64">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz$</pattern>
         <pattern id="sig_x86_64">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz\.sig$</pattern>
@@ -139,10 +139,10 @@
              blank passphrase for all operations. -->
     <gpg keyid="none" gnupghome="none" publish="no" prompt_passphrase="no">
       <!-- The below is only used if we are generating a key (i.e. keyid="none"). -->
-      <key type="rsa" keysize="4096" expire="0">
-        <name>{xpath%../../../../meta/dev/author/text()}</name>
-        <email>{xpath%../../../../meta/dev/email/text()}</email>
-        <comment>for {xpath%../../../../meta/names/pname/text()} [autogenerated] | {xpath%../../../../meta/uri/text()} | {xpath%../../../../meta/desc/text()}</comment>
+      <key algo="rsa" keysize="4096" expire="0">
+        <name>{xpath%../../../meta/dev/author/text()}</name>
+        <email>{xpath%../../../meta/dev/email/text()}</email>
+        <comment>for {xpath%../../../meta/names/pname/text()} [autogenerated] | {xpath%../../../meta/uri/text()} | {xpath%../../../meta/desc/text()}</comment>
       </key>
     </gpg>
     <sync>
@@ -263,10 +263,10 @@
       </client>
     </pki>
     <gpg keyid="none" gnupghome="none" publish="no" prompt_passphrase="no">
-      <key type="rsa" keysize="4096" expire="0">
-        <name>{xpath%../../../../meta/dev/author/text()}</name>
-        <email>{xpath%../../../../meta/dev/email/text()}</email>
-        <comment>for {xpath%../../../../meta/names/pname/text()} [autogenerated] | {xpath%../../../../meta/uri/text()} | {xpath%../../../../meta/desc/text()}</comment>
+      <key algo="rsa" keysize="4096" expire="0">
+        <name>{xpath%../../../meta/dev/author/text()}</name>
+        <email>{xpath%../../../meta/dev/email/text()}</email>
+        <comment>for {xpath%../../../meta/names/pname/text()} [autogenerated] | {xpath%../../../meta/uri/text()} | {xpath%../../../meta/desc/text()}</comment>
       </key>
     </gpg>
     <sync>

docs/examples/single_profile.xml

@@ -9,7 +9,7 @@
                      but now with the neat benefits of XPath! Everything you could do in build.ini's and more.
                      See https://www.w3schools.com/xml/xpath_syntax.asp
                      If you need a literal curly brace, double them (e.g. for "{foo}", use "{{foo}}"),
-                     UNLESS it's in a {regex%...} placeholder/filter (as part of the expression). -->
+                     UNLESS it's in a <regexes><pattern> as part of the expression. Those are taken as literal strings.  -->
                 <pname>{xpath%../name/text()}</pname>
             </names>
             <desc>A rescue/restore live environment.</desc>
@@ -24,7 +24,7 @@
             <!-- If the maximum level is reached, the substitution will evaluate as blank. -->
             <max_recurse>5</max_recurse>
             <!-- You need to store regex patterns here and reference them in a special way later, and it's only valid for certain
-                 items. See the manual for more information. -->
+                 items. See the manual for more information. NO btags within the patterns is allowed. -->
             <regexes>
                 <pattern id="tarball_x86_64">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz$</pattern>
                 <pattern id="sig_x86_64">archlinux-bootstrap-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64\.tar\.gz\.sig$</pattern>
@@ -62,7 +62,7 @@
                 <rootpath>/iso/latest</rootpath>
                 <tarball flags="regex,latest">{regex%tarball_x86_64}</tarball>
                 <checksum hash_algo="sha1"
-                          flags="none">sha1sums.txt</checksum>
+                          explicit="no">sha1sums.txt</checksum>
                 <sig keys="7F2D434B9741E8AC"    
                      keyserver="hkp://pool.sks-keyservers.net"
                      flags="regex,latest">{regex%sig_x86_64}</sig>
@@ -157,9 +157,9 @@
              prompt_passphrase="no">
             <!-- The below is only used if we are generating a key (i.e. keyid="none"). -->
             <key algo="rsa" keysize="4096" expire="0">
-                <name>{xpath%../../../../meta/dev/author/text()}</name>
-                <email>{xpath%../../../../meta/dev/email/text()}</email>
-                <comment>for {xpath%../../../../meta/names/pname/text()} [autogenerated] | {xpath%../../../../meta/uri/text()} | {xpath%../../../../meta/desc/text()}</comment>
+                <name>{xpath%../../../meta/dev/author/text()}</name>
+                <email>{xpath%../../../meta/dev/email/text()}</email>
+                <comment>for {xpath%../../../meta/names/pname/text()} [autogenerated] | {xpath%../../../meta/uri/text()} | {xpath%../../../meta/desc/text()}</comment>
             </key>
         </gpg>
         <sync>

docs/manual/TODO

@@ -8,3 +8,6 @@
 - in faq/ISOBIG.adoc and the doc section it references, make sure we reference that the package lists are now in the environment plugin!
 
 - change all references to build.ini to something like "BDisk configuration file"
+
+- reminder: users can specify a local file source for <sources><source> items by using "file:///absolute/path/to/file"
+-- todo: add http auth, ftp, ftps