checking in while i do some research

bdisk/bGPG.py

@@ -0,0 +1,134 @@
+import os
+import subprocess
+import datetime
+import jinja2
+import gpgme
+import psutil
+
+def genGPG(conf):
+    # https://media.readthedocs.org/pdf/pygpgme/latest/pygpgme.pdf
+    build = conf['build']
+    gpghome = conf['gpg']['mygpghome']
+    distkey = build['gpgkey']
+    templates_dir = '{0}/extra/templates'.format(build['basedir'])
+    mykey = False
+    pkeys = []
+    if conf['gpg']['mygpgkey'] != '':
+        mykey = conf['gpg']['mygpgkey']
+        if gpghome == '':
+            # Let's try the default.
+            gpghome = '{0}/.gnupg'.format(os.path.expanduser("~"))
+    else:
+        # No key ID was specified.
+        if gpghome == '':
+            # We'll generate a key if we can't find one here.
+            gpghome = build['dlpath'] + '/.gnupg'
+    os.environ['GNUPGHOME'] = gpghome
+    gpg = gpgme.Context()
+    if mykey:
+        try:
+            privkey = gpg.get_key(mykey, True)
+        except:
+            exit('{0}: ERROR: You specified using {1} but we have no secret key for that ID!'.format(
+                                                        datetime.datetime.now(),
+                                                        mykey))
+    else:
+        for key in gpg.keylist(None,True):
+            if key.can_sign:
+                pkeys.append(key)
+                break
+            #for subkey in key.subkeys:  # for parsing each and every subkey- this should be unnecessary
+                #if subkey.can_sign:
+                    #pkeys.append(gpg.get_key(subkey.fpr))
+        if len(pkeys) == 0:
+            print("{0}: [GPG] Generating a GPG key...".format(datetime.datetime.now()))
+            loader = jinja2.FileSystemLoader(templates_dir)
+            env = jinja2.Environment(loader = loader)
+            tpl = env.get_template('GPG.j2')
+            tpl_out = tpl.render(build = build, bdisk = bdisk)
+            privkey = gpg.get_key(gpg.genkey(tpl_out).fpr, True)
+            pkeys.append(privkey)
+    # Now we try to find and add the key for the base image.
+    gpg.keylist_mode = 2  # remote (keyserver)
+    try:
+        key = gpg.get_key(distkey)
+    except:
+        exit('{0}: ERROR: We cannot find key ID {1}!'.format(
+                                    datetime.datetime.now(),
+                                    distkey))
+    importkey = key.subkeys[0].fpr
+    gpg.keylist_mode = 1 # local keyring (default)
+    DEVNULL = open(os.devnull, 'w')
+    cmd = ['/usr/bin/gpg',
+            '--recv-keys',
+            '--batch',
+            '--yes',
+            '0x{0}'.format(importkey)]
+    subprocess.call(cmd, stdout = DEVNULL, stderr = subprocess.STDOUT)
+    sigkeys = []
+    for k in gpg.get_key(importkey).subkeys:
+        signkeys.append(k.fpr)
+
+
+    # RETURNS:
+    # our private/signing keys: privkey (is a list)
+    
+
+def killStaleAgent():
+    # Kill off any stale GPG agents running.
+    # Probably not even needed, but good to have.
+    procs = psutil.process_iter()
+    plst = []
+    for p in procs:
+        if (p.name() == 'gpg-agent' and p.uids()[0] == os.getuid()):
+            pd = psutil.Process(p.pid).as_dict()
+            if pd['cwd'] != '/':
+                plst.append(p.pid)
+    if len(plst) >= 1:
+        for p in plst:
+            psutil.Process(p).terminate()
+
+def signIMG(path, conf):
+    if conf['build']['gpg']:
+        # If we enabled GPG signing, we need to figure out if we
+        # are using a personal key or the automatically generated one.
+        if conf['gpg']['mygpghome'] != '':
+            gpghome = conf['gpg']['mygpghome']
+        else:
+            gpghome = conf['build']['dlpath'] + '/.gnupg'
+        if conf['gpg']['mygpgkey'] != '':
+            keyid = conf['gpg']['mygpgkey']
+        else:
+            keyid = False
+        # We want to kill off any stale gpg-agents so we spawn a new one.
+        killStaleAgent()
+        ## HERE BE DRAGONS. Converting to PyGPGME...
+        # List of Key instances used for signing with sign() and encrypt_sign().
+        gpg = gpgme.Context()
+        if keyid:
+            gpg.signers = gpg.get_key(keyid)
+        else:
+            # Try to "guess" the key ID.
+            # If we got here, it means we generated a key earlier during the tarball download...
+            # So we can use that!
+            pass
+        # And if we didn't specify one manually, we'll pick the first one we find.
+        # This way we can use the automatically generated one from prep.
+        if not keyid:
+            keyid = gpg.list_keys(True)[0]['keyid']
+        print('{0}: [BUILD] Signing {1} with {2}...'.format(
+                                        datetime.datetime.now(),
+                                        path,
+                                        keyid))
+        # TODO: remove this warning when upstream python-gnupg fixes
+        print('\t\t\t    If you see a "ValueError: Unknown status message: \'KEY_CONSIDERED\'" error, ' +
+                'it can be safely ignored.')
+        print('\t\t\t    If this is taking a VERY LONG time, try installing haveged and starting it. ' +
+                'This can be done safely in parallel with the build process.')
+        data_in = open(path, 'rb')
+        gpg.sign_file(data_in, keyid = keyid, detach = True,
+                            clearsign = False, output = '{0}.sig'.format(path))
+        data_in.close()
+
+def gpgVerify(sigfile, datafile, conf):
+    pass

bdisk/build.py

@@ -4,7 +4,7 @@ import shutil
 import glob
 import subprocess
 import hashlib
-import gnupg
+import psutil
 import jinja2
 import humanize
 import datetime
@@ -392,37 +392,6 @@ def genISO(conf):
     iso['Main']['fmt'] = 'Hybrid ISO'
     return(iso)
 
-def signIMG(path, conf):
-    if conf['build']['gpg']:
-        # If we enabled GPG signing, we need to figure out if we
-        # are using a personal key or the automatically generated one.
-        if conf['gpg']['mygpghome'] != '':
-            gpghome = conf['gpg']['mygpghome']
-        else:
-            gpghome = conf['build']['dlpath'] + '/.gnupg'
-        if conf['gpg']['mygpgkey'] != '':
-            keyid = conf['gpg']['mygpgkey']
-        else:
-            keyid = False
-        gpg = gnupg.GPG(gnupghome = gpghome, use_agent = True)
-        # And if we didn't specify one manually, we'll pick the first one we find.
-        # This way we can use the automatically generated one from prep.
-        if not keyid:
-            keyid = gpg.list_keys(True)[0]['keyid']
-        print('{0}: [BUILD] Signing {1} with {2}...'.format(
-                                        datetime.datetime.now(),
-                                        path,
-                                        keyid))
-        # TODO: remove this warning when upstream python-gnupg fixes
-        print('\t\t\t    If you see a "ValueError: Unknown status message: \'KEY_CONSIDERED\'" error, ' +
-                'it can be safely ignored.')
-        print('\t\t\t    If this is taking a VERY LONG time, try installing haveged and starting it. ' +
-                'This can be done safely in parallel with the build process.')
-        data_in = open(path, 'rb')
-        gpg.sign_file(data_in, keyid = keyid, detach = True,
-                            clearsign = False, output = '{0}.sig'.format(path))
-        data_in.close()
-
 def displayStats(iso):
     for i in iso['name']:
         print("{0}: == {1} {2} ==".format(datetime.datetime.now(), iso[i]['type'], iso[i]['fmt']))

bdisk/prep.py

@@ -2,11 +2,10 @@ import os
 import shutil
 import re
 import hashlib
-import gnupg
+import gpgme
 import tarfile
 import subprocess
 import re
-#import git
 import jinja2
 import datetime
 import humanize
@@ -45,16 +44,6 @@ def downloadTarball(build):
     if build['mirrorgpgsig'] != '':
         # we don't want to futz with the user's normal gpg.
         gpg = gnupg.GPG(gnupghome = dlpath + '/.gnupg')
-        print("{0}: [PREP] Generating a GPG key...".format(datetime.datetime.now()))
-        # python-gnupg 0.3.9 spits this error in Arch. it's harmless, but ugly af.
-        # TODO: remove this when the error doesn't happen anymore.
-        print("\t\t\t    If you see a \"ValueError: Unknown status message: 'KEY_CONSIDERED'\" error, it can be safely ignored.")
-        print("\t\t\t    If this is taking a VERY LONG time, try installing haveged and starting it. This can be " +
-                        "done safely in parallel with the build process.\n")
-        input_data = gpg.gen_key_input(name_email = 'tempuser@nodomain.tld', passphrase = 'placeholder_passphrase')
-        key = gpg.gen_key(input_data)  # this gives the "error"
-        keyid = build['gpgkey']  # this gives the "error" as well
-        gpg.recv_keys(build['gpgkeyserver'], keyid)
     for a in arch:
         pattern = re.compile('^.*' + a + '\.tar(\.(gz|bz2|xz))?$')
         tarball = [filename.group(0) for l in list(sha_dict.keys()) for filename in [pattern.search(l)] if filename][0]
@@ -89,24 +78,18 @@ def downloadTarball(build):
             if build['mirrorgpgsig'] == '.sig':
                 gpgsig_remote = rlsdir + tarball + '.sig'
             else:
-                gpgsig_remote = mirror + build['mirrorgpgsig']
+                gpgsig_remote = build['mirrorgpgsig']
-            gpg_sig = tarball + '.sig'
             sig_dl = urlopen(gpgsig_remote)
             sig = tarball_path[a] + '.sig'
             with open(sig, 'wb+') as f:
                 f.write(sig_dl.read())
             sig_dl.close()
-            tarball_data = open(tarball_path[a], 'rb')
+            gpg_verify = bGPG.gpgVerify(sig, tarball_path[a], conf)
-            tarball_data_in = tarball_data.read()
-            gpg_verify = gpg.verify_data(sig, tarball_data_in)
-            tarball_data.close()
             if not gpg_verify:
                 exit("{0}: There was a failure checking {1} against {2}. Please investigate.".format(
                                                                     datetime.datetime.now(),
                                                                     sig,
                                                                     tarball_path[a]))
-            os.remove(sig)
-
     return(tarball_path)
 
 def unpackTarball(tarball_path, build, keep = False):

docs/TODO

@@ -6,6 +6,7 @@
 -sizes of build iso files
 -GPG sigs on built files
 -fix the branding, etc. on ipxe. :(
+-convert docs to asciidoc, turn up instead of RTD (https://github.com/rtfd/readthedocs.org/issues/17#issuecomment-3752702)
 
 ## General ##
 

extra/dist.build.ini

@@ -178,10 +178,11 @@ mirrorchksum = ${mirrorpath}sha1sums.txt
 ; If you specify just '.sig' (or use the default
 ; and don't actually specify a mirrorfile),
 ; we'll try to guess based on the file from the sha1
-; checksums.
+; checksums. Note that this must evaluate to a full
+; URL (e.g.:
+;	${mirrorproto}://${mirror}${mirrorpath}somefile.sig)
 ; 0.) No whitespace (if specified)
 ; 1.) Must be the full path
-; 2.) Don't include the mirror domain or protocol
 mirrorgpgsig = ${mirrorfile}.sig
 
 ; What is a valid key ID that should be used to
@@ -296,8 +297,9 @@ i_am_a_racecar = yes
 ; What is a valid key ID that we should use to
 ; *sign* our release files?
 ; 0.) You will be prompted for a passphrase if your
-;     key has one/you don't have an open gpg-agent
+;     key has one/you don't have an open and authorized
-;     session.
+;     gpg-agent session. Make sure you have a working
+;     pinentry configuration set up!
 ; 1.) If you leave this blank we will use the key
 ;     we generate automatically earlier in the build
 ;     process.

extra/templates/GPG.j2

@@ -0,0 +1,14 @@
+{# For more options, see https://www.gnupg.org/documentation/manuals/gnupg/Unattended-GPG-key-generation.html
+#}<GnupgKeyParms format="internal">
+    Key-Type: RSA
+    Key-Length: 4096
+    Subkey-Type: RSA
+    Subkey-Length: 4096
+    Name-Real: {{ bdisk['dev'] }}
+    Name-Email: {{ bdisk['email'] }}
+    Name-Comment: via {{ bdisk['pname'] }} [autogenerated] | {{ bdisk['uri'] }} | {{ bdisk['desc'] }}
+    Expire-Date: 0
+    %no-ask-passphrase
+    %no-protection
+    %commit
+</GnupgKeyParms>