fixing package list

.gitignore

@@ -1,6 +1,7 @@
 # We don't want local build settings in case someone's using
 # the git dir as a place to store their build.ini
 /build.ini
+/build.ini.*
 /dist.build.ini
 *.bak
 

bdisk/bdisk.py

@@ -1,4 +1,6 @@
 #!/bin/env python3
+
+import argparse
 import host
 import prep
 import bchroot
@@ -10,15 +12,15 @@ import bsync
 import bGPG
 import os
 
-# we need to:
+
-# we also need to figure out how to implement "mentos" (old bdisk) like functionality, letting us reuse an existing chroot install if possible to save time for future builds.
+def bdisk(args):
-#   if not, though, it's no big deal.
+    # we also need to figure out how to implement "mentos" (old bdisk) like functionality, letting us reuse an
-# still on the todo: iPXE
+    # existing chroot install if possible to save time for future builds.
-if __name__ == '__main__':
+    # if not, though, it's no big deal.
     if os.getuid() != 0:
         exit('{0}: ERROR: BDisk *must* be run as the root user or with sudo!'.format(datetime.datetime.now()))
     print('{0}: Starting.'.format(datetime.datetime.now()))
-    conf = host.parseConfig(host.getConfig())[1]
+    conf = host.parseConfig(host.getConfig(conf_file = args['buildini']))[1]
     prep.dirChk(conf)
     conf['gpgobj'] = bGPG.genGPG(conf)
     prep.buildChroot(conf, keep = False)
@@ -50,3 +52,20 @@ if __name__ == '__main__':
     bsync.git(conf)
     bsync.rsync(conf)
     print('{0}: Finish.'.format(datetime.datetime.now()))
+
+def parseArgs():
+    args = argparse.ArgumentParser(description = 'BDisk - a tool for building live/rescue media.',
+                                   epilog = 'brent s. || 2017 || https://bdisk.square-r00t.net')
+    args.add_argument('buildini',
+                      metavar = '/path/to/build.ini',
+                      default = '/etc/bdisk/build.ini',
+                      nargs = '?',
+                      help = 'The full/absolute path to the build.ini to use for this run. The default is /etc/bdisk/build.ini, but see https://bdisk.square-r00t.net/#the_code_build_ini_code_file.')
+    return(args)
+
+def main():
+    args = vars(parseArgs().parse_args())
+    bdisk(args)
+
+if __name__ == '__main__':
+    main()

bdisk/bsync.py

@@ -32,7 +32,7 @@ def http(conf):
                                                     datetime.datetime.now(),
                                                     httpdir))
         for a in arch:
-            for i in ('md5', 'sfs', 'sha256'):
+            for i in ('md5', 'sfs', 'sha256', 'sha512'):
                 httpfiles['{0}/{1}/airootfs.{2}'.format(bdisk['name'], a, i)] = '{0}/{1}/airootfs.{2}'.format(bdisk['name'], a, i)
         httpfiles['VERSION_INFO.txt'] = 'VERSION_INFO.txt'
         if 'x86_64' in arch:
@@ -81,7 +81,7 @@ def tftp(conf):
                                                 datetime.datetime.now(),
                                                 tftpdir))
         for a in arch:
-            for i in ('md5', 'sfs', 'sha256'):
+            for i in ('md5', 'sfs', 'sha256', 'sha512'):
                 tftpfiles['{0}/{1}/airootfs.{2}'.format(bdisk['name'], a, i)] = '{0}/{1}/airootfs.{2}'.format(bdisk['name'], a, i)
         tftpfiles['VERSION_INFO.txt'] = 'VERSION_INFO.txt'
         if 'x86_64' in arch:

bdisk/build.py

@@ -21,6 +21,7 @@ def genImg(conf):
     basedir = build['basedir']
     prepdir = build['prepdir']
     hashes = {}
+    hashes['sha512'] = {}
     hashes['sha256'] = {}
     hashes['md5'] = {}
     squashfses = []
@@ -52,9 +53,10 @@ def genImg(conf):
                                             humanize.naturalsize(
                                                 os.path.getsize(squashimg))))
         # Generate the checksum files
-        print("{0}: [BUILD] Generating SHA256, MD5 checksums ({1})...".format(
+        print("{0}: [BUILD] Generating SHA512 SHA256, MD5 checksums ({1})...".format(
-                                                datetime.datetime.now(),
+                                                                datetime.datetime.now(),
                                                 squashimg))
+        hashes['sha512'][a] = hashlib.sha512()
         hashes['sha256'][a] = hashlib.sha256()
         hashes['md5'][a] = hashlib.md5()
         with open(squashimg, 'rb') as f:
@@ -63,8 +65,11 @@ def genImg(conf):
                 if not stream:
                     break
                 # NOTE: these items are hashlib objects, NOT strings!
+                hashes['sha512'][a].update(stream)
                 hashes['sha256'][a].update(stream)
                 hashes['md5'][a].update(stream)
+        with open(airoot + 'airootfs.sha512', 'w+') as f:
+            f.write("{0}  airootfs.sfs\n".format(hashes['sha512'][a].hexdigest()))
         with open(airoot + 'airootfs.sha256', 'w+') as f:
             f.write("{0}  airootfs.sfs\n".format(hashes['sha256'][a].hexdigest()))
         with open(airoot + 'airootfs.md5', 'w+') as f:

bdisk/ipxe.py

@@ -21,7 +21,8 @@ def buildIPXE(conf):
     srcdir = build['srcdir']
     embedscript = build['dlpath'] + '/EMBED'
     ipxe_src = srcdir + '/ipxe'
-    ipxe_git_uri = 'git://git.ipxe.org/ipxe.git'
+    #ipxe_git_uri = 'git://git.ipxe.org/ipxe.git'
+    ipxe_git_uri = 'http://git.ipxe.org/ipxe.git'
     print('{0}: [IPXE] Prep/fetch sources...'.format(
                                         datetime.datetime.now()))
     # Get the source

docs/TODO

@@ -1,6 +1,11 @@
 -investigate weird signing issue- if you specify a key to sign with, it appears that the squashed images (at least in the http dir) doesn't have a sig/asc. do they need to copy it over? or is it not even signing them?
+-switch from python-pygpgme to python-gpgme for better performance. also clean up bGPG in general; reference KANT.
+-more pythonic! classes (because inits help), use list or tuple constant for checksums, try vars-ing the configparser stuff (and move defaults to in-code?),
+   change path combinations to use os.path.join etc.
+-modularity: https://stackoverflow.com/a/8719100
+
+-mtree-like functionality; if mtree spec is found, apply that to files in overlay (or chroot even); otherwise copy from overlay and don't touch chroot 
 
-## Missing v2.x functionality ##
 -i_am_a_racecar optimizations
 - different distro guests (debian, etc.)- https://stackoverflow.com/questions/2349991/python-how-to-import-other-python-files/20749411#20749411
 -incorporate this into the manual?
@@ -27,6 +32,8 @@
 --iPXE's curl
 --initrd's curl
 -WISH: Better logging/debugging
+https://web.archive.org/web/20170726052946/http://www.lexev.org/en/2013/python-logging-every-day/
+
 -WISH: signing for secureboot releases (PreLoader and loader.efi handle this okay, but require manual intervention)
 -does loader.efi support splash backgrounds? can i implement that differently somehow?
 --yes, see e.g. https://www.reddit.com/r/archlinux/comments/3bwgf0/where_put_the_splasharchbmp_to_splash_screen_boot/

docs/manual/HEAD.adoc

@@ -1,6 +1,6 @@
 = BDisk User and Developer Manual
 Brent Saner <bts@square-r00t.net>
-v1.2, 2017-05-11
+v1.3, 2017-08-20
 :doctype: book
 :data-uri:
 :imagesdir: images

docs/manual/user/BUILDING.adoc

@@ -7,6 +7,8 @@ NOTE: Due to requiring various mounting and chrooting, BDisk must be run as the
 
 To initiate a build, simply run `<basedir>/bdisk/bdisk.py`. That's it! Everything should continue automatically.
 
+If you'd like to specify a path to a specific build configuration, you can use `<basedir>/bdisk/bdisk.py path/to/build.ini`. The default is _/etc/bdisk/build.ini_ (plus <<the_code_build_ini_code_,other locations>>).
+
 If you're using a packaged version you installed from your distro's package manager, you instead should run wherever it installs to. Most likely this is going to be `/usr/sbin/bdisk`. (On systemd build hosts that have done the https://www.freedesktop.org/wiki/Software/systemd/TheCaseForTheUsrMerge/[/usr merge^], you can use `/usr/sbin/bdisk` or `/sbin/bdisk`.)
 
 If you encounter any issues during the process, make sure you read the documentation -- if your issue still isn't addressed, please be sure to file a <<bug_reports_feature_requests,bug report>>!

docs/manual/user/BUILDINI.adoc

@@ -3,7 +3,7 @@ This file is where you can specify some of the very basics of BDisk building. It
 
 It's single-level, but divided into "sections". This is unfortunately a limitation of ConfigParser, but it should be easy enough to follow.
 
-Blank lines are ignored, as well as any lines beginning with `#` and `;`. There are some restrictions and recommendations for some values, so be sure to note them when they occur. Variables referencing other values in the `build.ini` are allowed in the format of `${value}` if it's in the same section; otherwise, `${section:value}` can be used.
+Blank lines are ignored, as well as any lines beginning with `#` and `;`. There are some restrictions and recommendations for some values, so be sure to note them when they occur. Variables referencing other values in the `build.ini` are allowed in the format of `${keyname}` if it's in the same section; otherwise, `${section:keyname}` can be used.
 
 If you want to use your own `build.ini` file (and you should!), the following paths are searched in order. The first one found will be used.
 
@@ -35,6 +35,7 @@ We'll go into more detail for each section below.
  [user]
  username = ${bdisk:uxname}
  name = Default user
+ groups = ${bdisk:uxname},admin
  password = $$6$$t92Uvm1ETLocDb1D$$BvI0Sa6CSXxzIKBinIaJHb1gLJWheoXp7WzdideAJN46aChFu3hKg07QaIJNk4dfIJ2ry3tEfo3FRvstKWasg/
  [source_x86_64]
  mirror = mirror.us.leaseweb.net
@@ -192,6 +193,15 @@ What comment/description/real name should be used for the user? For more informa
 
 . ASCII only
 
+==== `groups`
+What groups this user should be added to, comma-separated. They will be created if they don't exist yet. Standard *nix group names rules apply:
+
+. ASCII only
+. 32 characters or less
+. Can only contain lower-case letters, numeric digits, underscores, or dashes (and can end with a dollar sign)
+. Must start with a (lower-case) letter or underscore
+. No whitespace
+
 ==== `password`
 The escaped, salted, hashed string to use for the non-root user.
 

docs/manual/user/GETTING_STARTED.adoc

@@ -64,6 +64,8 @@ While not strictly necessary, these will greatly enhance your BDisk usage. I've
 
 NOTE: If you do not wish to install any of these or cannot install them, be sure to disable the relevant options in the `build.ini` file (we'll talk about that later). The default `extra/dist.build.ini` should be sane enough to not require any of these.
 
+* http://cdrtools.sourceforge.net/private/cdrecord.html[cdrtools^]
+** Needed for building iPXE.
 * http://gcc.gnu.org[gcc (multilib)^] (>=6.x)
 ** Needed for building iPXE.
 * http://gcc.gnu.org[gcc-libs (multilib)^] (>=6.x)

docs/manual/user/advanced/SSH.adoc

@@ -17,7 +17,7 @@ First, create a file: `<basedir>/overlay/etc/ssh/sshd_config` using the followin
  Subsystem	sftp	/usr/lib/ssh/sftp-server
  KexAlgorithms curve25519-sha256@libssh.org,diffie-hellman-group-exchange-sha256
  Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr
- MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-ripemd160-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-512,hmac-sha2-256,hmac-ripemd160,umac-128@openssh.com
+ MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-512,hmac-sha2-256,umac-128@openssh.com
 
 We'll also want to implement a more secure `ssh_config` file to avoid possible leaks. The following is `<basedir>/overlay/etc/ssh/ssh_config`:
 
@@ -28,7 +28,7 @@ We'll also want to implement a more secure `ssh_config` file to avoid possible l
      PubkeyAuthentication yes
      HostKeyAlgorithms ssh-ed25519-cert-v01@openssh.com,ssh-rsa-cert-v01@openssh.com,ssh-ed25519,ssh-rsa
      Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr
-     MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-ripemd160-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-512,hmac-sha2-256,hmac-ripemd160,umac-128@openssh.com
+     MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-512,hmac-sha2-256,umac-128@openssh.com
 
 We'll want to create our own moduli. This can take a long time, but only needs to be done once -- it doesn't need to be done for every build. The following commands should be run in `<basedir>/overlay/etc/ssh/`:
 

extra/dist.build.ini

@@ -26,6 +26,7 @@ user = yes
 [user]
 username = ${bdisk:uxname}
 name = Default user
+groups = ${bdisk:uxname},admin
 password =
 
 [source_x86_64]

extra/pkg.build.ini

@@ -26,6 +26,7 @@ user = yes
 [user]
 username = ${bdisk:uxname}
 name = Default user
+groups = ${bdisk:uxname},admin
 password =
 
 [source_x86_64]

extra/pre-build.d/etc/systemd/scripts/livecd.fix.sh

@@ -1,5 +1,7 @@
 #/bin/sh -
 
+locale-gen
+
 #chmod 4755 /opt/google/chrome-beta/chrome-sandbox
 chmod 4755 /usr/bin/sudo
 
@@ -20,7 +22,7 @@ do
 done
 
 function fuck_you_gimme_net() {
-IFACE=$(ifconfig -a -s | egrep -E '^((en|wl)p?|em)' | awk '{print $1}' | tr '\n' ' ' | sed -e 's/\ $//g')
+IFACE=$(ip -o link | awk '{print $2}' | egrep -E '^((en|wl)p?|em)' | sed -e 's/:$//g' | head -n1)
 for i in ${IFACE};
 do
 
@@ -39,15 +41,15 @@ do
    DEV='wireless-open'
   fi
 
- ifconfig ${i} down
+ ip link set dev ${i} down
  cp -a /etc/netctl/examples/${DEV} /etc/netctl/${i}
  sed -i -re "s/^([[:space:]]*Interface[[:space:]]*=).*/\1${i}/g" /etc/netctl/${i}
  if [ "${DEV}" == "wireless-open" ];
  then
-  ifconfig ${i} up && \
+  ip link set dev ${i} up && \
   ESSID=$(iwlist ${i} scanning | egrep -A5 -B5 '^[[:space:]]*Encryption key:off' | egrep '^[[:space:]]*ESSID:' | sed -re 's/^[[:space:]]*ESSID:(.*)/\1/g')
   sed -i -re "s/^([[:space:]]*ESSID[[:space:]]*=).*/\1${ESSID}/g" /etc/netctl/${i}
-  ifconfig ${i} down
+  ip link set ${i} down
  fi
  netctl restart ${i} > /dev/null 2>&1
  #cat /etc/resolvconf.conf.failover > /etc/resolvconf.conf

extra/pre-build.d/root/packages.both

@@ -14,13 +14,15 @@ atop
 autopsy
 autossh
 backuppc
-#bacula ## TODO: grab all the bacula packages in here
+## TODO: grab all the bacula packages in here
+#bacula 
 beep
 bin86
 bind-tools
 binutils
 bluez-utils
-bonnie++ ## TODO: ugh. apacman has a new "regex mode"... that you can't disable. https://github.com/oshazard/apacman/issues/79
+## TODO: ugh. apacman has a new "regex mode"... that you can't disable. https://github.com/oshazard/apacman/issues/79
+#bonnie++ 
 boxbackup-client
 boxbackup-server
 bozocrack-git
@@ -55,13 +57,15 @@ dd_rescue
 dd_rhelp
 debianutils
 debootstrap
-#dialog #giving a weird dependency issue
+## giving a weird dependency issue
+#dialog 
 diffutils
 djohn
 dmidecode
 dnssec-anchors
 dnstracer
-#dnsutils #replaced by bind-tools, https://www.archlinux.org/packages/extra/x86_64/bind-tools/
+## replaced by bind-tools, https://www.archlinux.org/packages/extra/x86_64/bind-tools/
+#dnsutils 
 dos2unix
 dropbear
 dstat
@@ -145,14 +149,17 @@ keyutils
 kismet-allplugins
 lftp
 links
-#logkeys-git # requires a /dev/input, which apparently isn't included in the chroots
+## requires a /dev/input, which apparently isn't included in the chroots
+#logkeys-git 
 lm_sensors
 lrzsz
 lshw
-#lsiutil # giving intermittent errors when trying to fetch source
+## giving intermittent errors when trying to fetch source
+#lsiutil 
 lsof
 lsscsi
-#lxde # apacman currently doesn't like package groups, so...
+## apacman currently doesn't like package groups, so...
+#lxde 
 gpicview
 lxappearance
 lxappearance-obconf
@@ -173,18 +180,20 @@ pcmanfm
 lynx
 #lzip
 macchanger
-#magicrescue # no longer maintained, upstream down
+## no longer maintained, upstream down
+#magicrescue 
 mbr
 mbuffer
 mcelog
 mdadm
 mdcrack
-# superseded by storcli
+## superseded by storcli
 #megaraid-cli
 memtester
 mfoc
 minicom
-#mondo # mindi-busybox fails to build 09.23.2016
+## mindi-busybox fails to build 09.23.2016
+#mondo 
 mtd-utils
 mtr
 mtree-git
@@ -198,13 +207,14 @@ net-snmp
 netselect
 nettle
 networkmanager-pptp
-nginx-devel
+nginx-mainline
 ngrep
 nmap
 nmon
 ntfs-3g
 ntfsfixboot
-#nwipe #broken since they moved to github(?)
+## broken since they moved to github(?)
+#nwipe 
 nwipe-git
 obnam
 open-iscsi
@@ -273,7 +283,8 @@ smartmontools
 smbclient
 s-nail
 socat
-#star ## do people even USE tape backups anymore?
+## do people even USE tape backups anymore?
+#star 
 storcli
 strace
 stress
@@ -317,7 +328,8 @@ vncrack
 vnstat
 vpnc
 weplab
-#whdd #currently depends on dialog, which is broke as shit
+## currently depends on dialog, which is broke as shit
+#whdd 
 whois
 wifite-mod-pixiewps-git
 wipe

extra/pre-build.d/root/packages.both.FULL

@@ -14,13 +14,15 @@ atop
 autopsy
 autossh
 backuppc
-#bacula ## TODO: grab all the bacula packages in here
+## TODO: grab all the bacula packages in here
+#bacula 
 beep
 bin86
 bind-tools
 binutils
 bluez-utils
-bonnie++ ## TODO: ugh. apacman has a new "regex mode"... that you can't disable. https://github.com/oshazard/apacman/issues/79
+## TODO: ugh. apacman has a new "regex mode"... that you can't disable. https://github.com/oshazard/apacman/issues/79
+#bonnie++ 
 boxbackup-client
 boxbackup-server
 bozocrack-git
@@ -55,13 +57,15 @@ dd_rescue
 dd_rhelp
 debianutils
 debootstrap
-#dialog #giving a weird dependency issue
+## giving a weird dependency issue
+#dialog 
 diffutils
 djohn
 dmidecode
 dnssec-anchors
 dnstracer
-#dnsutils #replaced by bind-tools, https://www.archlinux.org/packages/extra/x86_64/bind-tools/
+## replaced by bind-tools, https://www.archlinux.org/packages/extra/x86_64/bind-tools/
+#dnsutils 
 dos2unix
 dropbear
 dstat
@@ -145,14 +149,17 @@ keyutils
 kismet-allplugins
 lftp
 links
-#logkeys-git # requires a /dev/input, which apparently isn't included in the chroots
+## requires a /dev/input, which apparently isn't included in the chroots
+#logkeys-git 
 lm_sensors
 lrzsz
 lshw
-#lsiutil # giving intermittent errors when trying to fetch source
+## giving intermittent errors when trying to fetch source
+#lsiutil 
 lsof
 lsscsi
-#lxde # apacman currently doesn't like package groups, so...
+## apacman currently doesn't like package groups, so...
+#lxde 
 gpicview
 lxappearance
 lxappearance-obconf
@@ -173,18 +180,20 @@ pcmanfm
 lynx
 #lzip
 macchanger
-#magicrescue # no longer maintained, upstream down
+## no longer maintained, upstream down
+#magicrescue 
 mbr
 mbuffer
 mcelog
 mdadm
 mdcrack
-# superseded by storcli
+## superseded by storcli
 #megaraid-cli
 memtester
 mfoc
 minicom
-#mondo # mindi-busybox fails to build 09.23.2016
+## mindi-busybox fails to build 09.23.2016
+#mondo 
 mtd-utils
 mtr
 mtree-git
@@ -198,13 +207,14 @@ net-snmp
 netselect
 nettle
 networkmanager-pptp
-nginx-devel
+nginx-mainline
 ngrep
 nmap
 nmon
 ntfs-3g
 ntfsfixboot
-#nwipe #broken since they moved to github(?)
+## broken since they moved to github(?)
+#nwipe 
 nwipe-git
 obnam
 open-iscsi
@@ -273,13 +283,15 @@ smartmontools
 smbclient
 s-nail
 socat
-#star ## do people even USE tape backups anymore?
+## do people even USE tape backups anymore?
+#star 
 storcli
 strace
 stress
 sucrack
 symlinks
 sysstat
+systemd-sysvcompat
 tcpdump
 tcpslice
 tcptraceroute
@@ -316,7 +328,8 @@ vncrack
 vnstat
 vpnc
 weplab
-#whdd #currently depends on dialog, which is broke as shit
+## currently depends on dialog, which is broke as shit
+#whdd 
 whois
 wifite-mod-pixiewps-git
 wipe

extra/pre-build.d/root/pre-build.sh

@@ -141,7 +141,12 @@ then
 fi
 # Add the regular user
 useradd -m -s /bin/bash -c "${USERCOMMENT}" ${REGUSR}
-usermod -aG users,games,video,audio ${REGUSR}
+usermod -aG users,games,video,audio ${REGUSR}  # TODO: remove this in lieu of $REGUSR_GRPS? these are all kind of required, though, for regular users anyways
+for g in $(echo ${REGUSR_GRPS} | sed 's/,[[:space:]]*/ /g');
+do
+	getent group ${g} > /dev/null 2>&1 || groupadd ${g}
+	usermod -aG ${g} ${REGUSR}
+done
 passwd -d ${REGUSR}
 # Add them to sudoers
 mkdir -p /etc/sudoers.d

extra/templates/pre-build.d/root/VARS.txt.j2

@@ -4,6 +4,7 @@ export PNAME='{{ bdisk['name'] }}'
 export DISTPUB='{{ bdisk['dev'] }}'
 export DISTDESC='{{ bdisk['desc'] }}'
 export REGUSR='{{ user['username']|lower }}'
+export REGUSR_GRPS='{{ user['groups'] }}'
 export USERCOMMENT='{{ user['name'] }}'
 export REGUSR_PASS='{{ user['password'] }}'
 export ROOT_PASS='{{ bdisk['root_password'] }}'