From df172d73eb36419c5a1281d253fbc8d61bf405bd Mon Sep 17 00:00:00 2001 From: r00t Date: Sun, 4 Dec 2016 02:43:49 -0500 Subject: [PATCH] beta release. still some missing features (ipxe, http/tftp and remote syncing, etc.) --- bdisk/bchroot.py | 130 ++++++- bdisk/bdisk.py | 5 +- bdisk/build.py | 11 +- bdisk/prep.py | 10 +- docs/BDisk_User_Manual.v1.fodt | 346 ++++++++++++++++-- .../etc/systemd/scripts/pacmandb.sh | 9 + .../multi-user.target.wants/pacmandb.service | 0 .../etc/systemd/system/pacmandb.service | 3 +- extra/pre-build.d/root/pre-build.sh | 7 +- 9 files changed, 443 insertions(+), 78 deletions(-) create mode 100755 extra/pre-build.d/etc/systemd/scripts/pacmandb.sh rename {overlay => extra/pre-build.d}/etc/systemd/system/multi-user.target.wants/pacmandb.service (100%) rename {overlay => extra/pre-build.d}/etc/systemd/system/pacmandb.service (51%) diff --git a/bdisk/bchroot.py b/bdisk/bchroot.py index a3dd751..40e6b3e 100755 --- a/bdisk/bchroot.py +++ b/bdisk/bchroot.py @@ -3,6 +3,9 @@ import sys import psutil import subprocess import datetime +import tarfile +import humanize +import shutil def chroot(chrootdir, chroot_hostname, cmd = '/root/pre-build.sh'): @@ -11,43 +14,91 @@ def chroot(chrootdir, chroot_hostname, cmd = '/root/pre-build.sh'): mounts = [] for m in mountpoints: mounts.append(m.mountpoint) - # mount the chrootdir... onto itself. as a bind mount. it's so stupid, i know. see https://bugs.archlinux.org/task/46169 + cmounts = {} + for m in ('chroot', 'resolv', 'proc', 'sys', 'efi', 'dev', 'pts', 'shm', 'run', 'tmp'): + cmounts[m] = None + # chroot (bind mount... onto itself. it's so stupid, i know. see https://bugs.archlinux.org/task/46169) if chrootdir not in mounts: - subprocess.call(['/bin/mount', '--bind', chrootdir, chrootdir]) -### The following mountpoints don't seem to mount properly with pychroot. save it for v3.n+1. TODO. ### - # bind-mount so we can resolve things inside + cmounts['chroot'] = ['/bin/mount', + '--bind', + chrootdir, + chrootdir] + # resolv if (chrootdir + '/etc/resolv.conf') not in mounts: - subprocess.call(['/bin/mount', '--bind', '-o', 'ro', '/etc/resolv.conf', chrootdir + '/etc/resolv.conf']) - # mount -t proc to chrootdir + '/proc' here + cmounts['resolv'] = ['/bin/mount', + '--bind', + '-o', 'ro', + '/etc/resolv.conf', + chrootdir + '/etc/resolv.conf'] + # proc if (chrootdir + '/proc') not in mounts: - subprocess.call(['/bin/mount', '-t', 'proc', '-o', 'nosuid,noexec,nodev', 'proc', chrootdir + '/proc']) - # rbind mount /sys to chrootdir + '/sys' here + cmounts['proc'] = ['/bin/mount', + '-t', 'proc', + '-o', 'nosuid,noexec,nodev', + 'proc', + chrootdir + '/proc'] + # sys if (chrootdir + '/sys') not in mounts: - subprocess.call(['/bin/mount', '-t', 'sysfs', '-o', 'nosuid,noexec,nodev,ro', 'sys', chrootdir + '/sys']) - # mount the efivars in the chroot if it exists on the host. i mean, why not? + cmounts['sys'] = ['/bin/mount', + '-t', 'sysfs', + '-o', 'nosuid,noexec,nodev,ro', + 'sys', + chrootdir + '/sys'] + # efi (if it exists on the host) if '/sys/firmware/efi/efivars' in mounts: if (chrootdir + '/sys/firmware/efi/efivars') not in mounts: - subprocess.call(['/bin/mount', '-t', 'efivarfs', '-o', 'nosuid,noexec,nodev', 'efivarfs', chrootdir + '/sys/firmware/efi/efivars']) - # rbind mount /dev to chrootdir + '/dev' here + cmounts['efi'] = ['/bin/mount', + '-t', 'efivarfs', + '-o', 'nosuid,noexec,nodev', + 'efivarfs', + chrootdir + '/sys/firmware/efi/efivars'] + # dev if (chrootdir + '/dev') not in mounts: - subprocess.call(['/bin/mount', '-t', 'devtmpfs', '-o', 'mode=0755,nosuid', 'udev', chrootdir + '/dev']) + cmounts['dev'] = ['/bin/mount', + '-t', 'devtmpfs', + '-o', 'mode=0755,nosuid', + 'udev', + chrootdir + '/dev'] + # pts if (chrootdir + '/dev/pts') not in mounts: - subprocess.call(['/bin/mount', '-t', 'devpts', '-o', 'mode=0620,gid=5,nosuid,noexec', 'devpts', chrootdir + '/dev/pts']) + cmounts['pts'] = ['/bin/mount', + '-t', 'devpts', + '-o', 'mode=0620,gid=5,nosuid,noexec', + 'devpts', + chrootdir + '/dev/pts'] + # shm (if it exists on the host) if '/dev/shm' in mounts: if (chrootdir + '/dev/shm') not in mounts: - subprocess.call(['/bin/mount', '-t', 'tmpfs', '-o', 'mode=1777,nosuid,nodev', 'shm', chrootdir + '/dev/shm']) + cmounts['shm'] = ['/bin/mount', + '-t', 'tmpfs', + '-o', 'mode=1777,nosuid,nodev', + 'shm', + chrootdir + '/dev/shm'] + # run (if it exists on the host) if '/run' in mounts: if (chrootdir + '/run') not in mounts: - subprocess.call(['/bin/mount', '-t', 'tmpfs', '-o', 'nosuid,nodev,mode=0755', 'run', chrootdir + '/run']) + cmounts['run'] = ['/bin/mount', + '-t', 'tmpfs', + '-o', 'nosuid,nodev,mode=0755', + 'run', + chrootdir + '/run'] + # tmp (if it exists on the host) if '/tmp' in mounts: if (chrootdir + '/tmp') not in mounts: - subprocess.call(['/bin/mount', '-t', 'tmpfs', '-o', 'mode=1777,strictatime,nodev,nosuid', 'tmp', chrootdir + '/tmp']) - + cmounts['tmp'] = ['/bin/mount', + '-t', 'tmpfs', + '-o', 'mode=1777,strictatime,nodev,nosuid', + 'tmp', + chrootdir + '/tmp'] + # the order we mount here is VERY IMPORTANT. Sure, we could do "for m in cmounts:", but dicts aren't ordered until python 3.6 + # and this is SO important it's best that we be explicit as possible while we're still in alpha/beta stage. TODO? + for m in ('chroot', 'resolv', 'proc', 'sys', 'efi', 'dev', 'pts', 'shm', 'run', 'tmp'): + if cmounts[m]: + subprocess.call(cmounts[m]) print("{0}: Performing '{1}' in chroot for {2}...".format(datetime.datetime.now(), cmd, chrootdir)) - print("\t\t\t You can view the progress via:\n\n\t\ttail -f {0}/var/log/chroot_install.log\n".format(chrootdir)) + print("\t\t\t You can view the progress via:\n\t\t\t tail -f {0}/var/log/chroot_install.log".format(chrootdir)) real_root = os.open("/", os.O_RDONLY) os.chroot(chrootdir) - os.system('locale-gen > /dev/null 2>&1') os.system('/root/pre-build.sh') os.fchdir(real_root) os.chroot('.') @@ -56,3 +107,42 @@ def chroot(chrootdir, chroot_hostname, cmd = '/root/pre-build.sh'): def chrootUnmount(chrootdir): subprocess.call(['umount', '-lR', chrootdir]) + +def chrootTrim(build): + chrootdir = build['chrootdir'] + arch = build['arch'] + for a in arch: + # Compress the pacman and apacman caches. + for i in ('pacman', 'apacman'): + shutil.rmtree('{0}/root.{1}/var/cache/{2}'.format(chrootdir, a, i)) + os.makedirs('{0}/root.{1}/usr/local/{2}'.format(chrootdir, a, i), exist_ok = True) + tarball = '{0}/root.{1}/usr/local/{2}/{2}.db.tar.xz'.format(chrootdir, a, i) + dbdir = '{0}/root.{1}/var/lib/{2}/local'.format(chrootdir, a, i) + if os.path.isdir(dbdir): + print("{0}: Now compressing the {1} cache ({2}). Please wait...".format( + datetime.datetime.now(), + chrootdir, + a)) + if os.path.isfile(tarball): + os.remove(tarball) + with tarfile.open(name = tarball, mode = 'w:xz') as tar: # if this complains, use x:xz instead + tar.add(dbdir, arcname = os.path.basename(dbdir)) + shutil.rmtree(dbdir, ignore_errors = True) + print("{0}: Done creating {1} ({2}).\n\t\t\t {3} cleared.".format( + datetime.datetime.now(), + tarball, + humanize.naturalsize( + os.path.getsize(tarball)), + dbdir)) + # TODO: move the self-cleanup in pre-build.sh to here. + delme = ['/root/.gnupg', + '/root/.bash_history', + #'/var/log/chroot_install.log', # disable for now. maybe always disable if debug is enabled? TODO. + '/.git', + '/root/.viminfo'] + for i in delme: + fullpath = '{0}/root.{1}{2}'.format(chrootdir, a, i) + if os.path.isfile(fullpath): + os.remove(fullpath) + elif os.path.isdir(fullpath): + shutil.rmtree(fullpath, ignore_errors = True) diff --git a/bdisk/bdisk.py b/bdisk/bdisk.py index e7f24bb..75dffdb 100755 --- a/bdisk/bdisk.py +++ b/bdisk/bdisk.py @@ -6,11 +6,9 @@ import build import datetime # we need to: -# 9.) build.genImg (TODO)- build the squashed image, etc. see will_it_blend in old bdisk -# 9.5) copy the files also in the same script. after the commented-out mtree-generation -# # we also need to figure out how to implement "mentos" (old bdisk) like functionality, letting us reuse an existing chroot install if possible to save time for future builds. # if not, though, it's no big deal. +# still on the todo: iPXE if __name__ == '__main__': print('{0}: Starting.'.format(datetime.datetime.now())) conf = host.parseConfig(host.getConfig())[1] @@ -22,6 +20,7 @@ if __name__ == '__main__': bchroot.chroot(conf['build']['chrootdir'] + '/root.' + a, 'bdisk.square-r00t.net') bchroot.chrootUnmount(conf['build']['chrootdir'] + '/root.' + a) prep.postChroot(conf['build']) + bchroot.chrootTrim(conf['build']) build.genImg(conf['build'], conf['bdisk']) build.genUEFI(conf['build'], conf['bdisk']) fulliso = build.genISO(conf) diff --git a/bdisk/build.py b/bdisk/build.py index d4dc38b..acb1353 100755 --- a/bdisk/build.py +++ b/bdisk/build.py @@ -363,12 +363,6 @@ def genISO(conf): tempdir] DEVNULL = open(os.devnull, 'w') subprocess.call(cmd, stdout = DEVNULL, stderr = subprocess.STDOUT) - #proc = subprocess.Popen(cmd, stdout = subprocess.PIPE, bufsize = 1) - #for line in iter(proc.stdout.readline, b''): - #for line in iter(proc.stdout.readline, ''): - # print(line) - #p.stdout.close() - #p.wait() # Get size of ISO iso = {} iso['sha'] = hashlib.sha256() @@ -386,11 +380,10 @@ def genISO(conf): return(iso) def displayStats(iso): - print('== {0} {1} =='.format(iso['type'], iso['fmt'])) + print("{0}:\n== {1} {2} ==".format(datetime.datetime.now(), iso['type'], iso['fmt'])) print('Size: {0}'.format(iso['size'])) print('SHA256: {0}'.format(iso['sha'])) - print('Location: {0}'.format(iso['file'])) - print() + print('Location: {0}\n'.format(iso['file'])) def cleanUp(): # TODO: clear out all of tempdir? diff --git a/bdisk/prep.py b/bdisk/prep.py index 0ea43b4..4217504 100755 --- a/bdisk/prep.py +++ b/bdisk/prep.py @@ -48,8 +48,8 @@ def downloadTarball(build): print("\n{0}: Generating a GPG key. Please wait...".format(datetime.datetime.now())) # python-gnupg 0.3.9 spits this error in Arch. it's harmless, but ugly af. # TODO: remove this when the error doesn't happen anymore. - print("\tIf you see a \"ValueError: Unknown status message: 'KEY_CONSIDERED'\" error, it can be safely ignored.") - print("\tIf this is taking a VERY LONG time, try installing haveged and starting it. This can be " + + print("\t\t\t If you see a \"ValueError: Unknown status message: 'KEY_CONSIDERED'\" error,\n\t\t\t it can be safely ignored.") + print("\t\t\t If this is taking a VERY LONG time, try installing haveged and starting it.\n\t\t\t This can be" + "done safely in parallel with the build process.\n") input_data = gpg.gen_key_input(name_email = 'tempuser@nodomain.tld', passphrase = 'placeholder_passphrase') key = gpg.gen_key(input_data) @@ -76,7 +76,7 @@ def downloadTarball(build): tarball_path[a], humanize.naturalsize( os.path.getsize(tarball_path[a])))) - print("{0}: Checking that the hash checksum for {1} matches {2}, please wait...".format( + print("{0}: Checking that the hash checksum for {1}\n\t\t\t matches {2}, please wait...".format( datetime.datetime.now(), tarball_path[a], sha1)) @@ -166,14 +166,14 @@ def buildChroot(build, keep = False): # and copy over the files. again, chown to root. for file in prebuild_overlay['files']: shutil.copy2(extradir + '/pre-build.d/' + file, chrootdir + '/root.' + a + '/' + file, follow_symlinks = False) - os.chown(chrootdir + '/root.' + a + '/' + file, 0, 0) + os.chown(chrootdir + '/root.' + a + '/' + file, 0, 0, follow_symlinks = False) # do the same for arch-specific stuff. for dir in prebuild_arch_overlay[a]['dirs']: os.makedirs(chrootdir + '/root.' + a + '/' + dir, exist_ok = True) os.chown(chrootdir + '/root.' + a + '/' + dir, 0, 0) for file in prebuild_arch_overlay[a]['files']: shutil.copy2(extradir + '/pre-build.d/' + a + '/' + file, chrootdir + '/root.' + a + '/' + file, follow_symlinks = False) - os.chown(chrootdir + '/root.' + a + '/' + file, 0, 0) + os.chown(chrootdir + '/root.' + a + '/' + file, 0, 0, follow_symlinks = False) def prepChroot(build, bdisk, user): chrootdir = build['chrootdir'] diff --git a/docs/BDisk_User_Manual.v1.fodt b/docs/BDisk_User_Manual.v1.fodt index a772d84..5def90f 100644 --- a/docs/BDisk_User_Manual.v1.fodt +++ b/docs/BDisk_User_Manual.v1.fodt @@ -1,24 +1,24 @@ - 2016-12-01T11:27:37.6655108212016-12-01T12:45:33.017762399PT1H7M45S1LibreOffice/5.2.3.3$Linux_X86_64 LibreOffice_project/20m0$Build-3 + 2016-12-01T11:27:37.6655108212016-12-03T23:47:18.021215054PT11H14M18S19LibreOffice/5.2.3.3$Linux_X86_64 LibreOffice_project/20m0$Build-3 - 0 + 59055 0 40748 - 22782 + 21751 true false view2 - 11578 - 3494 + 14079 + 77479 0 - 0 + 59055 40746 - 22781 + 80804 0 1 false @@ -69,7 +69,7 @@ false false true - 91633 + 396750 false false true @@ -182,24 +182,81 @@ - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + @@ -245,65 +302,166 @@ - + + + + - + + + + + + + - - - + + - + + + + + + + - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + - + - - - / - - / - + + + , + + + + , + - + + + BDisk ManualPage 2 of 3 + + - 12/01/16 + Saturday, December 3, 2016 - + @@ -312,13 +470,125 @@ - BDISK - Manual v1.0 - Brent Saner - bts@square-r00t.net + BDISK + Manual v1.0 + Brent Saner + bts@square-r00t.net - + + + + Table of Contents + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Table of Contents + + Chapter I: Introduction3 + Section I.1: What is BDisk?3 + Section I.2: Who wrote it?3 + Section I.3: What is this document?3 + Section I.4: Conventions used in this document3 + Section I.5: Further information/resources3 + I.5.i: For Developers3 + + + + Introduction + What is BDisk? + BDisk refers to both a live distribution I use in my own uses (for rescue situations, recovery, etc.) but foremost and most importantly, it refers to the tool I use for building that distribution. This is what this project and documentation refer to when the word “BDisk” is used. + BDisk is GPLv3-licensed. This means that you can use it for business reasons, personal reasons, modify it, etc. There are a few restrictions I retain, however, on this (don’t worry; they’re all in line with the GPLv3). You can find the full license in docs/LICENSE. + Who wrote it? + I (Brent Saner) am a GNU/Linux Systems/Network Administrator/Engineer- I wear a lot of hats. I have a lot of side projects to keep me busy when I’m not working at ${dayjob}, mostly to assist in other side projects and become more efficient and proficient. + What is this document? + Conventions used in this document + Further information/resources + For Users + If you encounter any bugs (or have any suggestions on how to improve BDisk!), please file a bug report in my bug tracker. + For Developers + The source is available to browse online or can be checked out via git (via the git protocol or http protocol). It is also available via Arch Linux’s AUR. If you are interested in packaging BDisk for other distributions, please feel free to contact me. + \ No newline at end of file diff --git a/extra/pre-build.d/etc/systemd/scripts/pacmandb.sh b/extra/pre-build.d/etc/systemd/scripts/pacmandb.sh new file mode 100755 index 0000000..713ff52 --- /dev/null +++ b/extra/pre-build.d/etc/systemd/scripts/pacmandb.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +for i in pacman apacman; +do + if [ -f /usr/local/${i}.db.tar.xz ]; + then + /usr/bin/tar -Jxf /usr/local/${i}.db.tar.xz -C /var/lib/${i}/ + fi +done diff --git a/overlay/etc/systemd/system/multi-user.target.wants/pacmandb.service b/extra/pre-build.d/etc/systemd/system/multi-user.target.wants/pacmandb.service similarity index 100% rename from overlay/etc/systemd/system/multi-user.target.wants/pacmandb.service rename to extra/pre-build.d/etc/systemd/system/multi-user.target.wants/pacmandb.service diff --git a/overlay/etc/systemd/system/pacmandb.service b/extra/pre-build.d/etc/systemd/system/pacmandb.service similarity index 51% rename from overlay/etc/systemd/system/pacmandb.service rename to extra/pre-build.d/etc/systemd/system/pacmandb.service index 870e6dd..cda181f 100644 --- a/overlay/etc/systemd/system/pacmandb.service +++ b/extra/pre-build.d/etc/systemd/system/pacmandb.service @@ -3,7 +3,8 @@ Description=Restoring Installed Packages DB [Service] Type=oneshot -ExecStart=/usr/bin/tar -Jxf /usr/local/pacman.db.tar.xz -C /var/lib/pacman/ +#ExecStart=/usr/bin/tar -Jxf /usr/local/pacman.db.tar.xz -C /var/lib/pacman/ +ExecStart=/etc/systemd/system/scripts/pacmandb.sh RemainAfterExit=yes [Install] diff --git a/extra/pre-build.d/root/pre-build.sh b/extra/pre-build.d/root/pre-build.sh index e0bfdfc..20d97c3 100755 --- a/extra/pre-build.d/root/pre-build.sh +++ b/extra/pre-build.d/root/pre-build.sh @@ -12,7 +12,6 @@ exec 1>/var/log/chroot_install.log 2>&1 # we need this fix before anything. dirmngr /dev/null 2>&1 -locale-gen cleanPacorigs() { @@ -45,7 +44,9 @@ pacman -Syy # Just in case. cleanPacorigs # Install some prereqs -pacman -S --noconfirm --needed base syslinux wget rsync unzip jshon sed sudo abs xmlto bc docbook-xsl git +pacman -S sed +pacman -S --noconfirm --needed base syslinux wget rsync unzip jshon sudo abs xmlto bc docbook-xsl git +locale-gen # And get rid of files it wants to replace cleanPacorigs # Force update all currently installed packages in case the tarball's out of date @@ -175,6 +176,8 @@ fi paccache -rk0 localepurge-config localepurge +localepurge-config +localepurge rm -f /root/.bash_history rm -f /root/.viminfo rm -f /root/apacman-*.pkg.tar.xz