commit 9ca03aaf08236e8c2b496290167b2026959da8f8 Author: brent s Date: Wed Jan 20 17:43:19 2021 -0500 initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ff3b78a --- /dev/null +++ b/.gitignore @@ -0,0 +1,27 @@ +# https://git-scm.com/docs/gitignore +# https://help.github.com/articles/ignoring-files +# Example .gitignore files: https://github.com/github/gitignore +*.bak +screenlog* +*.swp +*.lck +*~ +.~lock.* +.editix +__pycache__/ +*.pyc +*.tar +*.tar.bz2 +*.tar.xz +*.tar.gz +*.tgz +*.txz +*.tbz +*.tbz2 +*.zip +*.run +*.7z +*.rar +*.sqlite3 +*.deb +.idea/ diff --git a/README b/README new file mode 100644 index 0000000..a726f76 --- /dev/null +++ b/README @@ -0,0 +1,5 @@ +This mini-project allows you to maintain segregated ESP failover in a multi-disk setup. + +If you have just one disk in your system, this project is useless to you. I promise. + +The examples assume use of UEFI and GRUB2, but it can easily be tweaked to use other bootloaders with a little research and modification to the configuration (and commenting out/modifying the writeConfs() function; I'll maybe make that a little smarter in the future). diff --git a/bootsync.hook b/bootsync.hook new file mode 100644 index 0000000..54ab194 --- /dev/null +++ b/bootsync.hook @@ -0,0 +1,27 @@ +# The following would be placed in /etc/pacman.d/hooks/ directory (you may need to create it if it doesn't exist) as bootsync.hook +# It assumes you have: +# * a properly configured /etc/bootsync.xml +# * /usr/local/bin/bootsync symlinked to /sys/BootSync/bootsync.py +[Trigger] +Operation = Install +Operation = Upgrade +Operation = Remove +Type = File +Target = boot/* +Target = usr/lib/modules/*/vmlinuz +Target = usr/lib/initcpio/* + +[Trigger] +Operation = Install +Operation = Upgrade +Operation = Remove +Type = Package +Target = linux +Target = mkinitcpio + +[Action] +When = PostTransaction +Exec = /usr/local/bin/bootsync +Depends = python-magic +Depends = python-psutil +Depends = python-lxml diff --git a/bootsync.py b/bootsync.py new file mode 100755 index 0000000..6c2f1c2 --- /dev/null +++ b/bootsync.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python3 + +import argparse +import hashlib +import json +import os +import platform +import re +import shutil +import subprocess +## +import magic # From http://darwinsys.com/file/, not https://github.com/ahupp/python-magic +import psutil +from lxml import etree + + +class BootSync(object): + def __init__(self, cfg = None, validate = True, dryrun = False, *args, **kwargs): + if not cfg: + self.cfgfile = '/etc/bootsync.xml' + else: + self.cfgfile = os.path.abspath(os.path.expanduser(cfg)) + self.ns = None + self.cfg = None + self.xml = None + self.schema = None + # This is the current live kernel. + self.currentKernVer = self._getRunningKernel() + # This is the installed kernel from the package manager. + self.kernelFile = None + self.installedKernVer = None + self.RequireReboot = False # If a reboot is needed (WARN, don't execute!) + self.blkids = {} + self.dummy_uuid = None + self.syncs = {} + ## + self.getCfg(validate = validate) + self.chkMounts(dryrun = dryrun) + self.chkReboot() + self.getChecks() + self.getBlkids() + + def getCfg(self, validate = True): + if not os.path.isfile(self.cfgfile): + raise FileNotFoundError('Configuration file {0} does not exist!'.format(self.cfgfile)) + try: + with open(self.cfgfile, 'rb') as f: + self.xml = etree.parse(f) + self.xml.xinclude() + self.cfg = self.xml.getroot() + except etree.XMLSyntaxError: + # self.logger.error('{0} is invalid XML'.format(self.cfgfile)) + raise ValueError(('{0} does not seem to be valid XML. ' + 'See sample.config.xml for an example configuration.').format(self.cfgfile)) + self.ns = self.cfg.nsmap.get(None, 'http://git.square-r00t.net/OpTools/tree/sys/BootSync/') + self.ns = '{{{0}}}'.format(self.ns) + if validate: + if not self.schema: + from urllib.request import urlopen + xsi = self.cfg.nsmap.get('xsi', 'http://www.w3.org/2001/XMLSchema-instance') + schemaLocation = '{{{0}}}schemaLocation'.format(xsi) + schemaURL = self.cfg.attrib.get(schemaLocation, + ('http://git.square-r00t.net/OpTools/plain/sys/BootSync/bootsync.xsd')) + with urlopen(schemaURL) as url: + self.schema = url.read() + self.schema = etree.XMLSchema(etree.XML(self.schema)) + self.schema.assertValid(self.xml) + return() + + def chkMounts(self, dryrun = False): + if not dryrun: + if os.geteuid() != 0: + raise PermissionError('You must be root to write to the appropriate destinations') + _mounts = {m.device: m.mountpoint for m in psutil.disk_partitions(all = True)} + for esp in self.cfg.findall('{0}partitions/{0}part'.format(self.ns)): + disk = esp.attrib['path'] + mount = os.path.abspath(os.path.expanduser(esp.attrib['mount'])) + if not dryrun: + if not os.path.isdir(mount): + os.makedirs(mount, exist_ok = True) + if disk not in _mounts: + with open(os.devnull, 'w') as devnull: + c = subprocess.run(['/usr/bin/mount', mount], + stderr = devnull) + if c.returncode == 1: # Not specified in fstab + subprocess.run(['/usr/bin/mount', disk, mount], + stderr = devnull) + elif c.returncode == 32: # Already mounted + pass + return() + + def chkReboot(self): + self._getInstalledKernel() + if not self.kernelFile: + return() # No isKernel="true" was specified in the config. + if self.installedKernVer != self.currentKernVer: + self.RequireReboot = True + # TODO: logger instead? + print(('NOTE: REBOOT REQUIRED. ' + 'New kernel is {0}. ' + 'Running kernel is {1}.').format(self.installedKernVer, + self.currentKernVer)) + return() + + def getBlkids(self): + cmd = ['/usr/bin/blkid', + '-o', 'export'] + if os.geteuid() != 0: + # TODO: logger? + print(('sudo is required to get device information. ' + 'You may be prompted to enter your sudo password.')) + cmd.insert(0, 'sudo') + c = subprocess.run(cmd, + stdout = subprocess.PIPE) + if c.returncode != 0: + raise RuntimeError('Could not fetch block ID information') + for p in c.stdout.decode('utf-8').split('\n\n'): + line = [i.strip() for i in p.splitlines()] + d = dict(map(lambda i: i.split('='), line)) + if d.get('TYPE') == 'squashfs': + continue + try: + self.blkids[d['DEVNAME']] = d.get('UUID', d['PARTUUID']) + except KeyError: + try: + self.blkids[d['DEVNAME']] = d['UUID'] + except KeyError: + continue + cmd = ['/usr/bin/findmnt', + '--json', + '-T', '/boot'] + # if os.geteuid() != 0: + # cmd.insert(0, 'sudo') + c = subprocess.run(cmd, + stdout = subprocess.PIPE) + self.dummy_uuid = self.blkids[json.loads(c.stdout.decode('utf-8'))['filesystems'][0]['source']] + return() + + def getChecks(self): + # Get the default hashtype (if one exists) + fc = self.cfg.find('{0}fileChecks'.format(self.ns)) + default_hashtype = fc.attrib.get('hashtype', 'md5').lower() + for f in fc.findall('{0}file'.format(self.ns)): + # We do /boot files manually in case it isn't specified as a + # separate mount. + file_hashtype = f.attrib.get('hashtype', default_hashtype).lower() + rel_fpath = f.text + fpath = os.path.join('/boot', rel_fpath) + canon_hash = self._get_hash(fpath, file_hashtype) + for esp in self.cfg.findall('{0}partitions/{0}part'.format(self.ns)): + mount = os.path.abspath(os.path.expanduser(esp.attrib['mount'])) + new_fpath = os.path.join(mount, rel_fpath) + file_hash = self._get_hash(new_fpath, file_hashtype) + if not file_hashtype or file_hash != canon_hash or not file_hash: + if rel_fpath not in self.syncs: + self.syncs[rel_fpath] = [] + self.syncs[rel_fpath].append(mount) + return() + + def sync(self, dryrun = False, *args, **kwargs): + if not dryrun: + if os.geteuid() != 0: + raise PermissionError('You must be root to write to the appropriate destinations') + # fileChecks are a *lot* easier. + for rel_fpath, mounts in self.syncs.items(): + for bootdir in mounts: + source = os.path.join('/boot', rel_fpath) + target = os.path.join(bootdir, rel_fpath) + destdir = os.path.dirname(target) + if not dryrun: + os.makedirs(destdir, exist_ok = True) + shutil.copy2(source, target) + bootmounts = [e.attrib['mount'] for e in self.cfg.findall('{0}partitions/{0}part'.format(self.ns))] + # syncPaths + syncpaths = self.cfg.find('{0}syncPaths'.format(self.ns)) + default_hashtype = syncpaths.attrib.get('hashtype', 'md5').lower() + for syncpath in syncpaths.findall('{0}path'.format(self.ns)): + source = os.path.abspath(os.path.expanduser(syncpath.attrib['source'])) + target = syncpath.attrib['target'] + pattern = syncpath.attrib['pattern'] + file_hashtype = syncpath.attrib.get('hashtype', default_hashtype) + # We don't use filecmp for this because: + # - dircmp doesn't recurse + # - the reports/lists don't retain relative paths + # - we can't regex out files + for root, dirs, files in os.walk(source): + prefix = re.sub(r'/?{0}/?'.format(source), '', root) + ptrn = re.compile(pattern) + for f in files: + fname_path = os.path.join(prefix, f) + bootsource = os.path.join(source, fname_path) + boottarget = os.path.join(target, fname_path) + if ptrn.search(f): + # Compare the contents. + orig_hash = self._get_hash(bootsource, file_hashtype) + for bootdir in bootmounts: + bootfile = os.path.join(bootdir, boottarget) + if not dryrun: + if not os.path.isfile(bootfile): + os.makedirs(os.path.dirname(bootfile), + exist_ok = True) + shutil.copy2(bootsource, bootfile) + else: + dest_hash = self._get_hash(bootfile, file_hashtype) + if not file_hashtype or orig_hash != dest_hash: + shutil.copy2(bootsource, bootfile) + return() + + + def writeConfs(self, dryrun = False, *args, **kwargs): + if not dryrun: + if os.geteuid() != 0: + raise PermissionError('You must be root to write to the appropriate destinations') + else: + return() + # Get a fresh config in place. + with open(os.devnull, 'wb') as DEVNULL: + c = subprocess.run(['/usr/bin/grub-mkconfig', + '-o', '/boot/grub/grub.cfg'], + stdout = DEVNULL, + stderr = DEVNULL) + if c.returncode != 0: + raise RuntimeError('An error occurred when generating the GRUB configuration file.') + with open('/boot/grub/grub.cfg', 'r') as f: + _grubcfg = f.read() + for esp in self.cfg.findall('{0}partitions/{0}part'.format(self.ns)): + mount = os.path.abspath(os.path.expanduser(esp.attrib['mount'])) + disk = os.path.abspath(os.path.expanduser(esp.attrib['path'])) + with open(os.path.join(mount, 'grub/grub.cfg'), 'w') as f: + for line in _grubcfg.splitlines(): + # If the array is in a degraded state, this will still let us at LEAST boot. + line = re.sub(r'\s+--hint=[\'"]?mduuid/[a-f0-9]{32}[\'"]?', '', line) + line = re.sub(r'^(\s*set\s+root=){0}$'.format(self.dummy_uuid), + self.blkids[disk], + line) + line = re.sub(r'(? + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/prep.txt b/prep.txt new file mode 100644 index 0000000..edfa440 --- /dev/null +++ b/prep.txt @@ -0,0 +1,57 @@ +PREPARATION: +0.) Comment out all /boot mounts in /etc/fstab and umount /boot if mounted as a separate mountpoint. + You want to use *the /boot on your / mount*. + +1.) Prepare each target partition (partitions/part below) as an ESP + (https://wiki.archlinux.org/index.php/EFI_system_partition#Format_the_partition). + +2.) Install GRUB2 to *each ESP*. See sample.config.xml for context for the below examples. + + grub-install \ + --boot-directory=/mnt/boot1 \ + --bootloader-id=Arch \ + --efi-directory=/mnt/boot1/ \ + --target=x86_64-efi \ + --no-nvram \ + --recheck + + grub-install \ + --boot-directory=/mnt/boot2 \ + --bootloader-id="Arch" \ + --efi-directory=/mnt/boot2/ \ + --target=x86_64-efi \ + --no-nvram \ + --recheck + +# These are not strictly necessary, as the same path is used in efibootmgr for the primary and the fallback. +# grub-install \ +# --boot-directory=/mnt/boot1 \ +# --bootloader-id="Arch (Fallback)" \ +# --efi-directory=/mnt/boot1/ \ +# --target=x86_64-efi \ +# --no-nvram \ +# --recheck +# +# grub-install \ +# --boot-directory=/mnt/boot2 \ +# --bootloader-id="Arch (Fallback)" \ +# --efi-directory=/mnt/boot2/ \ +# --target=x86_64-efi \ +# --no-nvram \ +# --recheck + +3.) Prepare the ESPs. See sample.config.xml for context for the below examples. + + efibootmgr \ + --create \ + --disk /dev/sdd \ + --part 1 \ + --loader /EFI/Arch/grubx64.efi \ + --label "Arch (Fallback)" + + efibootmgr \ + --create \ + --disk /dev/sdb \ + --part 1 \ + --loader /EFI/Arch/grubx64.efi \ + --label "Arch" diff --git a/sample.config.xml b/sample.config.xml new file mode 100644 index 0000000..b55a8ed --- /dev/null +++ b/sample.config.xml @@ -0,0 +1,90 @@ + + + + + + + + + + + + + + + + initramfs-linux.img + intel-ucode.img + memtest86+/memtest.bin + vmlinuz-linux + + + + + + + + + + + +