summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorbrent s <r00t@square-r00t.net>2019-08-18 20:24:39 -0400
committerbrent s <r00t@square-r00t.net>2019-08-18 20:24:39 -0400
commitc149a7b3b78e3384b8258344d9800c56af23059f (patch)
tree849ac9eab9a693e95e9f40218e866aeb6bddda93
parent3976fd631cc00a521308a7e5e71e9502416730d2 (diff)
downloadOpTools-c149a7b3b78e3384b8258344d9800c56af23059f.tar.xz
adding BootSync
-rwxr-xr-xsys/BootSync/bootsync.py311
-rw-r--r--sys/BootSync/bootsync.xsd95
-rw-r--r--sys/BootSync/prep.txt56
-rw-r--r--sys/BootSync/sample.config.xml49
4 files changed, 511 insertions, 0 deletions
diff --git a/sys/BootSync/bootsync.py b/sys/BootSync/bootsync.py
new file mode 100755
index 0000000..fd39c37
--- /dev/null
+++ b/sys/BootSync/bootsync.py
@@ -0,0 +1,311 @@
+#!/usr/bin/env python3
+
+import argparse
+import hashlib
+import json
+import os
+import platform
+import re
+import shutil
+import subprocess
+##
+import magic # From http://darwinsys.com/file/, not https://github.com/ahupp/python-magic
+import psutil
+from lxml import etree
+
+
+ # def get_file_kernel_ver(self, kpath):
+ # # Gets the version of a kernel file.
+ # kpath = os.path.abspath(os.path.expanduser(kpath))
+ # _kinfo = {}
+ # with open(kpath, 'rb') as f:
+ # _m = magic.detect_from_content(f.read())
+ # for i in _m.name.split(','):
+ # l = i.strip().split()
+ # # Note: this only grabs the version number.
+ # # If we want to get e.g. the build user/machine, date, etc.,
+ # # then we need to join l[1:].
+ # # We technically don't even need a dict, either. We can just iterate.
+ # # TODO.
+ # _kinfo[l[0].lower()] = (l[1] if len(l) > 1 else None)
+ # if 'version' not in _kinfo:
+ # raise RuntimeError('Cannot deterimine the version of {0}'.format(
+ # kpath))
+ # else:
+ # return (_kinfo['version'])
+
+class BootSync(object):
+ def __init__(self, cfg = None, *args, **kwargs):
+ if not cfg:
+ self.cfgfile = '/etc/bootsync.xml'
+ else:
+ self.cfgfile = os.path.abspath(os.path.expanduser(cfg))
+ self.ns = '{http://git.square-r00t.net/OpTools/tree/sys/BootSync/}'
+ self.cfg = None
+ self.xml = None
+ # This is the current live kernel.
+ self.currentKernVer = self._getRunningKernel()
+ # This is the installed kernel from the package manager.
+ self.kernelFile = None
+ self.installedKernVer = None
+ self.RequireReboot = False # If a reboot is needed (WARN, don't execute!)
+ self.blkids = {}
+ self.dummy_uuid = None
+ self.syncs = {}
+ ##
+ self.getCfg()
+ self.chkMounts()
+ self.chkReboot()
+ self.getHashes()
+ self.getBlkids()
+ # self.sync()
+ # self.writeConfs()
+
+ def getCfg(self):
+ if not os.path.isfile(self.cfgfile):
+ raise FileNotFoundError('Configuration file {0} does not exist!'.format(self.cfgfile))
+ try:
+ with open(self.cfgfile, 'rb') as f:
+ self.xml = etree.parse(f)
+ self.xml.xinclude()
+ self.cfg = self.xml.getroot()
+ except etree.XMLSyntaxError:
+ # self.logger.error('{0} is invalid XML'.format(self.cfgfile))
+ raise ValueError(('{0} does not seem to be valid XML. '
+ 'See sample.config.xml for an example configuration.').format(self.cfgfile))
+ return()
+
+ def chkMounts(self):
+ _mounts = {m.device: m.mountpoint for m in psutil.disk_partitions(all = True)}
+ for esp in self.cfg.findall('{0}partitions/{0}part'.format(self.ns)):
+ disk = esp.attrib['path']
+ mount = os.path.abspath(os.path.expanduser(esp.attrib['mount']))
+ if not os.path.isdir(mount):
+ os.makedirs(mount, exist_ok = True)
+ if disk not in _mounts:
+ with open(os.devnull, 'w') as devnull:
+ c = subprocess.run(['/usr/bin/mount', mount],
+ stderr = devnull)
+ if c.returncode == 1: # Not specified in fstab
+ subprocess.run(['/usr/bin/mount', disk, mount],
+ stderr = devnull)
+ elif c.returncode == 32: # Already mounted
+ pass
+ return()
+
+ def chkReboot(self):
+ self._getInstalledKernel()
+ if not self.kernelFile:
+ return() # No isKernel="true" was specified in the config.
+ if self.installedKernVer != self.currentKernVer:
+ self.RequireReboot = True
+ # TODO: logger instead?
+ print(('NOTE: REBOOT REQUIRED. '
+ 'New kernel is {0}. '
+ 'Running kernel is {1}.').format(self.installedKernVer,
+ self.currentKernVer))
+ return()
+
+ def getBlkids(self):
+ c = subprocess.run(['/usr/bin/blkid',
+ '-o', 'export'],
+ stdout = subprocess.PIPE)
+ if c.returncode != 0:
+ raise RuntimeError('Could not fetch block ID information')
+ for p in c.stdout.decode('utf-8').split('\n\n'):
+ line = [i.strip() for i in p.splitlines()]
+ d = dict(map(lambda i: i.split('='), line))
+ if d.get('TYPE') == 'squashfs':
+ continue
+ self.blkids[d['DEVNAME']] = d.get('PARTUUID', d['UUID'])
+ c = subprocess.run(['/usr/bin/findmnt',
+ '--json',
+ '-T', '/boot'],
+ stdout = subprocess.PIPE)
+ # I write ridiculous one-liners.
+ self.dummy_uuid = self.blkids[json.loads(c.stdout.decode('utf-8'))['filesystems'][0]['source']]
+ return()
+
+ def getHashes(self):
+ def _get_hash(fpathname):
+ fpathname = os.path.abspath(os.path.expanduser(fpathname))
+ _hash = hashlib.sha512()
+ with open(fpathname, 'rb') as fh:
+ _hash.update(fh.read())
+ return(_hash.hexdigest())
+ for f in self.cfg.findall('{0}fileChecks/{0}file'):
+ # We do /boot files manually in case it isn't specified as a
+ # separate mount.
+ rel_fpath = f.text
+ fpath = os.path.join('/boot', rel_fpath)
+ canon_hash = _get_hash(fpath)
+ for esp in self.cfg.findall('{0}partitions/{0}part'.format(self.ns)):
+ mount = os.path.abspath(os.path.expanduser(esp.attrib['mount']))
+ new_fpath = os.path.join(mount, f)
+ file_hash = _get_hash(new_fpath)
+ if file_hash != canon_hash:
+ if rel_fpath not in self.syncs:
+ self.syncs[rel_fpath] = []
+ self.syncs[rel_fpath].append(mount)
+ return()
+
+ def sync(self, dryrun = False, *args, **kwargs):
+ if not dryrun:
+ if os.geteuid() != 0:
+ raise PermissionError('You must be root to write to the appropriate destinations')
+ for f in self.syncs:
+ for m in self.syncs[f]:
+ orig = os.path.join('/boot', f)
+ dest = os.path.join(m, f)
+ if not dryrun:
+ shutil.copy2(orig, dest)
+ bootmounts = [e.attrib['mount'] for e in self.cfg.findall('{0}partitions/{0}part'.format(self.ns))]
+ # syncPaths
+ for syncpath in self.cfg.findall('{0}syncPaths/{0}path'.format(self.ns)):
+ source = os.path.abspath(os.path.expanduser(syncpath.attrib['source']))
+ target = syncpath.attrib['target']
+ pattern = syncpath.attrib['pattern']
+ # We don't use filecmp for this because:
+ # - dircmp doesn't recurse
+ # - the reports/lists don't retain relative paths
+ # - we can't regex out files
+ for root, dirs, files in os.walk(source):
+ prefix = re.sub('\/?{0}\/?'.format(source), '', root)
+ ptrn = re.compile(pattern)
+ for f in files:
+ fname_path = os.path.join(prefix, f)
+ bootsource = os.path.join(source, fname_path)
+ boottarget = os.path.join(target, fname_path)
+ if ptrn.search(f):
+ # Compare the contents.
+ with open(bootsource, 'rb') as fh:
+ orig_hash = hashlib.sha512(fh.read()).hexdigest()
+ for bootdir in bootmounts:
+ bootfile = os.path.join(bootdir, boottarget)
+ if not dryrun:
+ if not os.path.isfile(bootfile):
+ os.makedirs(os.path.dirname(bootfile),
+ exist_ok = True)
+ shutil.copy2(bootsource, bootfile)
+ else:
+ with open(bootfile, 'rb') as fh:
+ dest_hash = hashlib.sha512(fh.read()).hexdigest()
+ if orig_hash != dest_hash:
+ shutil.copy2(bootsource, bootfile)
+ # fileChecks are a *lot* easier.
+ for f in self.cfg.findall('{0}fileChecks/{0}file'.format(self.ns)):
+ source = os.path.join('/boot', f.text)
+ with open(source, 'rb') as fh:
+ orig_hash = hashlib.sha512(fh.read()).hexdigest()
+ for bootdir in bootmounts:
+ bootfile = os.path.join(bootdir, f.text)
+ if not dryrun:
+ if not os.path.isfile(bootfile):
+ os.makedirs(os.path.dirname(bootfile),
+ exist_ok = True)
+ shutil.copy2(source, bootfile)
+ else:
+ with open(bootfile, 'rb') as fh:
+ dest_hash = hashlib.sha512(fh.read()).hexdigest()
+ if orig_hash != dest_hash:
+ shutil.copy2(source, bootfile)
+ return()
+
+
+ def writeConfs(self, dryrun = False, *args, **kwargs):
+ if not dryrun:
+ if os.geteuid() != 0:
+ raise PermissionError('You must be root to write to the appropriate destinations')
+ else:
+ return()
+ # Get a fresh config in place.
+ with open(os.devnull, 'wb') as DEVNULL:
+ c = subprocess.run(['/usr/bin/grub-mkconfig',
+ '-o', '/boot/grub/grub.cfg'],
+ stdout = DEVNULL,
+ stderr = DEVNULL)
+ if c.returncode != 0:
+ raise RuntimeError('An error occurred when generating the GRUB configuration file.')
+ with open('/boot/grub/grub.cfg', 'r') as f:
+ _grubcfg = f.read()
+ for esp in self.cfg.findall('{0}partitions/{0}part'.format(self.ns)):
+ mount = os.path.abspath(os.path.expanduser(esp.attrib['mount']))
+ disk = os.path.abspath(os.path.expanduser(esp.attrib['path']))
+ with open(os.path.join(mount, 'grub/grub.cfg'), 'w') as f:
+ for line in _grubcfg.splitlines():
+ # if re.search(r'^\s*search\s+(.*)\s(-u|--fs-uuid)', line):
+ # pass
+ i = re.sub(r'(?<!\=UUID\=){0}'.format(self.dummy_uuid),
+ self.blkids[disk],
+ line)
+ # If the array is in a degraded state, this will still let us at LEAST boot.
+ i = re.sub(r'\s+--hint=[\'"]?mduuid/[a-f0-9]{32}[\'"]?', '', i)
+ f.write('{0}\n'.format(i))
+ return()
+
+ def _getRunningKernel(self):
+ _vers = []
+ # If we change the version string capture in get_file_kernel_ver(),
+ # this will need to be expanded as well.
+ # Really we only need to pick one, but #YOLO; why not sanity-check.
+ # ALL of these should match, hence the reduction with set() down to (what SHOULD be) just 1 item.
+ _vers.append(os.uname().release)
+ _vers.append(platform.release())
+ _vers.append(platform.uname().release)
+ _vers = sorted(list(set(_vers)))
+ if len(_vers) != 1:
+ raise RuntimeError('Cannot reliably determine current running kernel version!')
+ else:
+ return(_vers[0])
+
+ def _getInstalledKernel(self):
+ # Could we maybe remove the dependency for the "magic" module with a struct?
+ # http://lxr.linux.no/#linux+v2.6.39/Documentation/x86/boot.txt
+ # https://stackoverflow.com/a/11179559/733214
+ try:
+ len(self.cfg)
+ except TypeError:
+ raise RuntimeError('Tried to find the isKernel with no config set up and parsed')
+ for f in self.cfg.findall('{0}fileChecks/{0}file'.format(self.ns)):
+ isKernel = (True
+ if f.attrib.get('isKernel', 'false').lower() in ('true', '1')
+ else
+ False)
+ if isKernel:
+ self.kernelFile = f.text
+ if self.kernelFile:
+ with open(os.path.join('/boot', self.kernelFile), 'rb') as fh:
+ magicname = magic.detect_from_content(fh.read())
+ names = [i.strip().split(None, 1) for i in magicname.name.split(',') if i.strip() != '']
+ for n in names:
+ if len(n) != 2:
+ continue
+ k, v = n
+ # Note: this only grabs the version number.
+ # If we want to get e.g. the build user/machine, date, etc.,
+ # then we need to do a join. Shouldn't be necessary, though.
+ if k.lower() == 'version':
+ self.installedKernVer = v.split(None, 1)[0]
+ return()
+
+def parseArgs():
+ args = argparse.ArgumentParser(description = ('Sync files to assist using mdadm RAID arrays with UEFI'))
+ args.add_argument('-c', '--cfg',
+ dest = 'cfg',
+ default = '/etc/bootsync.xml',
+ help = ('The path to the bootsync configuration file. Default is /etc/bootsync.xml'))
+ args.add_argument('-n', '--dry-run',
+ dest = 'dryrun',
+ action = 'store_true',
+ help = ('If specified, don\'t write any changes'))
+ return(args)
+
+def main():
+ args = vars(parseArgs().parse_args())
+ bs = BootSync(**args)
+ bs.sync(**args)
+ bs.writeConfs(**args)
+ return()
+
+if __name__ == '__main__':
+ main()
diff --git a/sys/BootSync/bootsync.xsd b/sys/BootSync/bootsync.xsd
new file mode 100644
index 0000000..4ab4c7d
--- /dev/null
+++ b/sys/BootSync/bootsync.xsd
@@ -0,0 +1,95 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
+ targetNamespace="http://git.square-r00t.net/OpTools/tree/sys/BootSync/"
+ xmlns="http://git.square-r00t.net/OpTools/tree/sys/BootSync/"
+ xmlns:bootsync="http://git.square-r00t.net/OpTools/tree/sys/BootSync/"
+ elementFormDefault="qualified"
+ attributeFormDefault="unqualified">
+
+ <xs:simpleType name="diskdev">
+ <xs:restriction base="xs:string">
+ <xs:pattern value="/dev/([A-Za-z0-9+_-]+/)?[A-Za-z0-9+_-]+[0-9]?"/>
+ </xs:restriction>
+ </xs:simpleType>
+
+ <xs:simpleType name="path">
+ <xs:restriction base="xs:string">
+ <xs:pattern value="/([A-Za-z0-9+_.-]+/)*[A-Za-z0-9+_.-]+"/>
+ </xs:restriction>
+ </xs:simpleType>
+
+ <xs:simpleType name="relpath">
+ <xs:restriction base="xs:string">
+ <xs:pattern value="([A-Za-z0-9+_.-]+/)*[A-Za-z0-9+_.-]+"/>
+ </xs:restriction>
+ </xs:simpleType>
+
+ <xs:element name="bootsync">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="partitions" maxOccurs="1" minOccurs="1">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="part" minOccurs="2" maxOccurs="unbounded">
+ <xs:complexType>
+ <xs:attribute name="path" type="diskdev" use="required"/>
+ <xs:attribute name="mount" type="path" use="required"/>
+ </xs:complexType>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+ <xs:unique name="partition_unique_mount">
+ <xs:selector xpath="bootsync:part"/>
+ <xs:field xpath="@mount"/>
+ </xs:unique>
+ <xs:unique name="partition_unique_path">
+ <xs:selector xpath="bootsync:part"/>
+ <xs:field xpath="@path"/>
+ </xs:unique>
+ </xs:element>
+ <xs:element name="fileChecks" maxOccurs="1" minOccurs="0">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="file" maxOccurs="unbounded" minOccurs="1">
+ <xs:complexType>
+ <xs:simpleContent>
+ <xs:extension base="relpath">
+ <xs:attribute name="isKernel" type="xs:boolean"
+ use="optional" default="false"/>
+ </xs:extension>
+ </xs:simpleContent>
+ </xs:complexType>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+ <xs:unique name="filechk_unique">
+ <xs:selector xpath="bootsync:file"/>
+ <xs:field xpath="."/>
+ </xs:unique>
+ </xs:element>
+ <xs:element name="syncPaths" maxOccurs="1" minOccurs="1">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="path" maxOccurs="unbounded" minOccurs="1">
+ <xs:complexType>
+ <xs:attribute name="source" type="path" use="required"/>
+ <xs:attribute name="target" type="relpath" use="required"/>
+ <!-- TODO: make this optional? -->
+ <xs:attribute name="pattern" type="xs:string" use="required"/>
+ </xs:complexType>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+ <xs:unique name="syncpath_unique_source">
+ <xs:selector xpath="bootsync:path"/>
+ <xs:field xpath="@source"/>
+ </xs:unique>
+ <xs:unique name="syncpath_unique_target">
+ <xs:selector xpath="bootsync:path"/>
+ <xs:field xpath="@target"/>
+ </xs:unique>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+</xs:schema> \ No newline at end of file
diff --git a/sys/BootSync/prep.txt b/sys/BootSync/prep.txt
new file mode 100644
index 0000000..6514763
--- /dev/null
+++ b/sys/BootSync/prep.txt
@@ -0,0 +1,56 @@
+PREPARATION:
+0.) Comment out all /boot mounts in /etc/fstab and umount /boot if mounted as a separate mountpoint.
+ You want to use *the /boot on your / mount*.
+
+1.) Prepare each target partition (partitions/part below) as an ESP
+ (https://wiki.archlinux.org/index.php/EFI_system_partition#Format_the_partition).
+
+2.) Install GRUB2 to *each ESP*. See sample.config.xml for context for the below examples.
+
+ grub-install \
+ --boot-directory=/mnt/boot1 \
+ --bootloader-id=Arch \
+ --efi-directory=/mnt/boot1/ \
+ --target=x86_64-efi \
+ --no-nvram \
+ --recheck
+
+ grub-install \
+ --boot-directory=/mnt/boot1 \
+ --bootloader-id="Arch (Fallback)" \
+ --efi-directory=/mnt/boot1/ \
+ --target=x86_64-efi \
+ --no-nvram \
+ --recheck
+
+ grub-install \
+ --boot-directory=/mnt/boot2 \
+ --bootloader-id=Arch \
+ --efi-directory=/mnt/boot2/ \
+ --target=x86_64-efi \
+ --no-nvram \
+ --recheck
+
+ grub-install \
+ --boot-directory=/mnt/boot2 \
+ --bootloader-id="Arch (Fallback)" \
+ --efi-directory=/mnt/boot2/ \
+ --target=x86_64-efi \
+ --no-nvram \
+ --recheck
+
+3.) Prepare the ESPs. See sample.config.xml for context for the below examples.
+
+ efibootmgr \
+ --create \
+ --disk /dev/sdd \
+ --part 1 \
+ --loader /EFI/Arch/grubx64.efi \
+ --label "Arch (Fallback)"
+
+ efibootmgr \
+ --create \
+ --disk /dev/sdb \
+ --part 1 \
+ --loader /EFI/Arch/grubx64.efi \
+ --label "Arch"
diff --git a/sys/BootSync/sample.config.xml b/sys/BootSync/sample.config.xml
new file mode 100644
index 0000000..b982ae9
--- /dev/null
+++ b/sys/BootSync/sample.config.xml
@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!-- This example assumes Grub2 and UEFI. It should be flexible enough to tweak for other use cases
+ if you know what the hell you're doing. -->
+<!--
+SEE prep.txt FOR WHAT MUST BE DONE BEFORE RUNNING BOOTSYNC.
+-->
+<bootsync xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns="http://git.square-r00t.net/OpTools/tree/sys/BootSync/"
+ xsi:schemaLocation="http://git.square-r00t.net/OpTools/plain/sys/BootSync/bootsync.xsd">
+ <!-- The actual EFI System Partitions (ESP). At least two are required. -->
+ <!-- Each drive should be prepared ahead of time with efibootmgr and grub-install (see prep.txt). -->
+ <partitions>
+ <!-- These should all be unique values.
+ Don't do something dumb like mount the same partition on the same mountpoint. -->
+ <!-- Future versions of this script may support things like UUIDs or labels as an alternative to path=,
+ but in the meanwhile it relies on device paths explicitly. -->
+ <part path="/dev/sdb1" mount="/mnt/boot1"/>
+ <part path="/dev/sdd1" mount="/mnt/boot2"/>
+ </partitions>
+ <!-- If specified, the files in this container are checksummed to determine if a sync is required.
+ If fileChecks isn't provided, we will always sync no matter what.
+ These files will be evaluated to and synced to the ESP partitions with the same relative path.
+ i.e. (part[@mount])/(file)
+
+ "isKernel" is by default false, but if true it is treated specially to get a kernel version.
+ This is to notify you if a reboot is required. If no <file> item is isKernel="true", no reboot requirement
+ detection will be done.
+ Only *the last* <file> with isKernel="true" listed is used as the kernel identifier. -->
+ <fileChecks>
+ <!-- RELATIVE paths to /boot -->
+ <file>initramfs-linux.img</file>
+ <file>intel-ucode.img</file>
+ <file>memtest86+/memtest.bin</file>
+ <file isKernel="true">vmlinuz-linux</file>
+ </fileChecks>
+ <!-- These are system paths to sync to the ESPs. They are synced recursively.
+ "source" should be absolute.
+ "target" should be relative (to the ESP mount).
+ "pattern" is a regex to match to restrict *filenames* to sync. Use ".*" for all files. -->
+ <syncPaths>
+ <!-- For example, these are grub theme files. -->
+ <path source="/usr/share/grub/themes" target="grub/themes" pattern=".*"/>
+ <!-- These are grub modules - specifically, UEFI. -->
+ <path source="/usr/lib/grub/x86_64-efi" target="grub/x86_64-efi" pattern="^.*\.(mod|lst|sh)$"/>
+ <!-- And these are ISO files, in case you're like me and do loop mounting in Grub for rescue situations.
+ (e.g. https://wiki.archlinux.org/index.php/Multiboot_USB_drive#Using_GRUB_and_loopback_devices) -->
+ <path source="/boot/iso" target="iso" pattern="^.*\.(iso|img)$"/>
+ </syncPaths>
+</bootsync> \ No newline at end of file