adding BootSync

This commit is contained in:
brent s 2019-08-18 20:24:39 -04:00
parent 3976fd631c
commit c149a7b3b7
4 changed files with 511 additions and 0 deletions

311
sys/BootSync/bootsync.py Executable file
View File

@ -0,0 +1,311 @@
#!/usr/bin/env python3

import argparse
import hashlib
import json
import os
import platform
import re
import shutil
import subprocess
##
import magic # From http://darwinsys.com/file/, not https://github.com/ahupp/python-magic
import psutil
from lxml import etree


# def get_file_kernel_ver(self, kpath):
# # Gets the version of a kernel file.
# kpath = os.path.abspath(os.path.expanduser(kpath))
# _kinfo = {}
# with open(kpath, 'rb') as f:
# _m = magic.detect_from_content(f.read())
# for i in _m.name.split(','):
# l = i.strip().split()
# # Note: this only grabs the version number.
# # If we want to get e.g. the build user/machine, date, etc.,
# # then we need to join l[1:].
# # We technically don't even need a dict, either. We can just iterate.
# # TODO.
# _kinfo[l[0].lower()] = (l[1] if len(l) > 1 else None)
# if 'version' not in _kinfo:
# raise RuntimeError('Cannot deterimine the version of {0}'.format(
# kpath))
# else:
# return (_kinfo['version'])

class BootSync(object):
def __init__(self, cfg = None, *args, **kwargs):
if not cfg:
self.cfgfile = '/etc/bootsync.xml'
else:
self.cfgfile = os.path.abspath(os.path.expanduser(cfg))
self.ns = '{http://git.square-r00t.net/OpTools/tree/sys/BootSync/}'
self.cfg = None
self.xml = None
# This is the current live kernel.
self.currentKernVer = self._getRunningKernel()
# This is the installed kernel from the package manager.
self.kernelFile = None
self.installedKernVer = None
self.RequireReboot = False # If a reboot is needed (WARN, don't execute!)
self.blkids = {}
self.dummy_uuid = None
self.syncs = {}
##
self.getCfg()
self.chkMounts()
self.chkReboot()
self.getHashes()
self.getBlkids()
# self.sync()
# self.writeConfs()

def getCfg(self):
if not os.path.isfile(self.cfgfile):
raise FileNotFoundError('Configuration file {0} does not exist!'.format(self.cfgfile))
try:
with open(self.cfgfile, 'rb') as f:
self.xml = etree.parse(f)
self.xml.xinclude()
self.cfg = self.xml.getroot()
except etree.XMLSyntaxError:
# self.logger.error('{0} is invalid XML'.format(self.cfgfile))
raise ValueError(('{0} does not seem to be valid XML. '
'See sample.config.xml for an example configuration.').format(self.cfgfile))
return()

def chkMounts(self):
_mounts = {m.device: m.mountpoint for m in psutil.disk_partitions(all = True)}
for esp in self.cfg.findall('{0}partitions/{0}part'.format(self.ns)):
disk = esp.attrib['path']
mount = os.path.abspath(os.path.expanduser(esp.attrib['mount']))
if not os.path.isdir(mount):
os.makedirs(mount, exist_ok = True)
if disk not in _mounts:
with open(os.devnull, 'w') as devnull:
c = subprocess.run(['/usr/bin/mount', mount],
stderr = devnull)
if c.returncode == 1: # Not specified in fstab
subprocess.run(['/usr/bin/mount', disk, mount],
stderr = devnull)
elif c.returncode == 32: # Already mounted
pass
return()

def chkReboot(self):
self._getInstalledKernel()
if not self.kernelFile:
return() # No isKernel="true" was specified in the config.
if self.installedKernVer != self.currentKernVer:
self.RequireReboot = True
# TODO: logger instead?
print(('NOTE: REBOOT REQUIRED. '
'New kernel is {0}. '
'Running kernel is {1}.').format(self.installedKernVer,
self.currentKernVer))
return()

def getBlkids(self):
c = subprocess.run(['/usr/bin/blkid',
'-o', 'export'],
stdout = subprocess.PIPE)
if c.returncode != 0:
raise RuntimeError('Could not fetch block ID information')
for p in c.stdout.decode('utf-8').split('\n\n'):
line = [i.strip() for i in p.splitlines()]
d = dict(map(lambda i: i.split('='), line))
if d.get('TYPE') == 'squashfs':
continue
self.blkids[d['DEVNAME']] = d.get('PARTUUID', d['UUID'])
c = subprocess.run(['/usr/bin/findmnt',
'--json',
'-T', '/boot'],
stdout = subprocess.PIPE)
# I write ridiculous one-liners.
self.dummy_uuid = self.blkids[json.loads(c.stdout.decode('utf-8'))['filesystems'][0]['source']]
return()

def getHashes(self):
def _get_hash(fpathname):
fpathname = os.path.abspath(os.path.expanduser(fpathname))
_hash = hashlib.sha512()
with open(fpathname, 'rb') as fh:
_hash.update(fh.read())
return(_hash.hexdigest())
for f in self.cfg.findall('{0}fileChecks/{0}file'):
# We do /boot files manually in case it isn't specified as a
# separate mount.
rel_fpath = f.text
fpath = os.path.join('/boot', rel_fpath)
canon_hash = _get_hash(fpath)
for esp in self.cfg.findall('{0}partitions/{0}part'.format(self.ns)):
mount = os.path.abspath(os.path.expanduser(esp.attrib['mount']))
new_fpath = os.path.join(mount, f)
file_hash = _get_hash(new_fpath)
if file_hash != canon_hash:
if rel_fpath not in self.syncs:
self.syncs[rel_fpath] = []
self.syncs[rel_fpath].append(mount)
return()

def sync(self, dryrun = False, *args, **kwargs):
if not dryrun:
if os.geteuid() != 0:
raise PermissionError('You must be root to write to the appropriate destinations')
for f in self.syncs:
for m in self.syncs[f]:
orig = os.path.join('/boot', f)
dest = os.path.join(m, f)
if not dryrun:
shutil.copy2(orig, dest)
bootmounts = [e.attrib['mount'] for e in self.cfg.findall('{0}partitions/{0}part'.format(self.ns))]
# syncPaths
for syncpath in self.cfg.findall('{0}syncPaths/{0}path'.format(self.ns)):
source = os.path.abspath(os.path.expanduser(syncpath.attrib['source']))
target = syncpath.attrib['target']
pattern = syncpath.attrib['pattern']
# We don't use filecmp for this because:
# - dircmp doesn't recurse
# - the reports/lists don't retain relative paths
# - we can't regex out files
for root, dirs, files in os.walk(source):
prefix = re.sub('\/?{0}\/?'.format(source), '', root)
ptrn = re.compile(pattern)
for f in files:
fname_path = os.path.join(prefix, f)
bootsource = os.path.join(source, fname_path)
boottarget = os.path.join(target, fname_path)
if ptrn.search(f):
# Compare the contents.
with open(bootsource, 'rb') as fh:
orig_hash = hashlib.sha512(fh.read()).hexdigest()
for bootdir in bootmounts:
bootfile = os.path.join(bootdir, boottarget)
if not dryrun:
if not os.path.isfile(bootfile):
os.makedirs(os.path.dirname(bootfile),
exist_ok = True)
shutil.copy2(bootsource, bootfile)
else:
with open(bootfile, 'rb') as fh:
dest_hash = hashlib.sha512(fh.read()).hexdigest()
if orig_hash != dest_hash:
shutil.copy2(bootsource, bootfile)
# fileChecks are a *lot* easier.
for f in self.cfg.findall('{0}fileChecks/{0}file'.format(self.ns)):
source = os.path.join('/boot', f.text)
with open(source, 'rb') as fh:
orig_hash = hashlib.sha512(fh.read()).hexdigest()
for bootdir in bootmounts:
bootfile = os.path.join(bootdir, f.text)
if not dryrun:
if not os.path.isfile(bootfile):
os.makedirs(os.path.dirname(bootfile),
exist_ok = True)
shutil.copy2(source, bootfile)
else:
with open(bootfile, 'rb') as fh:
dest_hash = hashlib.sha512(fh.read()).hexdigest()
if orig_hash != dest_hash:
shutil.copy2(source, bootfile)
return()


def writeConfs(self, dryrun = False, *args, **kwargs):
if not dryrun:
if os.geteuid() != 0:
raise PermissionError('You must be root to write to the appropriate destinations')
else:
return()
# Get a fresh config in place.
with open(os.devnull, 'wb') as DEVNULL:
c = subprocess.run(['/usr/bin/grub-mkconfig',
'-o', '/boot/grub/grub.cfg'],
stdout = DEVNULL,
stderr = DEVNULL)
if c.returncode != 0:
raise RuntimeError('An error occurred when generating the GRUB configuration file.')
with open('/boot/grub/grub.cfg', 'r') as f:
_grubcfg = f.read()
for esp in self.cfg.findall('{0}partitions/{0}part'.format(self.ns)):
mount = os.path.abspath(os.path.expanduser(esp.attrib['mount']))
disk = os.path.abspath(os.path.expanduser(esp.attrib['path']))
with open(os.path.join(mount, 'grub/grub.cfg'), 'w') as f:
for line in _grubcfg.splitlines():
# if re.search(r'^\s*search\s+(.*)\s(-u|--fs-uuid)', line):
# pass
i = re.sub(r'(?<!\=UUID\=){0}'.format(self.dummy_uuid),
self.blkids[disk],
line)
# If the array is in a degraded state, this will still let us at LEAST boot.
i = re.sub(r'\s+--hint=[\'"]?mduuid/[a-f0-9]{32}[\'"]?', '', i)
f.write('{0}\n'.format(i))
return()

def _getRunningKernel(self):
_vers = []
# If we change the version string capture in get_file_kernel_ver(),
# this will need to be expanded as well.
# Really we only need to pick one, but #YOLO; why not sanity-check.
# ALL of these should match, hence the reduction with set() down to (what SHOULD be) just 1 item.
_vers.append(os.uname().release)
_vers.append(platform.release())
_vers.append(platform.uname().release)
_vers = sorted(list(set(_vers)))
if len(_vers) != 1:
raise RuntimeError('Cannot reliably determine current running kernel version!')
else:
return(_vers[0])

def _getInstalledKernel(self):
# Could we maybe remove the dependency for the "magic" module with a struct?
# http://lxr.linux.no/#linux+v2.6.39/Documentation/x86/boot.txt
# https://stackoverflow.com/a/11179559/733214
try:
len(self.cfg)
except TypeError:
raise RuntimeError('Tried to find the isKernel with no config set up and parsed')
for f in self.cfg.findall('{0}fileChecks/{0}file'.format(self.ns)):
isKernel = (True
if f.attrib.get('isKernel', 'false').lower() in ('true', '1')
else
False)
if isKernel:
self.kernelFile = f.text
if self.kernelFile:
with open(os.path.join('/boot', self.kernelFile), 'rb') as fh:
magicname = magic.detect_from_content(fh.read())
names = [i.strip().split(None, 1) for i in magicname.name.split(',') if i.strip() != '']
for n in names:
if len(n) != 2:
continue
k, v = n
# Note: this only grabs the version number.
# If we want to get e.g. the build user/machine, date, etc.,
# then we need to do a join. Shouldn't be necessary, though.
if k.lower() == 'version':
self.installedKernVer = v.split(None, 1)[0]
return()

def parseArgs():
args = argparse.ArgumentParser(description = ('Sync files to assist using mdadm RAID arrays with UEFI'))
args.add_argument('-c', '--cfg',
dest = 'cfg',
default = '/etc/bootsync.xml',
help = ('The path to the bootsync configuration file. Default is /etc/bootsync.xml'))
args.add_argument('-n', '--dry-run',
dest = 'dryrun',
action = 'store_true',
help = ('If specified, don\'t write any changes'))
return(args)

def main():
args = vars(parseArgs().parse_args())
bs = BootSync(**args)
bs.sync(**args)
bs.writeConfs(**args)
return()

if __name__ == '__main__':
main()

95
sys/BootSync/bootsync.xsd Normal file
View File

@ -0,0 +1,95 @@
<?xml version="1.0" encoding="UTF-8" ?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
targetNamespace="http://git.square-r00t.net/OpTools/tree/sys/BootSync/"
xmlns="http://git.square-r00t.net/OpTools/tree/sys/BootSync/"
xmlns:bootsync="http://git.square-r00t.net/OpTools/tree/sys/BootSync/"
elementFormDefault="qualified"
attributeFormDefault="unqualified">

<xs:simpleType name="diskdev">
<xs:restriction base="xs:string">
<xs:pattern value="/dev/([A-Za-z0-9+_-]+/)?[A-Za-z0-9+_-]+[0-9]?"/>
</xs:restriction>
</xs:simpleType>

<xs:simpleType name="path">
<xs:restriction base="xs:string">
<xs:pattern value="/([A-Za-z0-9+_.-]+/)*[A-Za-z0-9+_.-]+"/>
</xs:restriction>
</xs:simpleType>

<xs:simpleType name="relpath">
<xs:restriction base="xs:string">
<xs:pattern value="([A-Za-z0-9+_.-]+/)*[A-Za-z0-9+_.-]+"/>
</xs:restriction>
</xs:simpleType>

<xs:element name="bootsync">
<xs:complexType>
<xs:sequence>
<xs:element name="partitions" maxOccurs="1" minOccurs="1">
<xs:complexType>
<xs:sequence>
<xs:element name="part" minOccurs="2" maxOccurs="unbounded">
<xs:complexType>
<xs:attribute name="path" type="diskdev" use="required"/>
<xs:attribute name="mount" type="path" use="required"/>
</xs:complexType>
</xs:element>
</xs:sequence>
</xs:complexType>
<xs:unique name="partition_unique_mount">
<xs:selector xpath="bootsync:part"/>
<xs:field xpath="@mount"/>
</xs:unique>
<xs:unique name="partition_unique_path">
<xs:selector xpath="bootsync:part"/>
<xs:field xpath="@path"/>
</xs:unique>
</xs:element>
<xs:element name="fileChecks" maxOccurs="1" minOccurs="0">
<xs:complexType>
<xs:sequence>
<xs:element name="file" maxOccurs="unbounded" minOccurs="1">
<xs:complexType>
<xs:simpleContent>
<xs:extension base="relpath">
<xs:attribute name="isKernel" type="xs:boolean"
use="optional" default="false"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
</xs:element>
</xs:sequence>
</xs:complexType>
<xs:unique name="filechk_unique">
<xs:selector xpath="bootsync:file"/>
<xs:field xpath="."/>
</xs:unique>
</xs:element>
<xs:element name="syncPaths" maxOccurs="1" minOccurs="1">
<xs:complexType>
<xs:sequence>
<xs:element name="path" maxOccurs="unbounded" minOccurs="1">
<xs:complexType>
<xs:attribute name="source" type="path" use="required"/>
<xs:attribute name="target" type="relpath" use="required"/>
<!-- TODO: make this optional? -->
<xs:attribute name="pattern" type="xs:string" use="required"/>
</xs:complexType>
</xs:element>
</xs:sequence>
</xs:complexType>
<xs:unique name="syncpath_unique_source">
<xs:selector xpath="bootsync:path"/>
<xs:field xpath="@source"/>
</xs:unique>
<xs:unique name="syncpath_unique_target">
<xs:selector xpath="bootsync:path"/>
<xs:field xpath="@target"/>
</xs:unique>
</xs:element>
</xs:sequence>
</xs:complexType>
</xs:element>
</xs:schema>

56
sys/BootSync/prep.txt Normal file
View File

@ -0,0 +1,56 @@
PREPARATION:
0.) Comment out all /boot mounts in /etc/fstab and umount /boot if mounted as a separate mountpoint.
You want to use *the /boot on your / mount*.

1.) Prepare each target partition (partitions/part below) as an ESP
(https://wiki.archlinux.org/index.php/EFI_system_partition#Format_the_partition).

2.) Install GRUB2 to *each ESP*. See sample.config.xml for context for the below examples.

grub-install \
--boot-directory=/mnt/boot1 \
--bootloader-id=Arch \
--efi-directory=/mnt/boot1/ \
--target=x86_64-efi \
--no-nvram \
--recheck

grub-install \
--boot-directory=/mnt/boot1 \
--bootloader-id="Arch (Fallback)" \
--efi-directory=/mnt/boot1/ \
--target=x86_64-efi \
--no-nvram \
--recheck

grub-install \
--boot-directory=/mnt/boot2 \
--bootloader-id=Arch \
--efi-directory=/mnt/boot2/ \
--target=x86_64-efi \
--no-nvram \
--recheck

grub-install \
--boot-directory=/mnt/boot2 \
--bootloader-id="Arch (Fallback)" \
--efi-directory=/mnt/boot2/ \
--target=x86_64-efi \
--no-nvram \
--recheck

3.) Prepare the ESPs. See sample.config.xml for context for the below examples.

efibootmgr \
--create \
--disk /dev/sdd \
--part 1 \
--loader /EFI/Arch/grubx64.efi \
--label "Arch (Fallback)"

efibootmgr \
--create \
--disk /dev/sdb \
--part 1 \
--loader /EFI/Arch/grubx64.efi \
--label "Arch"

View File

@ -0,0 +1,49 @@
<?xml version="1.0" encoding="UTF-8" ?>
<!-- This example assumes Grub2 and UEFI. It should be flexible enough to tweak for other use cases
if you know what the hell you're doing. -->
<!--
SEE prep.txt FOR WHAT MUST BE DONE BEFORE RUNNING BOOTSYNC.
-->
<bootsync xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://git.square-r00t.net/OpTools/tree/sys/BootSync/"
xsi:schemaLocation="http://git.square-r00t.net/OpTools/plain/sys/BootSync/bootsync.xsd">
<!-- The actual EFI System Partitions (ESP). At least two are required. -->
<!-- Each drive should be prepared ahead of time with efibootmgr and grub-install (see prep.txt). -->
<partitions>
<!-- These should all be unique values.
Don't do something dumb like mount the same partition on the same mountpoint. -->
<!-- Future versions of this script may support things like UUIDs or labels as an alternative to path=,
but in the meanwhile it relies on device paths explicitly. -->
<part path="/dev/sdb1" mount="/mnt/boot1"/>
<part path="/dev/sdd1" mount="/mnt/boot2"/>
</partitions>
<!-- If specified, the files in this container are checksummed to determine if a sync is required.
If fileChecks isn't provided, we will always sync no matter what.
These files will be evaluated to and synced to the ESP partitions with the same relative path.
i.e. (part[@mount])/(file)

"isKernel" is by default false, but if true it is treated specially to get a kernel version.
This is to notify you if a reboot is required. If no <file> item is isKernel="true", no reboot requirement
detection will be done.
Only *the last* <file> with isKernel="true" listed is used as the kernel identifier. -->
<fileChecks>
<!-- RELATIVE paths to /boot -->
<file>initramfs-linux.img</file>
<file>intel-ucode.img</file>
<file>memtest86+/memtest.bin</file>
<file isKernel="true">vmlinuz-linux</file>
</fileChecks>
<!-- These are system paths to sync to the ESPs. They are synced recursively.
"source" should be absolute.
"target" should be relative (to the ESP mount).
"pattern" is a regex to match to restrict *filenames* to sync. Use ".*" for all files. -->
<syncPaths>
<!-- For example, these are grub theme files. -->
<path source="/usr/share/grub/themes" target="grub/themes" pattern=".*"/>
<!-- These are grub modules - specifically, UEFI. -->
<path source="/usr/lib/grub/x86_64-efi" target="grub/x86_64-efi" pattern="^.*\.(mod|lst|sh)$"/>
<!-- And these are ISO files, in case you're like me and do loop mounting in Grub for rescue situations.
(e.g. https://wiki.archlinux.org/index.php/Multiboot_USB_drive#Using_GRUB_and_loopback_devices) -->
<path source="/boot/iso" target="iso" pattern="^.*\.(iso|img)$"/>
</syncPaths>
</bootsync>