diff --git a/aif/disk/__init__.py b/aif/disk/__init__.py index f65e5eb..7b01166 100644 --- a/aif/disk/__init__.py +++ b/aif/disk/__init__.py @@ -1,5 +1,23 @@ -from . import block -from . import filesystem -from . import luks -from . import lvm -from . import mdadm +try: + from . import block +except ImportError: + from . import block_fallback as block +try: + from . import filesystem_fallback +except ImportError: + from . import filesystem_fallback as filesystem + +try: + from . import luks_fallback +except ImportError: + from . import luks_fallback as luks + +try: + from . import lvm_fallback +except ImportError: + from . import lvm_fallback as lvm + +try: + from . import mdadm_fallback +except ImportError: + from . import mdadm_fallback as mdadm diff --git a/aif/disk/_common.py b/aif/disk/_common.py new file mode 100644 index 0000000..02ef2c7 --- /dev/null +++ b/aif/disk/_common.py @@ -0,0 +1,9 @@ +import gi +gi.require_version('BlockDev', '2.0') +from gi.repository import BlockDev, GLib + +ps = BlockDev.PluginSpec() +ps.name = BlockDev.Plugin.LVM +ps.so_name = "libbd_lvm.so" + +BlockDev.init([ps]) diff --git a/aif/disk/block.py b/aif/disk/block.py index 15357e7..4736de8 100644 --- a/aif/disk/block.py +++ b/aif/disk/block.py @@ -1,225 +1,3 @@ -# To reproduce sgdisk behaviour in v1 of AIF-NG: -# https://gist.github.com/herry13/5931cac426da99820de843477e41e89e -# https://github.com/dcantrell/pyparted/blob/master/examples/query_device_capacity.py -# TODO: Remember to replicate genfstab behaviour. +from . import _common -import os -import re -import subprocess -try: - # https://stackoverflow.com/a/34812552/733214 - # https://github.com/karelzak/util-linux/blob/master/libmount/python/test_mount_context.py#L6 - import libmount as mount -except ImportError: - # We should never get here. util-linux is part of core (base) in Arch and uses "libmount". - import pylibmount as mount -## -import blkinfo -import parted # https://www.gnu.org/software/parted/api/index.html -import psutil -## -from aif.utils import xmlBool, size - -# TODO: https://serverfault.com/questions/356534/ssd-erase-block-size-lvm-pv-on-raw-device-alignment - - -PARTED_FSTYPES = sorted(list(dict(vars(parted.filesystem))['fileSystemType'].keys())) -PARTED_FLAGS = sorted(list(parted.partition.partitionFlag.values())) -IDX_FLAG = dict(parted.partition.partitionFlag) -FLAG_IDX = {v: k for k, v in IDX_FLAG.items()} - -# parted lib can do SI or IEC. So can we. -_pos_re = re.compile((r'^(?P-|\+)?\s*' - r'(?P[0-9]+)\s*' - # empty means size in sectors - r'(?P%|{0}|)\s*$'.format('|'.join(size.valid_storage)) - )) - - -def convertSizeUnit(pos): - orig_pos = pos - pos = _pos_re.search(pos) - if pos: - pos_or_neg = (pos.group('pos_or_neg') if pos.group('pos_or_neg') else None) - if pos_or_neg == '+': - from_beginning = True - elif pos_or_neg == '-': - from_beginning = False - else: - from_beginning = pos_or_neg - _size = int(pos.group('size')) - amt_type = pos.group('pct_unit_or_sct').strip() - else: - raise ValueError('Invalid size specified: {0}'.format(orig_pos)) - return((from_beginning, _size, amt_type)) - - -class Partition(object): - def __init__(self, part_xml, diskobj, start_sector, partnum, tbltype, part_type = None): - if tbltype not in ('gpt', 'msdos'): - raise ValueError('{0} must be one of gpt or msdos'.format(tbltype)) - if tbltype == 'msdos' and part_type not in ('primary', 'extended', 'logical'): - raise ValueError(('You must specify if this is a ' - 'primary, extended, or logical partition for msdos partition tables')) - self.xml = part_xml - self.id = part_xml.attrib['id'] - self.flags = set() - for f in self.xml.findall('partitionFlag'): - if f.text in PARTED_FLAGS: - self.flags.add(f.text) - self.flags = sorted(list(self.flags)) - self.partnum = partnum - if tbltype == 'msdos': - if partnum > 4: - self.part_type = parted.PARTITION_LOGICAL - else: - if part_type == 'extended': - self.part_type = parted.PARTITION_EXTENDED - elif part_type == 'logical': - self.part_type = parted.PARTITION_LOGICAL - else: - self.part_type = parted.PARTITION_NORMAL - self.fstype = self.xml.attrib['fsType'].lower() - if self.fstype not in PARTED_FSTYPES: - raise ValueError(('{0} is not a valid partition filesystem type; ' - 'must be one of: {1}').format(self.xml.attrib['fsType'], - ', '.join(sorted(PARTED_FSTYPES)))) - self.disk = diskobj - self.device = self.disk.device - self.devpath = '{0}{1}'.format(self.device.path, partnum) - self.is_hiformatted = False - sizes = {} - for s in ('start', 'stop'): - x = dict(zip(('from_bgn', 'size', 'type'), - convertSizeUnit(self.xml.attrib[s]))) - sectors = x['size'] - if x['type'] == '%': - sectors = int(self.device.getLength() / x['size']) - else: - sectors = int(size.convertStorage(x['size'], x['type'], target = 'B') / self.device.sectorSize) - sizes[s] = (sectors, x['from_bgn']) - if sizes['start'][1] is not None: - if sizes['start'][1]: - self.begin = sizes['start'][0] + 0 - else: - self.begin = self.device.getLength() - sizes['start'][0] - else: - self.begin = sizes['start'][0] + start_sector - if sizes['stop'][1] is not None: - if sizes['stop'][1]: - self.end = sizes['stop'][0] + 0 - else: - # This *technically* should be - 34, at least for gpt, but the alignment optimizer fixes it for us. - self.end = (self.device.getLength() - 1) - sizes['stop'][0] - else: - self.end = self.begin + sizes['stop'][0] - # TECHNICALLY we could craft the Geometry object with "length = ...", but it doesn't let us be explicit - # in configs. So we manually crunch the numbers and do it all at the end. - self.geometry = parted.Geometry(device = self.device, - start = self.begin, - end = self.end) - self.filesystem = parted.FileSystem(type = self.fstype, - geometry = self.geometry) - self.partition = parted.Partition(disk = diskobj, - type = self.part_type, - geometry = self.geometry, - fs = self.filesystem) - for f in self.flags[:]: - flag_id = FLAG_IDX[f] - if self.partition.isFlagAvailable(flag_id): - self.partition.setFlag(flag_id) - else: - self.flags.remove(f) - if tbltype == 'gpt' and self.xml.attrib.get('name'): - # The name attribute setting is b0rk3n, so we operate on the underlying PedPartition object. - # https://github.com/dcantrell/pyparted/issues/49#issuecomment-540096687 - # https://github.com/dcantrell/pyparted/issues/65 - # self.partition.name = self.xml.attrib.get('name') - _pedpart = self.partition.getPedPartition() - _pedpart.set_name(self.xml.attrib.get('name')) - - def detect(self): - pass - - -class Disk(object): - def __init__(self, disk_xml): - self.xml = disk_xml - self.devpath = self.xml.attrib['device'] - self._initDisk() - - def _initDisk(self): - self.tabletype = self.xml.attrib.get('diskFormat', 'gpt').lower() - if self.tabletype in ('bios', 'mbr', 'dos'): - self.tabletype = 'msdos' - validlabels = parted.getLabels() - if self.tabletype not in validlabels: - raise ValueError(('Disk format {0} is not valid for this architecture;' - 'must be one of: {1}'.format(self.tabletype, ', '.join(list(validlabels))))) - self.device = parted.getDevice(self.devpath) - self.disk = parted.freshDisk(self.device, self.tabletype) - self.is_lowformatted = False - self.is_hiformatted = False - self.is_partitioned = False - self.partitions = [] - return() - - def diskFormat(self): - if self.is_lowformatted: - return() - # This is a safeguard. We do *not* want to low-format a disk that is mounted. - for p in psutil.disk_partitions(all = True): - if self.devpath in p: - raise RuntimeError('{0} is mounted; we are cowardly refusing to low-format it'.format(self.devpath)) - self.disk.deleteAllPartitions() - self.disk.commit() - self.is_lowformatted = True - self.is_partitioned = False - return() - - def getPartitions(self): - # For GPT, this *technically* should be 34 -- or, more precisely, 2048 (see FAQ in manual), but the alignment - # optimizer fixes it for us automatically. - # But for DOS tables, it's required. - if self.tabletype == 'msdos': - start_sector = 2048 - else: - start_sector = 0 - self.partitions = [] - xml_partitions = self.xml.findall('part') - for idx, part in enumerate(xml_partitions): - partnum = idx + 1 - if self.tabletype == 'gpt': - p = Partition(part, self.disk, start_sector, partnum, self.tabletype) - else: - parttype = 'primary' - if len(xml_partitions) > 4: - if partnum == 4: - parttype = 'extended' - elif partnum > 4: - parttype = 'logical' - p = Partition(part, self.disk, start_sector, partnum, self.tabletype, part_type = parttype) - start_sector = p.end + 1 - self.partitions.append(p) - return() - - def partFormat(self): - if self.is_partitioned: - return() - if not self.is_lowformatted: - self.diskFormat() - # This is a safeguard. We do *not* want to partition a disk that is mounted. - for p in psutil.disk_partitions(all = True): - if self.devpath in p: - raise RuntimeError('{0} is mounted; we are cowardly refusing to low-format it'.format(self.devpath)) - if not self.partitions: - self.getPartitions() - if not self.partitions: - return() - for p in self.partitions: - self.disk.addPartition(partition = p, constraint = self.device.optimalAlignedConstraint) - self.disk.commit() - p.devpath = p.partition.path - p.is_hiformatted = True - self.is_partitioned = True - return() +BlockDev = _common.BlockDev diff --git a/aif/disk/block_fallback.py b/aif/disk/block_fallback.py new file mode 100644 index 0000000..fb04e92 --- /dev/null +++ b/aif/disk/block_fallback.py @@ -0,0 +1,225 @@ +# To reproduce sgdisk behaviour in v1 of AIF-NG: +# https://gist.github.com/herry13/5931cac426da99820de843477e41e89e +# https://github.com/dcantrell/pyparted/blob/master/examples/query_device_capacity.py +# TODO: Remember to replicate genfstab behaviour. + +import os +import re +import subprocess +try: + # https://stackoverflow.com/a/34812552/733214 + # https://github.com/karelzak/util-linux/blob/master/libmount/python/test_mount_context.py#L6 + import libmount as mount +except ImportError: + # We should never get here. util-linux is part of core (base) in Arch and uses "libmount". + import pylibmount as mount +## +# import blkinfo +import parted # https://www.gnu.org/software/parted/api/index.html +import psutil +## +from aif.utils import xmlBool, size + +# TODO: https://serverfault.com/questions/356534/ssd-erase-block-size-lvm-pv-on-raw-device-alignment + + +PARTED_FSTYPES = sorted(list(dict(vars(parted.filesystem))['fileSystemType'].keys())) +PARTED_FLAGS = sorted(list(parted.partition.partitionFlag.values())) +IDX_FLAG = dict(parted.partition.partitionFlag) +FLAG_IDX = {v: k for k, v in IDX_FLAG.items()} + +# parted lib can do SI or IEC. So can we. +_pos_re = re.compile((r'^(?P-|\+)?\s*' + r'(?P[0-9]+)\s*' + # empty means size in sectors + r'(?P%|{0}|)\s*$'.format('|'.join(size.valid_storage)) + )) + + +def convertSizeUnit(pos): + orig_pos = pos + pos = _pos_re.search(pos) + if pos: + pos_or_neg = (pos.group('pos_or_neg') if pos.group('pos_or_neg') else None) + if pos_or_neg == '+': + from_beginning = True + elif pos_or_neg == '-': + from_beginning = False + else: + from_beginning = pos_or_neg + _size = int(pos.group('size')) + amt_type = pos.group('pct_unit_or_sct').strip() + else: + raise ValueError('Invalid size specified: {0}'.format(orig_pos)) + return((from_beginning, _size, amt_type)) + + +class Partition(object): + def __init__(self, part_xml, diskobj, start_sector, partnum, tbltype, part_type = None): + if tbltype not in ('gpt', 'msdos'): + raise ValueError('{0} must be one of gpt or msdos'.format(tbltype)) + if tbltype == 'msdos' and part_type not in ('primary', 'extended', 'logical'): + raise ValueError(('You must specify if this is a ' + 'primary, extended, or logical partition for msdos partition tables')) + self.xml = part_xml + self.id = part_xml.attrib['id'] + self.flags = set() + for f in self.xml.findall('partitionFlag'): + if f.text in PARTED_FLAGS: + self.flags.add(f.text) + self.flags = sorted(list(self.flags)) + self.partnum = partnum + if tbltype == 'msdos': + if partnum > 4: + self.part_type = parted.PARTITION_LOGICAL + else: + if part_type == 'extended': + self.part_type = parted.PARTITION_EXTENDED + elif part_type == 'logical': + self.part_type = parted.PARTITION_LOGICAL + else: + self.part_type = parted.PARTITION_NORMAL + self.fstype = self.xml.attrib['fsType'].lower() + if self.fstype not in PARTED_FSTYPES: + raise ValueError(('{0} is not a valid partition filesystem type; ' + 'must be one of: {1}').format(self.xml.attrib['fsType'], + ', '.join(sorted(PARTED_FSTYPES)))) + self.disk = diskobj + self.device = self.disk.device + self.devpath = '{0}{1}'.format(self.device.path, partnum) + self.is_hiformatted = False + sizes = {} + for s in ('start', 'stop'): + x = dict(zip(('from_bgn', 'size', 'type'), + convertSizeUnit(self.xml.attrib[s]))) + sectors = x['size'] + if x['type'] == '%': + sectors = int(self.device.getLength() / x['size']) + else: + sectors = int(size.convertStorage(x['size'], x['type'], target = 'B') / self.device.sectorSize) + sizes[s] = (sectors, x['from_bgn']) + if sizes['start'][1] is not None: + if sizes['start'][1]: + self.begin = sizes['start'][0] + 0 + else: + self.begin = self.device.getLength() - sizes['start'][0] + else: + self.begin = sizes['start'][0] + start_sector + if sizes['stop'][1] is not None: + if sizes['stop'][1]: + self.end = sizes['stop'][0] + 0 + else: + # This *technically* should be - 34, at least for gpt, but the alignment optimizer fixes it for us. + self.end = (self.device.getLength() - 1) - sizes['stop'][0] + else: + self.end = self.begin + sizes['stop'][0] + # TECHNICALLY we could craft the Geometry object with "length = ...", but it doesn't let us be explicit + # in configs. So we manually crunch the numbers and do it all at the end. + self.geometry = parted.Geometry(device = self.device, + start = self.begin, + end = self.end) + self.filesystem = parted.FileSystem(type = self.fstype, + geometry = self.geometry) + self.partition = parted.Partition(disk = diskobj, + type = self.part_type, + geometry = self.geometry, + fs = self.filesystem) + for f in self.flags[:]: + flag_id = FLAG_IDX[f] + if self.partition.isFlagAvailable(flag_id): + self.partition.setFlag(flag_id) + else: + self.flags.remove(f) + if tbltype == 'gpt' and self.xml.attrib.get('name'): + # The name attribute setting is b0rk3n, so we operate on the underlying PedPartition object. + # https://github.com/dcantrell/pyparted/issues/49#issuecomment-540096687 + # https://github.com/dcantrell/pyparted/issues/65 + # self.partition.name = self.xml.attrib.get('name') + _pedpart = self.partition.getPedPartition() + _pedpart.set_name(self.xml.attrib.get('name')) + + def detect(self): + pass + + +class Disk(object): + def __init__(self, disk_xml): + self.xml = disk_xml + self.devpath = self.xml.attrib['device'] + self._initDisk() + + def _initDisk(self): + self.tabletype = self.xml.attrib.get('diskFormat', 'gpt').lower() + if self.tabletype in ('bios', 'mbr', 'dos'): + self.tabletype = 'msdos' + validlabels = parted.getLabels() + if self.tabletype not in validlabels: + raise ValueError(('Disk format {0} is not valid for this architecture;' + 'must be one of: {1}'.format(self.tabletype, ', '.join(list(validlabels))))) + self.device = parted.getDevice(self.devpath) + self.disk = parted.freshDisk(self.device, self.tabletype) + self.is_lowformatted = False + self.is_hiformatted = False + self.is_partitioned = False + self.partitions = [] + return() + + def diskFormat(self): + if self.is_lowformatted: + return() + # This is a safeguard. We do *not* want to low-format a disk that is mounted. + for p in psutil.disk_partitions(all = True): + if self.devpath in p: + raise RuntimeError('{0} is mounted; we are cowardly refusing to low-format it'.format(self.devpath)) + self.disk.deleteAllPartitions() + self.disk.commit() + self.is_lowformatted = True + self.is_partitioned = False + return() + + def getPartitions(self): + # For GPT, this *technically* should be 34 -- or, more precisely, 2048 (see FAQ in manual), but the alignment + # optimizer fixes it for us automatically. + # But for DOS tables, it's required. + if self.tabletype == 'msdos': + start_sector = 2048 + else: + start_sector = 0 + self.partitions = [] + xml_partitions = self.xml.findall('part') + for idx, part in enumerate(xml_partitions): + partnum = idx + 1 + if self.tabletype == 'gpt': + p = Partition(part, self.disk, start_sector, partnum, self.tabletype) + else: + parttype = 'primary' + if len(xml_partitions) > 4: + if partnum == 4: + parttype = 'extended' + elif partnum > 4: + parttype = 'logical' + p = Partition(part, self.disk, start_sector, partnum, self.tabletype, part_type = parttype) + start_sector = p.end + 1 + self.partitions.append(p) + return() + + def partFormat(self): + if self.is_partitioned: + return() + if not self.is_lowformatted: + self.diskFormat() + # This is a safeguard. We do *not* want to partition a disk that is mounted. + for p in psutil.disk_partitions(all = True): + if self.devpath in p: + raise RuntimeError('{0} is mounted; we are cowardly refusing to low-format it'.format(self.devpath)) + if not self.partitions: + self.getPartitions() + if not self.partitions: + return() + for p in self.partitions: + self.disk.addPartition(partition = p, constraint = self.device.optimalAlignedConstraint) + self.disk.commit() + p.devpath = p.partition.path + p.is_hiformatted = True + self.is_partitioned = True + return() diff --git a/aif/disk/filesystem.py b/aif/disk/filesystem.py index b614ebb..4736de8 100644 --- a/aif/disk/filesystem.py +++ b/aif/disk/filesystem.py @@ -1,80 +1,3 @@ -import os -import re -import subprocess -## -import psutil -## -import aif.disk.block -import aif.disk.luks -import aif.disk.lvm -import aif.disk.mdadm +from . import _common -# I wish there was a better way of doing this. -# https://unix.stackexchange.com/a/98680 -FS_FSTYPES = [] -with open('/proc/filesystems', 'r') as fh: - for line in fh.readlines(): - l = [i.strip() for i in line.split()] - if not l: - continue - if len(l) == 1: - FS_FSTYPES.append(l[0]) - else: - FS_FSTYPES.append(l[1]) -_mod_dir = os.path.join('/lib/modules', - os.uname().release, - 'kernel/fs') -_strip_mod_suffix = re.compile(r'(?P)\.ko(\.(x|g)?z)?$', re.IGNORECASE) -for i in os.listdir(_mod_dir): - path = os.path.join(_mod_dir, i) - fs_name = None - if os.path.isdir(path): - fs_name = i - elif os.path.isfile(path): - mod_name = _strip_mod_suffix.search(i) - fs_name = mod_name.group('fsname') - if fs_name: - # The kernel *probably* has autoloading enabled, but in case it doesn't... - # TODO: logging! - if os.getuid() == 0: - subprocess.run(['modprobe', fs_name]) - FS_FSTYPES.append(fs_name) - - -class FS(object): - def __init__(self, fs_xml, sourceobj): - self.xml = fs_xml - if not isinstance(sourceobj, (aif.disk.block.Disk, - aif.disk.block.Partition, - aif.disk.luks.LUKS, - aif.disk.lvm.LV, - aif.disk.mdadm.Array)): - raise ValueError(('sourceobj must be of type ' - 'aif.disk.block.Partition, ' - 'aif.disk.luks.LUKS, ' - 'aif.disk.lvm.LV, or' - 'aif.disk.mdadm.Array')) - self.source = sourceobj - self.devpath = sourceobj.devpath - self.formatted = False - self.fstype = self.xml.attrib.get('type') - - def format(self): - if self.formatted: - return () - # This is a safeguard. We do *not* want to high-format a disk that is mounted. - for p in psutil.disk_partitions(all = True): - if self.devpath in p: - raise RuntimeError(('{0} is mounted;' - 'we are cowardly refusing to apply a filesystem to it').format(self.devpath)) - # TODO! Logging - cmd = ['mkfs', - '-t', self.fstype] - for o in self.xml.findall('opt'): - cmd.append(o.attrib['name']) - if o.text: - cmd.append(o.text) - cmd.append(self.devpath) - subprocess.run(cmd) - self.is_hiformatted = True - return() +BlockDev = _common.BlockDev diff --git a/aif/disk/filesystem_fallback.py b/aif/disk/filesystem_fallback.py new file mode 100644 index 0000000..7fb8b47 --- /dev/null +++ b/aif/disk/filesystem_fallback.py @@ -0,0 +1,80 @@ +import os +import re +import subprocess +## +import psutil +## +import aif.disk.block_fallback as block +import aif.disk.luks_fallback as luks +import aif.disk.lvm_fallback as lvm +import aif.disk.mdadm_fallback as mdadm + +# I wish there was a better way of doing this. +# https://unix.stackexchange.com/a/98680 +FS_FSTYPES = [] +with open('/proc/filesystems', 'r') as fh: + for line in fh.readlines(): + l = [i.strip() for i in line.split()] + if not l: + continue + if len(l) == 1: + FS_FSTYPES.append(l[0]) + else: + FS_FSTYPES.append(l[1]) +_mod_dir = os.path.join('/lib/modules', + os.uname().release, + 'kernel/fs') +_strip_mod_suffix = re.compile(r'(?P)\.ko(\.(x|g)?z)?$', re.IGNORECASE) +for i in os.listdir(_mod_dir): + path = os.path.join(_mod_dir, i) + fs_name = None + if os.path.isdir(path): + fs_name = i + elif os.path.isfile(path): + mod_name = _strip_mod_suffix.search(i) + fs_name = mod_name.group('fsname') + if fs_name: + # The kernel *probably* has autoloading enabled, but in case it doesn't... + # TODO: logging! + if os.getuid() == 0: + subprocess.run(['modprobe', fs_name]) + FS_FSTYPES.append(fs_name) + + +class FS(object): + def __init__(self, fs_xml, sourceobj): + self.xml = fs_xml + if not isinstance(sourceobj, (aif.disk.block_fallback.Disk, + aif.disk.block_fallback.Partition, + aif.disk.luks_fallback.LUKS, + aif.disk.lvm_fallback.LV, + aif.disk.mdadm_fallback.Array)): + raise ValueError(('sourceobj must be of type ' + 'aif.disk.block.Partition, ' + 'aif.disk.luks.LUKS, ' + 'aif.disk.lvm.LV, or' + 'aif.disk.mdadm.Array')) + self.source = sourceobj + self.devpath = sourceobj.devpath + self.formatted = False + self.fstype = self.xml.attrib.get('type') + + def format(self): + if self.formatted: + return () + # This is a safeguard. We do *not* want to high-format a disk that is mounted. + for p in psutil.disk_partitions(all = True): + if self.devpath in p: + raise RuntimeError(('{0} is mounted;' + 'we are cowardly refusing to apply a filesystem to it').format(self.devpath)) + # TODO! Logging + cmd = ['mkfs', + '-t', self.fstype] + for o in self.xml.findall('opt'): + cmd.append(o.attrib['name']) + if o.text: + cmd.append(o.text) + cmd.append(self.devpath) + subprocess.run(cmd) + self.is_hiformatted = True + return() diff --git a/aif/disk/luks.py b/aif/disk/luks.py index 9670f13..4736de8 100644 --- a/aif/disk/luks.py +++ b/aif/disk/luks.py @@ -1,9 +1,3 @@ -import aif.disk.block -import aif.disk.lvm -import aif.disk.mdadm +from . import _common - -class LUKS(object): - def __init__(self, partobj): - self.devpath = None - pass +BlockDev = _common.BlockDev diff --git a/aif/disk/luks_fallback.py b/aif/disk/luks_fallback.py new file mode 100644 index 0000000..5e7a526 --- /dev/null +++ b/aif/disk/luks_fallback.py @@ -0,0 +1,9 @@ +import aif.disk.block_fallback as block +import aif.disk.lvm_fallback as lvm +import aif.disk.mdadm_fallback as mdadm + + +class LUKS(object): + def __init__(self, partobj): + self.devpath = None + pass diff --git a/aif/disk/lvm.py b/aif/disk/lvm.py index 5334ace..4736de8 100644 --- a/aif/disk/lvm.py +++ b/aif/disk/lvm.py @@ -1,29 +1,3 @@ -try: - import dbus - has_mod = True -except ImportError: - # This is ineffecient; the native dbus module is preferred. - # In Arch, this can be installed via the 'extra' repository package "python-dbus". - import subprocess - has_mod = False -## -import aif.disk.block -import aif.disk.luks -import aif.disk.mdadm +from . import _common - -class PV(object): - def __init__(self, partobj): - self.devpath = None - pass - - -class VG(object): - def __init__(self, vg_xml, lv_objs): - self.devpath = None - pass - - -class LV(object): - def __init__(self, lv_xml, pv_objs): - pass +BlockDev = _common.BlockDev diff --git a/aif/disk/lvm_fallback.py b/aif/disk/lvm_fallback.py new file mode 100644 index 0000000..10799bc --- /dev/null +++ b/aif/disk/lvm_fallback.py @@ -0,0 +1,31 @@ +try: + import gi + gi.require_version('BlockDev', '2.0') + from gi.repository import BlockDev, GLib + has_mod = True +except ImportError: + # This is ineffecient; the native gobject-introspection module is preferred. + # In Arch, this can be installed via the "extra" repository packages "libblockdev" and "python-gobject". + import subprocess + has_mod = False +## +import aif.disk.block_fallback as block +import aif.disk.luks_fallback as luks +import aif.disk.mdadm_fallback as mdadm + + +class PV(object): + def __init__(self, partobj): + self.devpath = None + pass + + +class VG(object): + def __init__(self, vg_xml, lv_objs): + self.devpath = None + pass + + +class LV(object): + def __init__(self, lv_xml, pv_objs): + pass diff --git a/aif/disk/mdadm.py b/aif/disk/mdadm.py index aecc299..4736de8 100644 --- a/aif/disk/mdadm.py +++ b/aif/disk/mdadm.py @@ -1,260 +1,3 @@ -import copy -import datetime -import math -import re -import subprocess -import uuid -## -import mdstat -## -import aif.disk.block -import aif.disk.luks -import aif.disk.lvm +from . import _common - -SUPPORTED_LEVELS = (0, 1, 4, 5, 6, 10) -SUPPORTED_METADATA = ('0', '0.90', '1', '1.0', '1.1', '1.2', 'default', 'ddf', 'imsm') -SUPPORTED_LAYOUTS = {5: (re.compile(r'^((left|right)-a?symmetric|[lr][as]|' - r'parity-(fir|la)st|' - r'ddf-(N|zero)-restart|ddf-N-continue)$'), - 'left-symmetric'), - 6: (re.compile(r'^((left|right)-a?symmetric(-6)?|[lr][as]|' - r'parity-(fir|la)st|' - r'ddf-(N|zero)-restart|ddf-N-continue|' - r'parity-first-6)$'), - None), - 10: (re.compile(r'^[nof][0-9]+$'), - None)} - -_mdblock_size_re = re.compile(r'^(?P[0-9]+)\s+' - r'\((?P[0-9.]+)\s+GiB\s+' - r'(?P[0-9.]+)\s+GB\)') -_mdblock_unused_re = re.compile(r'^before=(?P[0-9]+)\s+sectors,' - r'\s+after=(?P[0-9]+)\s+sectors$') -_mdblock_badblock_re = re.compile(r'^(?P[0-9]+)\s+entries' - r'[A-Za-z\s]+' - r'(?P[0-9]+)\s+sectors$') - -def _itTakesTwo(n): - # So dumb. - isPowerOf2 = math.ceil(math.log(n, 2)) == math.floor(math.log(n, 2)) - return(isPowerOf2) - -def _safeChunks(n): - if (n % 4) != 0: - return(False) - return(True) - - -class Member(object): - def __init__(self, member_xml, partobj): - self.xml = member_xml - self.device = partobj - if not isinstance(self.device, (aif.disk.block.Partition, - aif.disk.block.Disk, - aif.disk.mdadm.Array, - aif.disk.lvm.LV, - aif.disk.luks.LUKS)): - raise ValueError(('partobj must be of type aif.disk.block.Partition, ' - 'aif.disk.block.Disk, or aif.disk.mdadm.Array')) - self.devpath = self.device.devpath - self.is_superblocked = None - self.superblock = None - self._parseDeviceBlock() - - def _parseDeviceBlock(self): - # I can't believe the mdstat module doesn't really have a way to do this. - super = subprocess.run(['mdadm', '--examine', self.devpath], - stdout = subprocess.PIPE, - stderr = subprocess.PIPE) - if super.returncode != 0: - # TODO: logging? - self.is_superblocked = False - return(None) - block = {} - for idx, line in enumerate(super.stdout.decode('utf-8').splitlines()): - line = line.strip() - if idx == 0: # This is just the same as self.device.devpath. - continue - if line == '': - continue - k, v = [i.strip() for i in line.split(':', 1)] - orig_k = k - k = re.sub(r'\s+', '_', k.lower()) - if k in ('raid_devices', 'events'): - v = int(v) - elif k == 'magic': - v = bytes.fromhex(v) - elif k == 'name': - # TODO: Will this *always* give 2 values? - name, local_to = [i.strip() for i in v.split(None, 1)] - local_to = re.sub(r'[()]', '', local_to) - v = (name, local_to) - elif k == 'raid_level': - v = re.sub(r'^raid', '', v) - elif k == 'checksum': - cksum, status = [i.strip() for i in v.split('-')] - v = (bytes.fromhex(cksum), status) - elif k == 'unused_space': - r = _mdblock_unused_re.search(v) - if not r: - raise ValueError(('Could not parse {0} for ' - '{1}\'s superblock').format(orig_k, - self.devpath)) - v = {} - for i in ('before', 'after'): - v[i] = int(r.group(i)) # in sectors - elif k == 'bad_block_log': - k = 'badblock_log_entries' - r = _mdblock_badblock_re.search(v) - if not r: - raise ValueError(('Could not parse {0} for ' - '{1}\'s superblock').format(orig_k, - self.devpath)) - v = {} - for i in ('entries', 'offset'): - v[i] = int(r.group(i)) # offset is in sectors - elif k == 'array_state': - v = [i.strip() for i in v.split(None, 1)][0].split() - elif k == 'device_uuid': - v = uuid.UUID(hex = v.replace(':', '-')) - elif re.search((r'^(creation|update)_time$'), k): - # TODO: Is this portable/correct? Or do I need to do '%a %b %d %H:%M:%s %Y'? - v = datetime.datetime.strptime(v, '%c') - elif re.search(r'^((avail|used)_dev|array)_size$', k): - r = _mdblock_size_re.search(v) - if not r: - raise ValueError(('Could not parse {0} for ' - '{1}\'s superblock').format(orig_k, - self.devpath)) - v = {} - for i in ('sectors', 'GB', 'GiB'): - v[i] = float(r.group(i)) - if i == 'sectors': - v[i] = int(v[i]) - elif re.search(r'^(data|super)_offset$', k): - v = int(v.split(None, 1)[0]) - block[k] = v - self.superblock = block - self.is_superblocked = True - return() - - def prepare(self): - if self.is_superblocked: - # TODO: logging - subprocess.run(['mdadm', '--misc', '--zero-superblock', self.devpath]) - self.is_superblocked = False - return() - -class Array(object): - def __init__(self, array_xml, homehost, devpath = None): - self.xml = array_xml - self.id = array_xml.attrib['id'] - self.level = int(self.xml.attrib['level']) - if self.level not in SUPPORTED_LEVELS: - raise ValueError('RAID level must be one of: {0}'.format(', '.join([str(i) for i in SUPPORTED_LEVELS]))) - self.metadata = self.xml.attrib.get('meta', '1.2') - if self.metadata not in SUPPORTED_METADATA: - raise ValueError('Metadata version must be one of: {0}'.format(', '.join(SUPPORTED_METADATA))) - self.chunksize = int(self.xml.attrib.get('chunkSize', 512)) - if self.level in (4, 5, 6, 10): - if not _itTakesTwo(self.chunksize): - # TODO: log.warn instead of raise exception? Will mdadm lose its marbles if it *isn't* a proper number? - raise ValueError('chunksize must be a power of 2 for the RAID level you specified') - if self.level in (0, 4, 5, 6, 10): - if not _safeChunks(self.chunksize): - # TODO: log.warn instead of raise exception? Will mdadm lose its marbles if it *isn't* a proper number? - raise ValueError('chunksize must be divisible by 4 for the RAID level you specified') - self.layout = self.xml.attrib.get('layout', 'none') - if self.level in SUPPORTED_LAYOUTS.keys(): - matcher, layout_default = SUPPORTED_LAYOUTS[self.level] - if not matcher.search(self.layout): - if layout_default: - self.layout = layout_default - else: - self.layout = None # TODO: log.warn? - else: - self.layout = None - self.devname = self.xml.attrib['name'] - self.devpath = devpath - self.updateStatus() - self.homehost = homehost - self.members = [] - self.state = None - self.info = None - - def addMember(self, memberobj): - if not isinstance(memberobj, Member): - raise ValueError('memberobj must be of type aif.disk.mdadm.Member') - memberobj.prepare() - self.members.append(memberobj) - return() - - def start(self, scan = False): - if not any((self.members, self.devpath)): - raise RuntimeError('Cannot assemble an array with no members (for hints) or device path') - cmd = ['mdadm', '--assemble', self.devpath] - if not scan: - for m in self.members: - cmd.append(m.devpath) - else: - cmd.append('--scan') - # TODO: logging! - subprocess.run(cmd) - self.state = 'assembled' - return() - - def create(self): - if not self.members: - raise RuntimeError('Cannot create an array with no members') - cmd = ['mdadm', '--create', - '--level={0}'.format(self.level), - '--metadata={0}'.format(self.metadata), - '--chunk={0}'.format(self.chunksize), - '--homehost={0}'.format(self.homehost), - '--raid-devices={0}'.format(len(self.members))] - if self.layout: - cmd.append('--layout={0}'.format(self.layout)) - cmd.append(self.devpath) - for m in self.members: - cmd.append(m.devpath) - # TODO: logging! - subprocess.run(cmd) - self.writeConf() - self.state = 'new' - return() - - def stop(self): - # TODO: logging - subprocess.run(['mdadm', '--stop', self.devpath]) - self.state = 'disassembled' - return() - - def updateStatus(self): - _info = mdstat.parse() - for k, v in _info['devices'].items(): - if k != self.devname: - del(_info['devices'][k]) - self.info = copy.deepcopy(_info) - return() - - def writeConf(self, conf = '/etc/mdadm.conf'): - with open(conf, 'r') as fh: - conflines = fh.read().splitlines() - # TODO: logging - arrayinfo = subprocess.run(['mdadm', '--detail', '--brief', self.devpath], - stdout = subprocess.PIPE).stdout.decode('utf-8').strip() - if arrayinfo not in conflines: - r = re.compile(r'^ARRAY\s+{0}'.format(self.devpath)) - nodev = True - for l in conflines: - if r.search(l): - nodev = False - # TODO: logging? - # and/or Raise an exception here; - # an array already exists with that name but not with the same opts/GUID/etc. - break - if nodev: - with open(conf, 'a') as fh: - fh.write('{0}\n'.format(arrayinfo)) - return() +BlockDev = _common.BlockDev diff --git a/aif/disk/mdadm_fallback.py b/aif/disk/mdadm_fallback.py new file mode 100644 index 0000000..fd6e417 --- /dev/null +++ b/aif/disk/mdadm_fallback.py @@ -0,0 +1,260 @@ +import copy +import datetime +import math +import re +import subprocess +import uuid +## +import mdstat +## +import aif.disk.block_fallback as block +import aif.disk.luks_fallback as luks +import aif.disk.lvm_fallback as lvm + + +SUPPORTED_LEVELS = (0, 1, 4, 5, 6, 10) +SUPPORTED_METADATA = ('0', '0.90', '1', '1.0', '1.1', '1.2', 'default', 'ddf', 'imsm') +SUPPORTED_LAYOUTS = {5: (re.compile(r'^((left|right)-a?symmetric|[lr][as]|' + r'parity-(fir|la)st|' + r'ddf-(N|zero)-restart|ddf-N-continue)$'), + 'left-symmetric'), + 6: (re.compile(r'^((left|right)-a?symmetric(-6)?|[lr][as]|' + r'parity-(fir|la)st|' + r'ddf-(N|zero)-restart|ddf-N-continue|' + r'parity-first-6)$'), + None), + 10: (re.compile(r'^[nof][0-9]+$'), + None)} + +_mdblock_size_re = re.compile(r'^(?P[0-9]+)\s+' + r'\((?P[0-9.]+)\s+GiB\s+' + r'(?P[0-9.]+)\s+GB\)') +_mdblock_unused_re = re.compile(r'^before=(?P[0-9]+)\s+sectors,' + r'\s+after=(?P[0-9]+)\s+sectors$') +_mdblock_badblock_re = re.compile(r'^(?P[0-9]+)\s+entries' + r'[A-Za-z\s]+' + r'(?P[0-9]+)\s+sectors$') + +def _itTakesTwo(n): + # So dumb. + isPowerOf2 = math.ceil(math.log(n, 2)) == math.floor(math.log(n, 2)) + return(isPowerOf2) + +def _safeChunks(n): + if (n % 4) != 0: + return(False) + return(True) + + +class Member(object): + def __init__(self, member_xml, partobj): + self.xml = member_xml + self.device = partobj + if not isinstance(self.device, (block.Partition, + block.Disk, + Array, + lvm.LV, + luks.LUKS)): + raise ValueError(('partobj must be of type aif.disk.block.Partition, ' + 'aif.disk.block.Disk, or aif.disk.mdadm.Array')) + self.devpath = self.device.devpath + self.is_superblocked = None + self.superblock = None + self._parseDeviceBlock() + + def _parseDeviceBlock(self): + # I can't believe the mdstat module doesn't really have a way to do this. + super = subprocess.run(['mdadm', '--examine', self.devpath], + stdout = subprocess.PIPE, + stderr = subprocess.PIPE) + if super.returncode != 0: + # TODO: logging? + self.is_superblocked = False + return(None) + block = {} + for idx, line in enumerate(super.stdout.decode('utf-8').splitlines()): + line = line.strip() + if idx == 0: # This is just the same as self.device.devpath. + continue + if line == '': + continue + k, v = [i.strip() for i in line.split(':', 1)] + orig_k = k + k = re.sub(r'\s+', '_', k.lower()) + if k in ('raid_devices', 'events'): + v = int(v) + elif k == 'magic': + v = bytes.fromhex(v) + elif k == 'name': + # TODO: Will this *always* give 2 values? + name, local_to = [i.strip() for i in v.split(None, 1)] + local_to = re.sub(r'[()]', '', local_to) + v = (name, local_to) + elif k == 'raid_level': + v = re.sub(r'^raid', '', v) + elif k == 'checksum': + cksum, status = [i.strip() for i in v.split('-')] + v = (bytes.fromhex(cksum), status) + elif k == 'unused_space': + r = _mdblock_unused_re.search(v) + if not r: + raise ValueError(('Could not parse {0} for ' + '{1}\'s superblock').format(orig_k, + self.devpath)) + v = {} + for i in ('before', 'after'): + v[i] = int(r.group(i)) # in sectors + elif k == 'bad_block_log': + k = 'badblock_log_entries' + r = _mdblock_badblock_re.search(v) + if not r: + raise ValueError(('Could not parse {0} for ' + '{1}\'s superblock').format(orig_k, + self.devpath)) + v = {} + for i in ('entries', 'offset'): + v[i] = int(r.group(i)) # offset is in sectors + elif k == 'array_state': + v = [i.strip() for i in v.split(None, 1)][0].split() + elif k == 'device_uuid': + v = uuid.UUID(hex = v.replace(':', '-')) + elif re.search((r'^(creation|update)_time$'), k): + # TODO: Is this portable/correct? Or do I need to do '%a %b %d %H:%M:%s %Y'? + v = datetime.datetime.strptime(v, '%c') + elif re.search(r'^((avail|used)_dev|array)_size$', k): + r = _mdblock_size_re.search(v) + if not r: + raise ValueError(('Could not parse {0} for ' + '{1}\'s superblock').format(orig_k, + self.devpath)) + v = {} + for i in ('sectors', 'GB', 'GiB'): + v[i] = float(r.group(i)) + if i == 'sectors': + v[i] = int(v[i]) + elif re.search(r'^(data|super)_offset$', k): + v = int(v.split(None, 1)[0]) + block[k] = v + self.superblock = block + self.is_superblocked = True + return() + + def prepare(self): + if self.is_superblocked: + # TODO: logging + subprocess.run(['mdadm', '--misc', '--zero-superblock', self.devpath]) + self.is_superblocked = False + return() + +class Array(object): + def __init__(self, array_xml, homehost, devpath = None): + self.xml = array_xml + self.id = array_xml.attrib['id'] + self.level = int(self.xml.attrib['level']) + if self.level not in SUPPORTED_LEVELS: + raise ValueError('RAID level must be one of: {0}'.format(', '.join([str(i) for i in SUPPORTED_LEVELS]))) + self.metadata = self.xml.attrib.get('meta', '1.2') + if self.metadata not in SUPPORTED_METADATA: + raise ValueError('Metadata version must be one of: {0}'.format(', '.join(SUPPORTED_METADATA))) + self.chunksize = int(self.xml.attrib.get('chunkSize', 512)) + if self.level in (4, 5, 6, 10): + if not _itTakesTwo(self.chunksize): + # TODO: log.warn instead of raise exception? Will mdadm lose its marbles if it *isn't* a proper number? + raise ValueError('chunksize must be a power of 2 for the RAID level you specified') + if self.level in (0, 4, 5, 6, 10): + if not _safeChunks(self.chunksize): + # TODO: log.warn instead of raise exception? Will mdadm lose its marbles if it *isn't* a proper number? + raise ValueError('chunksize must be divisible by 4 for the RAID level you specified') + self.layout = self.xml.attrib.get('layout', 'none') + if self.level in SUPPORTED_LAYOUTS.keys(): + matcher, layout_default = SUPPORTED_LAYOUTS[self.level] + if not matcher.search(self.layout): + if layout_default: + self.layout = layout_default + else: + self.layout = None # TODO: log.warn? + else: + self.layout = None + self.devname = self.xml.attrib['name'] + self.devpath = devpath + self.updateStatus() + self.homehost = homehost + self.members = [] + self.state = None + self.info = None + + def addMember(self, memberobj): + if not isinstance(memberobj, Member): + raise ValueError('memberobj must be of type aif.disk.mdadm.Member') + memberobj.prepare() + self.members.append(memberobj) + return() + + def start(self, scan = False): + if not any((self.members, self.devpath)): + raise RuntimeError('Cannot assemble an array with no members (for hints) or device path') + cmd = ['mdadm', '--assemble', self.devpath] + if not scan: + for m in self.members: + cmd.append(m.devpath) + else: + cmd.append('--scan') + # TODO: logging! + subprocess.run(cmd) + self.state = 'assembled' + return() + + def create(self): + if not self.members: + raise RuntimeError('Cannot create an array with no members') + cmd = ['mdadm', '--create', + '--level={0}'.format(self.level), + '--metadata={0}'.format(self.metadata), + '--chunk={0}'.format(self.chunksize), + '--homehost={0}'.format(self.homehost), + '--raid-devices={0}'.format(len(self.members))] + if self.layout: + cmd.append('--layout={0}'.format(self.layout)) + cmd.append(self.devpath) + for m in self.members: + cmd.append(m.devpath) + # TODO: logging! + subprocess.run(cmd) + self.writeConf() + self.state = 'new' + return() + + def stop(self): + # TODO: logging + subprocess.run(['mdadm', '--stop', self.devpath]) + self.state = 'disassembled' + return() + + def updateStatus(self): + _info = mdstat.parse() + for k, v in _info['devices'].items(): + if k != self.devname: + del(_info['devices'][k]) + self.info = copy.deepcopy(_info) + return() + + def writeConf(self, conf = '/etc/mdadm.conf'): + with open(conf, 'r') as fh: + conflines = fh.read().splitlines() + # TODO: logging + arrayinfo = subprocess.run(['mdadm', '--detail', '--brief', self.devpath], + stdout = subprocess.PIPE).stdout.decode('utf-8').strip() + if arrayinfo not in conflines: + r = re.compile(r'^ARRAY\s+{0}'.format(self.devpath)) + nodev = True + for l in conflines: + if r.search(l): + nodev = False + # TODO: logging? + # and/or Raise an exception here; + # an array already exists with that name but not with the same opts/GUID/etc. + break + if nodev: + with open(conf, 'a') as fh: + fh.write('{0}\n'.format(arrayinfo)) + return()