r00t2/aif-ng
1
0
Fork 0
Code Issues Pull Requests Projects Releases Activity

filesystem and mounting done, other minor tweaks. need to do lvm, mdadm, luks

Browse Source
This commit is contained in:
brent s 2019-11-06 02:21:04 -05:00
parent 33ea96d1e1
commit d4de31dd67
8 changed files with 492 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
aif.xsd
View File

@ -262,6 +262,7 @@
</xs:simpleContent> </xs:simpleContent>
</xs:complexType> </xs:complexType>


<!-- TODO: "swap" shouldn't be here. Split off into a different type. -->
<xs:simpleType name="t_filepath"> <xs:simpleType name="t_filepath">
<xs:restriction base="xs:string"> <xs:restriction base="xs:string">
<xs:pattern value="((/[^/]+)+/?|swap)"/> <xs:pattern value="((/[^/]+)+/?|swap)"/>
@ -384,6 +385,7 @@
</xs:complexType> </xs:complexType>
</xs:element> </xs:element>
</xs:sequence> </xs:sequence>
<xs:attribute name="id" type="xs:ID" use="required"/>
<xs:attribute name="device" type="aif:t_diskdev" use="required"/> <xs:attribute name="device" type="aif:t_diskdev" use="required"/>
<xs:attribute name="diskFormat" type="aif:t_diskfmt" use="required"/> <xs:attribute name="diskFormat" type="aif:t_diskfmt" use="required"/>
</xs:complexType> </xs:complexType>
@ -406,6 +408,7 @@
<xs:element name="opt" minOccurs="0" maxOccurs="unbounded" <xs:element name="opt" minOccurs="0" maxOccurs="unbounded"
type="aif:t_cmdopts"/> type="aif:t_cmdopts"/>
</xs:sequence> </xs:sequence>
<xs:attribute name="id" type="xs:ID" use="required"/>
<xs:attribute name="source" type="xs:IDREF" use="required"/> <xs:attribute name="source" type="xs:IDREF" use="required"/>
<!-- We validate this in-code because there's way too many and <!-- We validate this in-code because there's way too many and
it's way too variable per-host. --> it's way too variable per-host. -->

39
aif/constants_fallback.py
View File

@ -19,6 +19,44 @@ EXTERNAL_DEPS = ['blkinfo',
'validators'] 'validators']
# PARTED FLAG INDEXING # PARTED FLAG INDEXING
PARTED_FSTYPES = sorted(list(dict(vars(parted.filesystem))['fileSystemType'].keys())) PARTED_FSTYPES = sorted(list(dict(vars(parted.filesystem))['fileSystemType'].keys()))
PARTED_FSTYPES_GUIDS = {'affs0': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'affs1': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'affs2': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'affs3': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'affs4': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'affs5': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'affs6': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'affs7': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'amufs': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'amufs0': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'amufs1': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'amufs2': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'amufs3': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'amufs4': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'amufs5': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'apfs1': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'apfs2': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'asfs': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'btrfs': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'ext2': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'ext3': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'ext4': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'fat16': uuid.UUID(hex = 'EBD0A0A2-B9E5-4433-87C0-68B6B72699C7'),
'fat32': uuid.UUID(hex = 'EBD0A0A2-B9E5-4433-87C0-68B6B72699C7'),
'hfs': uuid.UUID(hex = '48465300-0000-11AA-AA11-00306543ECAC'),
'hfs+': uuid.UUID(hex = '48465300-0000-11AA-AA11-00306543ECAC'),
'hfsx': uuid.UUID(hex = '48465300-0000-11AA-AA11-00306543ECAC'),
'hp-ufs': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'jfs': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'linux-swap(v0)': uuid.UUID(hex = '0657FD6D-A4AB-43C4-84E5-0933C84B4F4F'),
'linux-swap(v1)': uuid.UUID(hex = '0657FD6D-A4AB-43C4-84E5-0933C84B4F4F'),
'nilfs2': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'ntfs': uuid.UUID(hex = 'EBD0A0A2-B9E5-4433-87C0-68B6B72699C7'),
'reiserfs': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'sun-ufs': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'swsusp': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4'),
'udf': uuid.UUID(hex = 'EBD0A0A2-B9E5-4433-87C0-68B6B72699C7'),
'xfs': uuid.UUID(hex = '0FC63DAF-8483-4772-8E79-3D69D8477DE4')}
PARTED_FLAGS = sorted(list(parted.partition.partitionFlag.values())) PARTED_FLAGS = sorted(list(parted.partition.partitionFlag.values()))
PARTED_IDX_FLAG = dict(parted.partition.partitionFlag) PARTED_IDX_FLAG = dict(parted.partition.partitionFlag)
PARTED_FLAG_IDX = {v: k for k, v in PARTED_IDX_FLAG.items()} PARTED_FLAG_IDX = {v: k for k, v in PARTED_IDX_FLAG.items()}
@ -125,6 +163,7 @@ GPT_FSTYPE_GUIDS = ((1, 'EFI System', uuid.UUID(hex = 'C12A7328-F81F-11D2-BA4B-0
(87, 'Plan 9 partition', uuid.UUID(hex = 'C91818F9-8025-47AF-89D2-F030D7000C2C')), (87, 'Plan 9 partition', uuid.UUID(hex = 'C91818F9-8025-47AF-89D2-F030D7000C2C')),
(88, 'HiFive Unleashed FSBL', uuid.UUID(hex = '5B193300-FC78-40CD-8002-E86C45580B47')), (88, 'HiFive Unleashed FSBL', uuid.UUID(hex = '5B193300-FC78-40CD-8002-E86C45580B47')),
(89, 'HiFive Unleashed BBL', uuid.UUID(hex = '2E54B353-1271-4842-806F-E436D6AF6985'))) (89, 'HiFive Unleashed BBL', uuid.UUID(hex = '2E54B353-1271-4842-806F-E436D6AF6985')))
GPT_GUID_IDX = {k[2]: k[0] for k in GPT_FSTYPE_GUIDS}
# MSDOS FSTYPES IDENTIFIERS # MSDOS FSTYPES IDENTIFIERS
# Second verse, same as the first - kind of. The msdos type identifers just use a byte identifier rather than UUID. # Second verse, same as the first - kind of. The msdos type identifers just use a byte identifier rather than UUID.
# https://git.kernel.org/pub/scm/utils/util-linux/util-linux.git/plain/include/pt-mbr-partnames.h # https://git.kernel.org/pub/scm/utils/util-linux/util-linux.git/plain/include/pt-mbr-partnames.h

220
aif/disk/block.py
View File

@ -1,6 +1,7 @@
import re import os
import uuid
## ##
import parted import blkinfo
import psutil # Do I need this if I can have libblockdev's mounts API? Is there a way to get current mounts? import psutil # Do I need this if I can have libblockdev's mounts API? Is there a way to get current mounts?
## ##
import aif.constants import aif.constants
@ -10,7 +11,7 @@ from . import _common


_BlockDev = _common.BlockDev _BlockDev = _common.BlockDev



# TODO: LOGGING!
class Partition(object): class Partition(object):
def __init__(self, part_xml, diskobj, start_sector, partnum, tbltype, part_type = None): def __init__(self, part_xml, diskobj, start_sector, partnum, tbltype, part_type = None):
# Belive it or not, dear reader, but this *entire method* is just to set attributes. # Belive it or not, dear reader, but this *entire method* is just to set attributes.
@ -32,7 +33,49 @@ class Partition(object):
self.part_type = _BlockDev.PartTypeReq.EXTENDED self.part_type = _BlockDev.PartTypeReq.EXTENDED
elif part_type == 'logical': elif part_type == 'logical':
self.part_type = _BlockDev.PartTypeReq.LOGICAL self.part_type = _BlockDev.PartTypeReq.LOGICAL
elif tbltype == 'gpt':
self.part_type = _BlockDev.PartTypeReq.NORMAL
self.flags = [] self.flags = []
self.partnum = partnum
self.fs_type = self.xml.attrib['fsType']
self.disk = diskobj
self.device = self.disk.path
self.devpath = '{0}{1}'.format(self.device, self.partnum)
self.is_hiformatted = False
sizes = {}
for s in ('start', 'stop'):
x = dict(zip(('from_bgn', 'size', 'type'),
aif.utils.convertSizeUnit(self.xml.attrib[s])))
sectors = x['size']
if x['type'] == '%':
sectors = int(int(self.disk.size / self.disk.sector_size) * (0.01 * x['size']))
else:
sectors = int(aif.utils.size.convertStorage(x['size'],
x['type'],
target = 'B') / self.disk.sector_size)
sizes[s] = (sectors, x['from_bgn'])
if sizes['start'][1] is not None:
if sizes['start'][1]:
self.begin = sizes['start'][0] + 0
else:
self.begin = int(self.disk.size / self.disk.sector_size) - sizes['start'][0]
else:
self.begin = sizes['start'][0] + start_sector
if sizes['stop'][1] is not None:
if sizes['stop'][1]:
self.end = sizes['stop'][0] + 0
else:
# This *technically* should be - 34, at least for gpt, but the alignment optimizer fixes it for us.
self.end = (int(self.disk.size / self.disk.sector_size) - 1) - sizes['stop'][0]
else:
self.end = self.begin + sizes['stop'][0]
self.size = (self.end - self.begin)
self.part_name = self.xml.attrib.get('name')
self.partition = None
self._initFlags()
self._initFstype()

def _initFlags(self):
for f in self.xml.findall('partitionFlag'): for f in self.xml.findall('partitionFlag'):
# *Technically* we could use e.g. getattr(_BlockDev.PartFlag, f.text.upper()), *but* we lose compat # *Technically* we could use e.g. getattr(_BlockDev.PartFlag, f.text.upper()), *but* we lose compat
# with parted's flags if we do that. :| So we do some funky logic both here and in the constants. # with parted's flags if we do that. :| So we do some funky logic both here and in the constants.
@ -43,56 +86,40 @@ class Partition(object):
else: else:
continue continue
self.flags.append(_BlockDev.PartFlag(flag_id)) self.flags.append(_BlockDev.PartFlag(flag_id))
self.partnum = partnum return()
self.fstype = self.xml.attrib['fsType'].lower()
if self.fstype not in aif.constants.PARTED_FSTYPES: # There isn't a way to do this with BlockDev? :| def _initFstype(self):
raise ValueError(('{0} is not a valid partition filesystem type; ' _err = ('{0} is not a valid partition filesystem type; '
'must be one of: {1}').format(self.xml.attrib['fsType'], 'must be one of {1} or an fdisk-compatible GPT GUID').format(
', '.join(sorted(aif.constants.PARTED_FSTYPES)))) self.xml.attrib['fsType'],
self.disk = diskobj ', '.join(sorted(aif.constants.PARTED_FSTYPES)))
self.device = self.disk.path if self.fs_type in aif.constants.PARTED_FSTYPES_GUIDS.keys():
self.devpath = '{0}{1}'.format(self.device.path, partnum) self.fs_type = aif.constants.PARTED_FSTYPES_GUIDS[self.fs_type]
self.is_hiformatted = False
sizes = {}
for s in ('start', 'stop'):
x = dict(zip(('from_bgn', 'size', 'type'),
aif.utils.convertSizeUnit(self.xml.attrib[s])))
sectors = x['size']
if x['type'] == '%':
sectors = int(self.device.getLength() / x['size'])
else:
sectors = int(aif.utils.size.convertStorage(x['size'],
x['type'],
target = 'B') / self.device.sectorSize)
sizes[s] = (sectors, x['from_bgn'])
if sizes['start'][1] is not None:
if sizes['start'][1]:
self.begin = sizes['start'][0] + 0
else:
self.begin = self.device.getLength() - sizes['start'][0] # TODO: is there a way to get this in BD?
else: else:
self.begin = sizes['start'][0] + start_sector try:
if sizes['stop'][1] is not None: self.fs_type = uuid.UUID(hex = self.fs_type)
if sizes['stop'][1]: except ValueError:
self.end = sizes['stop'][0] + 0 raise ValueError(_err)
else: if self.fs_type not in aif.constants.GPT_GUID_IDX.keys():
# This *technically* should be - 34, at least for gpt, but the alignment optimizer fixes it for us. raise ValueError(_err)
self.end = (self.device.getLength() - 1) - sizes['stop'][0] # TODO: is there a way to get this in BD? return()
else:
self.end = self.begin + sizes['stop'][0] def format(self):
# TECHNICALLY we could craft the Geometry object with "length = ...", but it doesn't let us be explicit # This is a safeguard. We do *not* want to partition a disk that is mounted.
# in configs. So we manually crunch the numbers and do it all at the end. aif.utils.checkMounted(self.devpath)
# TODO: switch parted objects to BlockDev self.partition = _BlockDev.part.create_part(self.device,
# self.geometry = parted.Geometry(device = self.device, self.part_type,
# start = self.begin, self.begin,
# end = self.end) self.size,
# self.filesystem = parted.FileSystem(type = self.fstype, _BlockDev.PartAlign.OPTIMAL)
# geometry = self.geometry) self.devpath = self.partition.path
# self.partition = parted.Partition(disk = diskobj, _BlockDev.part.set_part_type(self.device, self.devpath, str(self.fs_type).upper())
# type = self.part_type, if self.part_name:
# geometry = self.geometry, _BlockDev.part.set_part_name(self.device, self.devpath, self.part_name)
# fs = self.filesystem) if self.flags:
self.part_name = self.xml.attrib.get('name') for f in self.flags:
_BlockDev.part.set_part_flag(self.device, self.devpath, f, True)
return()


# #
# def detect(self): # def detect(self):
@ -101,34 +128,89 @@ class Partition(object):


class Disk(object): class Disk(object):
def __init__(self, disk_xml): def __init__(self, disk_xml):
# TODO: BlockDev.part.set_disk_flag(<disk>,
# BlockDev.PartDiskFlag(1),
# True) ??
# https://lazka.github.io/pgi-docs/BlockDev-2.0/enums.html#BlockDev.PartDiskFlag
# https://unix.stackexchange.com/questions/325886/bios-gpt-do-we-need-a-boot-flag
self.xml = disk_xml self.xml = disk_xml
self.devpath = self.xml.attrib['device'] self.devpath = os.path.realpath(self.xml.attrib['device'])
aif.disk._common.addBDPlugin('part')
self.is_lowformatted = None self.is_lowformatted = None
self.is_hiformatted = None self.is_hiformatted = None
self.is_partitioned = None self.is_partitioned = None
self.partitions = None self.partitions = None
self._initDisk() self._initDisk()
aif.disk._common.addBDPlugin('part')


def _initDisk(self): def _initDisk(self):
pass if self.devpath == 'auto':
self.devpath = '/dev/{0}'.format(blkinfo.BlkDiskInfo().get_disks()[0]['kname'])
if not os.path.isfile(self.devpath):
raise ValueError('{0} does not exist; please specify an explicit device path'.format(self.devpath))
self.table_type = self.xml.attrib.get('diskFormat', 'gpt').lower()
if self.table_type in ('bios', 'mbr', 'dos', 'msdos'):
self.table_type = _BlockDev.PartTableType.MSDOS
elif self.table_type == 'gpt':
self.table_type = _BlockDev.PartTableType.GPT
else:
raise ValueError(('Disk format {0} is not valid for this architecture;'
'must be one of: gpt or msdos'.format(self.table_type)))
self.device = self.disk = _BlockDev.part.get_disk_spec(self.devpath)
self.is_lowformatted = False
self.is_hiformatted = False
self.is_partitioned = False
self.partitions = []
return()


def diskFormat(self): def diskFormat(self):
pass if self.is_lowformatted:
return ()
# This is a safeguard. We do *not* want to low-format a disk that is mounted.
aif.utils.checkMounted(self.devpath)
# TODO: BlockDev.part.set_disk_flag(<disk>,
# BlockDev.PartDiskFlag(1),
# True) ??
# https://lazka.github.io/pgi-docs/BlockDev-2.0/enums.html#BlockDev.PartDiskFlag
# https://unix.stackexchange.com/questions/325886/bios-gpt-do-we-need-a-boot-flag
_BlockDev.part.create_table(self.devpath, self.table_type, True)
self.is_lowformatted = True
self.is_partitioned = False
return()


def getPartitions(self): def getPartitions(self):
pass # For GPT, this *technically* should be 34 -- or, more precisely, 2048 (see FAQ in manual), but the alignment
# optimizer fixes it for us automatically.
# But for DOS tables, it's required.
if self.table_type == 'msdos':
start_sector = 2048
else:
start_sector = 0
self.partitions = []
xml_partitions = self.xml.findall('part')
for idx, part in enumerate(xml_partitions):
partnum = idx + 1
if self.table_type == 'gpt':
p = Partition(part, self.disk, start_sector, partnum, self.table_type)
else:
parttype = 'primary'
if len(xml_partitions) > 4:
if partnum == 4:
parttype = 'extended'
elif partnum > 4:
parttype = 'logical'
p = Partition(part, self.disk, start_sector, partnum, self.table_type, part_type = parttype)
start_sector = p.end + 1
self.partitions.append(p)
return()


def partFormat(self): def partFormat(self):
pass if self.is_partitioned:

return()

if not self.is_lowformatted:
class Mount(object): self.diskFormat()
def __init__(self, mount_xml, partobj): # This is a safeguard. We do *not* want to partition a disk that is mounted.
pass aif.utils.checkMounted(self.devpath)
_common.addBDPlugin('fs') if not self.partitions:
self.getPartitions()
if not self.partitions:
return()
for p in self.partitions:
p.format()
p.is_hiformatted = True
self.is_partitioned = True
return ()

53
aif/disk/block_fallback.py
View File

@ -3,6 +3,7 @@
# https://github.com/dcantrell/pyparted/blob/master/examples/query_device_capacity.py # https://github.com/dcantrell/pyparted/blob/master/examples/query_device_capacity.py
# TODO: Remember to replicate genfstab behaviour. # TODO: Remember to replicate genfstab behaviour.


import os
import re import re
try: try:
# https://stackoverflow.com/a/34812552/733214 # https://stackoverflow.com/a/34812552/733214
@ -12,6 +13,7 @@ except ImportError:
# We should never get here. util-linux is part of core (base) in Arch and uses "libmount". # We should never get here. util-linux is part of core (base) in Arch and uses "libmount".
import pylibmount as mount import pylibmount as mount
## ##
import blkinfo
import parted # https://www.gnu.org/software/parted/api/index.html import parted # https://www.gnu.org/software/parted/api/index.html
import psutil import psutil
## ##
@ -48,14 +50,14 @@ class Partition(object):
self.part_type = parted.PARTITION_LOGICAL self.part_type = parted.PARTITION_LOGICAL
else: else:
self.part_type = parted.PARTITION_NORMAL self.part_type = parted.PARTITION_NORMAL
self.fstype = self.xml.attrib['fsType'].lower() self.fs_type = self.xml.attrib['fsType'].lower()
if self.fstype not in aif.constants.PARTED_FSTYPES: if self.fs_type not in aif.constants.PARTED_FSTYPES:
raise ValueError(('{0} is not a valid partition filesystem type; ' raise ValueError(('{0} is not a valid partition filesystem type; '
'must be one of: {1}').format(self.xml.attrib['fsType'], 'must be one of: {1}').format(self.xml.attrib['fsType'],
', '.join(sorted(aif.constants.PARTED_FSTYPES)))) ', '.join(sorted(aif.constants.PARTED_FSTYPES))))
self.disk = diskobj self.disk = diskobj
self.device = self.disk.device self.device = self.disk.device
self.devpath = '{0}{1}'.format(self.device.path, partnum) self.devpath = '{0}{1}'.format(self.device.path, self.partnum)
self.is_hiformatted = False self.is_hiformatted = False
sizes = {} sizes = {}
for s in ('start', 'stop'): for s in ('start', 'stop'):
@ -63,7 +65,7 @@ class Partition(object):
aif.utils.convertSizeUnit(self.xml.attrib[s]))) aif.utils.convertSizeUnit(self.xml.attrib[s])))
sectors = x['size'] sectors = x['size']
if x['type'] == '%': if x['type'] == '%':
sectors = int(self.device.getLength() / x['size']) sectors = int(self.device.getLength() * (0.01 * x['size']))
else: else:
sectors = int(aif.utils.size.convertStorage(x['size'], sectors = int(aif.utils.size.convertStorage(x['size'],
x['type'], x['type'],
@ -89,7 +91,7 @@ class Partition(object):
self.geometry = parted.Geometry(device = self.device, self.geometry = parted.Geometry(device = self.device,
start = self.begin, start = self.begin,
end = self.end) end = self.end)
self.filesystem = parted.FileSystem(type = self.fstype, self.filesystem = parted.FileSystem(type = self.fs_type,
geometry = self.geometry) geometry = self.geometry)
self.partition = parted.Partition(disk = diskobj, self.partition = parted.Partition(disk = diskobj,
type = self.part_type, type = self.part_type,
@ -116,7 +118,8 @@ class Partition(object):
class Disk(object): class Disk(object):
def __init__(self, disk_xml): def __init__(self, disk_xml):
self.xml = disk_xml self.xml = disk_xml
self.devpath = self.xml.attrib['device'] self.id = self.xml.attrib['id']
self.devpath = os.path.realpath(self.xml.attrib['device'])
self.is_lowformatted = None self.is_lowformatted = None
self.is_hiformatted = None self.is_hiformatted = None
self.is_partitioned = None self.is_partitioned = None
@ -124,15 +127,19 @@ class Disk(object):
self._initDisk() self._initDisk()


def _initDisk(self): def _initDisk(self):
self.tabletype = self.xml.attrib.get('diskFormat', 'gpt').lower() if self.devpath == 'auto':
if self.tabletype in ('bios', 'mbr', 'dos'): self.devpath = '/dev/{0}'.format(blkinfo.BlkDiskInfo().get_disks()[0]['kname'])
self.tabletype = 'msdos' if not os.path.isfile(self.devpath):
raise ValueError('{0} does not exist; please specify an explicit device path'.format(self.devpath))
self.table_type = self.xml.attrib.get('diskFormat', 'gpt').lower()
if self.table_type in ('bios', 'mbr', 'dos'):
self.table_type = 'msdos'
validlabels = parted.getLabels() validlabels = parted.getLabels()
if self.tabletype not in validlabels: if self.table_type not in validlabels:
raise ValueError(('Disk format {0} is not valid for this architecture;' raise ValueError(('Disk format {0} is not valid for this architecture;'
'must be one of: {1}'.format(self.tabletype, ', '.join(list(validlabels))))) 'must be one of: {1}'.format(self.table_type, ', '.join(list(validlabels)))))
self.device = parted.getDevice(self.devpath) self.device = parted.getDevice(self.devpath)
self.disk = parted.freshDisk(self.device, self.tabletype) self.disk = parted.freshDisk(self.device, self.table_type)
self.is_lowformatted = False self.is_lowformatted = False
self.is_hiformatted = False self.is_hiformatted = False
self.is_partitioned = False self.is_partitioned = False
@ -143,9 +150,7 @@ class Disk(object):
if self.is_lowformatted: if self.is_lowformatted:
return() return()
# This is a safeguard. We do *not* want to low-format a disk that is mounted. # This is a safeguard. We do *not* want to low-format a disk that is mounted.
for p in psutil.disk_partitions(all = True): aif.utils.checkMounted(self.devpath)
if self.devpath in p:
raise RuntimeError('{0} is mounted; we are cowardly refusing to low-format it'.format(self.devpath))
self.disk.deleteAllPartitions() self.disk.deleteAllPartitions()
self.disk.commit() self.disk.commit()
self.is_lowformatted = True self.is_lowformatted = True
@ -156,7 +161,7 @@ class Disk(object):
# For GPT, this *technically* should be 34 -- or, more precisely, 2048 (see FAQ in manual), but the alignment # For GPT, this *technically* should be 34 -- or, more precisely, 2048 (see FAQ in manual), but the alignment
# optimizer fixes it for us automatically. # optimizer fixes it for us automatically.
# But for DOS tables, it's required. # But for DOS tables, it's required.
if self.tabletype == 'msdos': if self.table_type == 'msdos':
start_sector = 2048 start_sector = 2048
else: else:
start_sector = 0 start_sector = 0
@ -164,8 +169,8 @@ class Disk(object):
xml_partitions = self.xml.findall('part') xml_partitions = self.xml.findall('part')
for idx, part in enumerate(xml_partitions): for idx, part in enumerate(xml_partitions):
partnum = idx + 1 partnum = idx + 1
if self.tabletype == 'gpt': if self.table_type == 'gpt':
p = Partition(part, self.disk, start_sector, partnum, self.tabletype) p = Partition(part, self.disk, start_sector, partnum, self.table_type)
else: else:
parttype = 'primary' parttype = 'primary'
if len(xml_partitions) > 4: if len(xml_partitions) > 4:
@ -173,7 +178,7 @@ class Disk(object):
parttype = 'extended' parttype = 'extended'
elif partnum > 4: elif partnum > 4:
parttype = 'logical' parttype = 'logical'
p = Partition(part, self.disk, start_sector, partnum, self.tabletype, part_type = parttype) p = Partition(part, self.disk, start_sector, partnum, self.table_type, part_type = parttype)
start_sector = p.end + 1 start_sector = p.end + 1
self.partitions.append(p) self.partitions.append(p)
return() return()
@ -184,9 +189,7 @@ class Disk(object):
if not self.is_lowformatted: if not self.is_lowformatted:
self.diskFormat() self.diskFormat()
# This is a safeguard. We do *not* want to partition a disk that is mounted. # This is a safeguard. We do *not* want to partition a disk that is mounted.
for p in psutil.disk_partitions(all = True): aif.utils.checkMounted(self.devpath)
if self.devpath in p:
raise RuntimeError('{0} is mounted; we are cowardly refusing to low-format it'.format(self.devpath))
if not self.partitions: if not self.partitions:
self.getPartitions() self.getPartitions()
if not self.partitions: if not self.partitions:
@ -198,9 +201,3 @@ class Disk(object):
p.is_hiformatted = True p.is_hiformatted = True
self.is_partitioned = True self.is_partitioned = True
return() return()


class Mount(object):
def __init__(self, mount_xml, partobj):
self.xml = mount_xml
pass

113
aif/disk/filesystem.py
View File

@ -1,3 +1,114 @@
import os
import subprocess
##
import psutil
##
import aif.disk.block as block
import aif.disk.luks as luks
import aif.disk.lvm as lvm
import aif.disk.mdadm as mdadm
import aif.utils
from . import _common from . import _common


BlockDev = _common.BlockDev _BlockDev = _common.BlockDev


FS_FSTYPES = aif.utils.kernelFilesystems()


class FS(object):
def __init__(self, fs_xml, sourceobj):
# http://storaged.org/doc/udisks2-api/latest/gdbus-org.freedesktop.UDisks2.Filesystem.html#gdbus-interface-org-freedesktop-UDisks2-Filesystem.top_of_page
# http://storaged.org/doc/udisks2-api/latest/ ?
self.xml = fs_xml
self.id = self.xml.attrib['id']
if not isinstance(sourceobj, (block.Disk,
block.Partition,
luks.LUKS,
lvm.LV,
mdadm.Array)):
raise ValueError(('sourceobj must be of type '
'aif.disk.block.Partition, '
'aif.disk.luks.LUKS, '
'aif.disk.lvm.LV, or'
'aif.disk.mdadm.Array'))
self.source = sourceobj
self.devpath = sourceobj.devpath
self.formatted = False
self.fstype = self.xml.attrib.get('type')
if self.fstype not in FS_FSTYPES:
raise ValueError('{0} is not a supported filesystem type on this system'.format(self.fstype))

def format(self):
if self.formatted:
return ()
# This is a safeguard. We do *not* want to high-format a disk that is mounted.
aif.utils.checkMounted(self.devpath)
# TODO: Can I format with DBus/gobject-introspection? I feel like I *should* be able to, but BlockDev's fs
# plugin is *way* too limited in terms of filesystems and UDisks doesn't let you format that high-level.
# TODO! Logging
cmd = ['mkfs',
'-t', self.fstype]
for o in self.xml.findall('opt'):
cmd.append(o.attrib['name'])
if o.text:
cmd.append(o.text)
cmd.append(self.devpath)
subprocess.run(cmd)
self.formatted = True
return()


class Mount(object):
# http://storaged.org/doc/udisks2-api/latest/gdbus-org.freedesktop.UDisks2.Filesystem.html#gdbus-method-org-freedesktop-UDisks2-Filesystem.Mount
def __init__(self, mount_xml, fsobj):
self.xml = mount_xml
if not isinstance(fsobj, FS):
raise ValueError('partobj must be of type aif.disk.filesystem.FS')
_common.addBDPlugin('fs') # We *could* use the UDisks dbus to mount too, but best to stay within libblockdev.
self.id = self.xml.attrib['id']
self.fs = fsobj
self.source = self.fs.devpath
self.target = os.path.realpath(self.xml.attrib['target'])
self.opts = {}
for o in self.xml.findall('opt'):
self.opts[o.attrib['name']] = o.text
self.mounted = False

def _parseOpts(self):
opts = []
for k, v in self.opts.items():
if v and v is not True: # Python's boolean determination is weird sometimes.
opts.append('{0}={1}'.format(k, v))
else:
opts.append(k)
return(opts)

def mount(self):
if self.mounted:
return()
os.makedirs(self.target, exist_ok = True)
opts = self._parseOpts()
_BlockDev.fs.mount(self.source,
self.target,
self.fs.fstype,
(','.join(opts) if opts else None))
self.mounted = True
return()

def unmount(self, lazy = False, force = False):
self.updateMount()
if not self.mounted and not force:
return()
_BlockDev.fs.unmount(self.target,
lazy,
force)
self.mounted = False
return()

def updateMount(self):
if self.source in [p.device for p in psutil.disk_partitions(all = True)]:
self.mounted = True
else:
self.mounted = False
return()

118
aif/disk/filesystem_fallback.py
View File

@ -1,5 +1,4 @@
import os import os
import re
import subprocess import subprocess
## ##
import psutil import psutil
@ -8,56 +7,26 @@ import aif.disk.block_fallback as block
import aif.disk.luks_fallback as luks import aif.disk.luks_fallback as luks
import aif.disk.lvm_fallback as lvm import aif.disk.lvm_fallback as lvm
import aif.disk.mdadm_fallback as mdadm import aif.disk.mdadm_fallback as mdadm
import aif.utils


# I wish there was a better way of doing this.
# https://unix.stackexchange.com/a/98680 FS_FSTYPES = aif.utils.kernelFilesystems()
FS_FSTYPES = []
with open('/proc/filesystems', 'r') as fh:
for line in fh.readlines():
l = [i.strip() for i in line.split()]
if not l:
continue
if len(l) == 1:
FS_FSTYPES.append(l[0])
else:
FS_FSTYPES.append(l[1])
_mod_dir = os.path.join('/lib/modules',
os.uname().release,
'kernel/fs')
_strip_mod_suffix = re.compile(r'(?P<fsname>)\.ko(\.(x|g)?z)?$', re.IGNORECASE)
try:
for i in os.listdir(_mod_dir):
path = os.path.join(_mod_dir, i)
fs_name = None
if os.path.isdir(path):
fs_name = i
elif os.path.isfile(path):
mod_name = _strip_mod_suffix.search(i)
fs_name = mod_name.group('fsname')
if fs_name:
# The kernel *probably* has autoloading enabled, but in case it doesn't...
# TODO: logging!
if os.getuid() == 0:
subprocess.run(['modprobe', fs_name])
FS_FSTYPES.append(fs_name)
except FileNotFoundError:
# We're running on a kernel that doesn't have modules
pass




class FS(object): class FS(object):
def __init__(self, fs_xml, sourceobj): def __init__(self, fs_xml, sourceobj):
self.xml = fs_xml self.xml = fs_xml
if not isinstance(sourceobj, (aif.disk.block_fallback.Disk, if not isinstance(sourceobj, (block.Disk,
aif.disk.block_fallback.Partition, block.Partition,
aif.disk.luks_fallback.LUKS, luks.LUKS,
aif.disk.lvm_fallback.LV, lvm.LV,
aif.disk.mdadm_fallback.Array)): mdadm.Array)):
raise ValueError(('sourceobj must be of type ' raise ValueError(('sourceobj must be of type '
'aif.disk.block.Partition, ' 'aif.disk.block.Partition, '
'aif.disk.luks.LUKS, ' 'aif.disk.luks.LUKS, '
'aif.disk.lvm.LV, or' 'aif.disk.lvm.LV, or'
'aif.disk.mdadm.Array')) 'aif.disk.mdadm.Array'))
self.id = self.xml.attrib['id']
self.source = sourceobj self.source = sourceobj
self.devpath = sourceobj.devpath self.devpath = sourceobj.devpath
self.formatted = False self.formatted = False
@ -67,10 +36,7 @@ class FS(object):
if self.formatted: if self.formatted:
return () return ()
# This is a safeguard. We do *not* want to high-format a disk that is mounted. # This is a safeguard. We do *not* want to high-format a disk that is mounted.
for p in psutil.disk_partitions(all = True): aif.utils.checkMounted(self.devpath)
if self.devpath in p:
raise RuntimeError(('{0} is mounted;'
'we are cowardly refusing to apply a filesystem to it').format(self.devpath))
# TODO! Logging # TODO! Logging
cmd = ['mkfs', cmd = ['mkfs',
'-t', self.fstype] '-t', self.fstype]
@ -80,5 +46,67 @@ class FS(object):
cmd.append(o.text) cmd.append(o.text)
cmd.append(self.devpath) cmd.append(self.devpath)
subprocess.run(cmd) subprocess.run(cmd)
self.is_hiformatted = True self.formatted = True
return()


class Mount(object):
def __init__(self, mount_xml, fsobj):
self.xml = mount_xml
self.id = self.xml.attrib['id']
if not isinstance(fsobj, FS):
raise ValueError('partobj must be of type aif.disk.filesystem.FS')
self.id = self.xml.attrib['id']
self.fs = fsobj
self.source = self.fs.devpath
self.target = os.path.realpath(self.xml.attrib['target'])
self.opts = {}
for o in self.xml.findall('opt'):
self.opts[o.attrib['name']] = o.text
self.mounted = False

def _parseOpts(self):
opts = []
for k, v in self.opts.items():
if v and v is not True: # Python's boolean determination is weird sometimes.
opts.append('{0}={1}'.format(k, v))
else:
opts.append(k)
return(opts)

def mount(self):
if self.mounted:
return()
os.makedirs(self.target, exist_ok = True)
opts = self._parseOpts()
# TODO: logging
cmd = ['/usr/bin/mount',
'--types', self.fs.fstype]
if opts:
cmd.extend(['--options', ','.join(opts)])
cmd.extend([self.source, self.target])
subprocess.run(cmd)
self.mounted = True
return()

def unmount(self, lazy = False, force = False):
self.updateMount()
if not self.mounted and not force:
return()
# TODO: logging
cmd = ['/usr/bin/umount']
if lazy:
cmd.append('--lazy')
if force:
cmd.append('--force')
cmd.append(self.target)
subprocess.run(cmd)
self.mounted = False
return()

def updateMount(self):
if self.source in [p.device for p in psutil.disk_partitions(all = True)]:
self.mounted = True
else:
self.mounted = False
return() return()

86
aif/utils.py
View File

@ -1,25 +1,14 @@
import os import os
import re import re
import subprocess
##
import psutil




def hasBin(binary_name): def checkMounted(devpath):
paths = [] if devpath in [p.device for p in psutil.disk_partitions(all = True)]:
for p in os.environ.get('PATH', '/usr/bin:/bin').split(':'): raise RuntimeError('{0} is mounted; we are cowardly refusing to destructive operations on it'.format(devpath))
if binary_name in os.listdir(os.path.realpath(p)): return()
return(os.path.join(p, binary_name))
return(False)


def xmlBool(xmlobj):
# https://bugs.launchpad.net/lxml/+bug/1850221
if isinstance(xmlobj, bool):
return (xmlobj)
if xmlobj.lower() in ('1', 'true'):
return(True)
elif xmlobj.lower() in ('0', 'false'):
return(False)
else:
return(None)




def collapseKeys(d, keylist = None): def collapseKeys(d, keylist = None):
@ -45,6 +34,65 @@ def collapseValues(d, vallist = None):
return(vallist) return(vallist)




def hasBin(binary_name):
paths = []
for p in os.environ.get('PATH', '/usr/bin:/bin').split(':'):
if binary_name in os.listdir(os.path.realpath(p)):
return(os.path.join(p, binary_name))
return(False)


def kernelFilesystems():
# I wish there was a better way of doing this.
# https://unix.stackexchange.com/a/98680
FS_FSTYPES = ['swap']
with open('/proc/filesystems', 'r') as fh:
for line in fh.readlines():
l = [i.strip() for i in line.split()]
if not l:
continue
if len(l) == 1:
FS_FSTYPES.append(l[0])
else:
FS_FSTYPES.append(l[1])
_mod_dir = os.path.join('/lib/modules',
os.uname().release,
'kernel/fs')
_strip_mod_suffix = re.compile(r'(?P<fsname>)\.ko(\.(x|g)?z)?$', re.IGNORECASE)
try:
for i in os.listdir(_mod_dir):
path = os.path.join(_mod_dir, i)
fs_name = None
if os.path.isdir(path):
fs_name = i
elif os.path.isfile(path):
mod_name = _strip_mod_suffix.search(i)
fs_name = mod_name.group('fsname')
if fs_name:
# The kernel *probably* has autoloading enabled, but in case it doesn't...
# TODO: logging!
if os.getuid() == 0:
subprocess.run(['modprobe', fs_name])
FS_FSTYPES.append(fs_name)
except FileNotFoundError:
# We're running on a kernel that doesn't have modules
pass
FS_FSTYPES = sorted(list(set(FS_FSTYPES)))
return(FS_FSTYPES)


def xmlBool(xmlobj):
# https://bugs.launchpad.net/lxml/+bug/1850221
if isinstance(xmlobj, bool):
return (xmlobj)
if xmlobj.lower() in ('1', 'true'):
return(True)
elif xmlobj.lower() in ('0', 'false'):
return(False)
else:
return(None)


class _Sizer(object): class _Sizer(object):
def __init__(self): def __init__(self):
# We use different methods for converting between storage and BW, and different multipliers for each subtype. # We use different methods for converting between storage and BW, and different multipliers for each subtype.
@ -188,6 +236,7 @@ size = _Sizer()




# We do this as base level so they aren't compiled on every invocation/instantiation. # We do this as base level so they aren't compiled on every invocation/instantiation.
# Unfortunately it has to be at the bottom so we can call the instantiated _Sizer() class.
# parted lib can do SI or IEC. So can we. # parted lib can do SI or IEC. So can we.
_pos_re = re.compile((r'^(?P<pos_or_neg>-|\+)?\s*' _pos_re = re.compile((r'^(?P<pos_or_neg>-|\+)?\s*'
r'(?P<size>[0-9]+)\s*' r'(?P<size>[0-9]+)\s*'
@ -212,3 +261,4 @@ def convertSizeUnit(pos):
else: else:
raise ValueError('Invalid size specified: {0}'.format(orig_pos)) raise ValueError('Invalid size specified: {0}'.format(orig_pos))
return((from_beginning, _size, amt_type)) return((from_beginning, _size, amt_type))


31
examples/aif.xml
View File

@ -5,7 +5,7 @@
version="v2_rewrite"><!-- When we release, this should match the tagged release (e.g. 0.2.0) --> version="v2_rewrite"><!-- When we release, this should match the tagged release (e.g. 0.2.0) -->
<storage> <storage>
<blockDevices> <blockDevices>
<disk device="/dev/sda" diskFormat="gpt"> <disk id="sda" device="/dev/sda" diskFormat="gpt">
<!-- Partitions are numbered *in the order they are specified*. --> <!-- Partitions are numbered *in the order they are specified*. -->
<!-- e.g. "boot" would be /dev/sda1, "secrets1" would be /dev/sda2, etc. --> <!-- e.g. "boot" would be /dev/sda1, "secrets1" would be /dev/sda2, etc. -->
<part id="boot" name="BOOT" label="/boot" start="0%" stop="10%" fsType="fat32"> <part id="boot" name="BOOT" label="/boot" start="0%" stop="10%" fsType="fat32">
@ -23,7 +23,7 @@
<part id="raid1_d2" start="55%" stop="80%" fsType="ext4"> <part id="raid1_d2" start="55%" stop="80%" fsType="ext4">
<partitionFlag>raid</partitionFlag> <partitionFlag>raid</partitionFlag>
</part> </part>
<part id="swap" start="80%" stop="90%" fsType="linux-swap(v1)"> <part id="swapdisk" start="80%" stop="90%" fsType="linux-swap(v1)">
<partitionFlag>swap</partitionFlag> <partitionFlag>swap</partitionFlag>
</part> </part>
<!-- You can also create a partition with no flags (and not use). --> <!-- You can also create a partition with no flags (and not use). -->
@ -38,7 +38,8 @@
<luks> <luks>
<luksDev id="luks_secrets" name="secrets" source="secrets1"> <luksDev id="luks_secrets" name="secrets" source="secrets1">
<!-- You can assign multiple secrets (or "keys") to a LUKS volume. --> <!-- You can assign multiple secrets (or "keys") to a LUKS volume. -->
<secrets> <secrets
>
<!-- A simple passphrase. --> <!-- A simple passphrase. -->
<passphrase>secrets1</passphrase> <passphrase>secrets1</passphrase>
</secrets> </secrets>
@ -77,19 +78,29 @@
</array> </array>
</mdadm> </mdadm>
<fileSystems> <fileSystems>
<fs source="boot" type="vfat"> <fs id="esp" source="boot" type="vfat">
<!-- Supports mkfs arguments. Leave off the filesystem type and device name, obviously; <!-- Supports mkfs arguments. Leave off the filesystem type and device name, obviously;
those are handled by the above attributes. --> those are handled by the above attributes. -->
<opt name="-F">32</opt> <opt name="-F">32</opt>
<opt name="-n">ESP</opt> <opt name="-n">ESP</opt>
</fs> </fs>
<fs source="luks_secrets" type="ext4"> <fs id="luks" source="luks_secrets" type="ext4">
<opt name="-L">seekrit</opt> <opt name="-L">seekrit</opt>
</fs> </fs>
<fs id="swap" source="swap" type="swap"/>
<fs id="vg1-lv1" source="lv1" type="ext4"/>
<fs id="mdraid" source="mdadm1" type="ext4"/>
</fileSystems> </fileSystems>
<mountPoints> <mountPoints>
<!-- And you use the id to reference mountpoints as well. --> <!-- And you use the id to reference mountpoints as well. Important to note, we mount *filesystems*,
<mount source="luks_secrets" target="/mnt/aif"> not partitions/disks/etc. -->
<!-- Note that targets should be *outside* of the chroot!
e.g. /aif/storage/mountPoints[@target="/mnt/aif/boot"]
and
/aif/system[@chrootPath="/mnt/aif"]
would lead to the filesystem being accessible *inside* the chroot (and thus the completed install)
at /boot. -->
<mount source="luks" target="/mnt/aif">
<opt name="rw"/> <opt name="rw"/>
<opt name="relatime"/> <opt name="relatime"/>
<opt name="compress">lzo</opt> <opt name="compress">lzo</opt>
@ -98,10 +109,10 @@
<opt name="subvolid">5</opt> <opt name="subvolid">5</opt>
<opt name="subvol">/</opt> <opt name="subvol">/</opt>
</mount> </mount>
<mount source="boot" target="/mnt/aif/boot"/> <mount source="esp" target="/mnt/aif/boot"/>
<mount source="swap" target="swap"/> <mount source="swap" target="swap"/>
<mount source="lv1" target="/mnt/aif/mnt/pool"/> <mount source="vg1-lv1" target="/mnt/aif/mnt/pool"/>
<mount source="mdadm1" target="/mnt/aif/mnt/raid"/> <mount source="mdraid" target="/mnt/aif/mnt/raid"/>
</mountPoints> </mountPoints>
</storage> </storage>
<network hostname="aiftest.square-r00t.net"> <network hostname="aiftest.square-r00t.net">