diff --git a/aif.xsd b/aif.xsd index 5f007e3..73d601a 100644 --- a/aif.xsd +++ b/aif.xsd @@ -48,10 +48,10 @@ + - This element validates a filesystem type to be specified for formatting a partition. See sgdisk -L (or - the table at http://www.rodsbooks.com/gdisk/walkthrough.html) for valid filesystem codes. + This element validates a filesystem type to be specified for formatting a partition (NOT applying a filesystem!); valid values are: affs0, affs1, affs2, affs3, affs4, affs5, affs6, affs7, amufs, amufs0, amufs1, amufs2, amufs3, amufs4, amufs5, apfs1, apfs2, asfs, btrfs, ext2, ext3, ext4, fat16, fat32, hfs, hfs+, hfsx, hp-ufs, jfs, linux-swap(v0), linux-swap(v1), nilfs2, ntfs, reiserfs, sun-ufs, swsusp, udf, xfs diff --git a/aif/config.py b/aif/config.py index ff4df42..7501007 100644 --- a/aif/config.py +++ b/aif/config.py @@ -2,6 +2,8 @@ import os ## from lxml import etree +#https://stackoverflow.com/questions/30232031/how-can-i-strip-namespaces-out-of-an-lxml-tree/30233635#30233635 ? + class Config(object): def __init__(self): self.xml = None diff --git a/aif/constants.py b/aif/constants.py index e69de29..8dc0424 100644 --- a/aif/constants.py +++ b/aif/constants.py @@ -0,0 +1,4 @@ +import parted + + +PARTED_FSTYPES = list(dict(vars(parted.filesystem))['fileSystemType'].keys()) diff --git a/aif/disk.py b/aif/disk.py index ec47a6c..9ead501 100644 --- a/aif/disk.py +++ b/aif/disk.py @@ -19,6 +19,7 @@ import parted # https://www.gnu.org/software/parted/api/index.html import psutil ## from .aif_util import xmlBool +from .constants import PARTED_FSTYPES # parted lib can do SI or IEC (see table to right at https://en.wikipedia.org/wiki/Binary_prefix) @@ -60,55 +61,76 @@ def convertSizeUnit(pos): class Partition(object): - def __init__(self, disk_xml, diskobj, start_sector): - self.xml = disk_xml - device = diskobj.device + def __init__(self, part_xml, diskobj, start_sector, partnum, tbltype): + if tbltype not in ('gpt', 'msdos'): + raise ValueError('{0} must be one of gpt or msdos'.format(tbltype)) + self.xml = part_xml + self.partnum = partnum + self.fstype = self.xml.attrib['fsType'].lower() + if self.fstype not in PARTED_FSTYPES: + raise ValueError(('{0} is not a valid partition filesystem type; ' + 'must be one of: {1}').format(self.xml.attrib['fsType'], + ', '.join(sorted(PARTED_FSTYPES)))) + self.disk = diskobj + self.device = self.disk.device + self.dev = '{0}{1}'.format(self.device.path, partnum) sizes = {} for s in ('start', 'stop'): x = dict(zip(('from_bgn', 'size', 'type'), convertSizeUnit(self.xml.attrib[s]))) sectors = x['size'] if x['type'] == '%': - sectors = int(device.getLength() / x['size']) + sectors = int(self.device.getLength() / x['size']) elif x['type'] in _units.keys(): - sectors = int(x['size'] << _units[x['type']] / device.sectorSize) + sectors = int(x['size'] << _units[x['type']] / self.device.sectorSize) sizes[s] = (sectors, x['from_bgn']) if sizes['start'][1] is not None: if sizes['start'][1]: self.begin = sizes['start'][0] + 0 else: - self.begin = device.getLength() - sizes['start'][0] + self.begin = self.device.getLength() - sizes['start'][0] else: self.begin = sizes['start'][0] + start_sector if sizes['stop'][1] is not None: if sizes['stop'][1]: self.end = sizes['stop'][0] + 0 else: - self.end = device.getLength() - sizes['stop'][0] + # This *technically* should be - 34, but the alignment optimizer fixes it for us. + self.end = (self.device.getLength() - 1) - sizes['stop'][0] else: self.end = self.begin + sizes['stop'][0] # TECHNICALLY we could craft the Geometry object with "length = ...", but it doesn't let us be explicit # in configs. So we manually crunch the numbers and do it all at the end. - self.geometry = parted.Geometry(device = device, + self.geometry = parted.Geometry(device = self.device, start = self.begin, end = self.end) - self.filesystem = parted.FileSystem(type = self.xml.attrib['fsType'], - ) + self.filesystem = parted.FileSystem(type = self.fstype, + geometry = self.geometry) self.partition = parted.Partition(disk = diskobj, type = parted.PARTITION_NORMAL, geometry = self.geometry, - fs = ) + fs = self.filesystem) + if tbltype == 'gpt' and self.xml.attrib.get('name'): + # The name attribute setting is b0rk3n, so we operate on the underlying PedPartition object. + # https://github.com/dcantrell/pyparted/issues/49#issuecomment-540096687 + # https://github.com/dcantrell/pyparted/issues/65 + # self.partition.name = self.xml.attrib.get('name') + _pedpart = self.partition.getPedPartition() + _pedpart.set_name(self.xml.attrib.get('name')) + class Disk(object): def __init__(self, disk_xml): self.xml = disk_xml self.devpath = self.xml.attrib['device'] + self.partitions = [] self._initDisk() def _initDisk(self): self.device = parted.getDevice(self.devpath) try: self.disk = parted.newDisk(self.device) + self.is_new = False if xmlBool(self.xml.attrib.get('forceReformat')): self.is_lowformatted = False self.is_hiformatted = False @@ -121,43 +143,71 @@ class Disk(object): if d.get('fstype', '').strip() != '': self.is_hiformatted = True break - self.is_partitioned = True except parted._ped.DiskException: self.disk = None + self.is_new = True self.is_lowformatted = False self.is_hiformatted = False - self.is_partitioned = False + self.is_partitioned = False return() - def diskformat(self): + def diskFormat(self): if self.is_lowformatted: return() # This is a safeguard. We do *not* want to low-format a disk that is mounted. for p in psutil.disk_partitions(all = True): if self.devpath in p: raise RuntimeError('{0} is mounted; we are cowardly refusing to low-format it'.format(self.devpath)) - self.disk.deleteAllPartitions() - tabletype = self.xml.attrib.get('diskFormat', 'gpt').lower() - if tabletype in ('bios', 'mbr'): - tabletype = 'msdos' + if not self.is_new: + self.disk.deleteAllPartitions() + self.tabletype = self.xml.attrib.get('diskFormat', 'gpt').lower() + if self.tabletype in ('bios', 'mbr', 'dos'): + self.tabletype = 'msdos' validlabels = parted.getLabels() - if tabletype not in validlabels: + if self.tabletype not in validlabels: raise ValueError(('Disk format {0} is not valid for this architecture;' - 'must be one of: {1}'.format(tabletype, ', '.join(list(validlabels))))) - self.disk = parted.freshDisk(self.device, tabletype) - - pass + 'must be one of: {1}'.format(self.tabletype, ', '.join(list(validlabels))))) + self.disk = parted.freshDisk(self.device, self.tabletype) self.is_lowformatted = True - self.is_partitioned = True return() - def fsformat(self): + def fsFormat(self): if self.is_hiformatted: return() # This is a safeguard. We do *not* want to high-format a disk that is mounted. for p in psutil.disk_partitions(all = True): if self.devpath in p: raise RuntimeError('{0} is mounted; we are cowardly refusing to high-format it'.format(self.devpath)) - + # TODO! pass return() + + def getPartitions(self): + # For GPT, this *technically* should be 34 -- or, more precisely, 2048 (see FAQ in manual), but the alignment + # optimizer fixes it for us automatically. + start_sector = 0 + self.partitions = [] + for idx, part in enumerate(self.xml.findall('part')): + p = Partition(part, self.disk, start_sector, idx + 1, self.tabletype) + start_sector = p.end + 1 + self.partitions.append(p) + return() + + def partFormat(self): + if self.is_partitioned: + return() + if not self.is_lowformatted: + self.diskFormat() + # This is a safeguard. We do *not* want to partition a disk that is mounted. + for p in psutil.disk_partitions(all = True): + if self.devpath in p: + raise RuntimeError('{0} is mounted; we are cowardly refusing to low-format it'.format(self.devpath)) + if not self.partitions: + self.getPartitions() + if not self.partitions: + return() + for p in self.partitions: + self.disk.addPartition(partition = p, constraint = self.device.optimalAlignedConstraint) + self.disk.commit() + self.is_partitioned = True + return() diff --git a/docs/README.adoc b/docs/README.adoc index 3c9bfee..ff9f125 100644 --- a/docs/README.adoc +++ b/docs/README.adoc @@ -199,7 +199,7 @@ The `/aif` element is the https://en.wikipedia.org/wiki/Root_element[root elemen The `/aif/storage` element contains <>, <>, and <> elements. ==== `` -The `/aif/storage/disk` element holds information about disks on the system, and within this element are one (or more) <> elements. +The `/aif/storage/disk` element holds information about disks on the system, and within this element are one (or more) <> elements. Note that any `disk` elements specified here will be *entirely reformatted*; operate under the assumption that ANY and ALL pre-existing data on the specified device will be IRREVOCABLY LOST. [options="header"] |====================== @@ -544,7 +544,19 @@ There are several script types availabe for `execution`. Currently, these are: *pre* scripts are run (in numerical `order`) before the disks are even formatted. *pkg* scripts are run (in numerical `order`) right before the <> are installed (this allows you to configure an <> such as https://aur.archlinux.org/packages/apacman/[apacman^]) -- these are run *inside* the chroot of the new install. *post* scripts are run inside the chroot like *pkg*, but are executed very last thing, just before the reboot. = Further Information -Here you will find further info, other resources, and such relating to AIF-NG. +Here you will find further info and other resources relating to AIF-NG. + +== FAQ + +=== "I specified start sector as 0 for a GPT-labeled disk but it starts at sector 2048 instead. What gives?" +GPT requires 33 sectors for the table at the beginning (and 32 sectors at the end) for the actual table. That plus an extra (usually) 512 bytes at the beginning for something called a https://en.wikipedia.org/wiki/GUID_Partition_Table#Protective_MBR_(LBA_0)[Protective MBR^] (this prevents disk utilities from overwriting the GPT label automatically in case they only recognize "msdos" labels and assume the disk is not formatted yet). + +Most disks these days use something called https://en.wikipedia.org/wiki/Advanced_Format[Advanced Format^]. These align their sectors to factors of 8, so sector 34 can't be used - it'd have to be sector 40. Additionally, various other low-level disk interactions (e.g. RAID stripe sizes) require a much larger boundary between partitions. If you're interested in a little more detail, you may find https://metebalci.com/blog/a-quick-tour-of-guid-partition-table-gpt/[this^] interesting (specifically https://metebalci.com/blog/a-quick-tour-of-guid-partition-table-gpt/#gpt-partition-entry-array[this section^], paragraph starting with `You may also ask why the first partition starts from LBA 2048...`). + +TL;DR: "It's the safest way to make sure your disk doesn't suffer massive degradation in performance, your RAID doesn't eat partitions, etc." Don't worry, it typically only ends up being about 1MB of "wasted" space surrounding partitions. I've written plaintext documentation larger than 1MB. + +=== "Why isn't my last GPT partition extending to the last sector?" +See above. == Bug Reports/Feature Requests NOTE: It is possible to submit a bug or feature request without registering in my bugtracker. One of my pet peeves is needing to create an account/register on a bugtracker simply to report a bug! The following links only require an email address to file a bug (which is necessary in case I need any further clarification from you or to keep you updated on the status of the bug/feature request -- so please be sure to use a valid email address). diff --git a/docs/TODO b/docs/TODO index 3dab1a9..38bc186 100644 --- a/docs/TODO +++ b/docs/TODO @@ -46,3 +46,6 @@ https://www.w3schools.com/xml/schema_intro.asp https://www.w3schools.com/xml/schema_example.asp https://msdn.microsoft.com/en-us/library/dd489258.aspx +if i ever need a list of GPT GUIDs, maybe to do some fancy GUID-to-name-and-back mapping? +https://en.wikipedia.org/wiki/GUID_Partition_Table#Partition_type_GUIDs +(mapping can be done via https://stackoverflow.com/questions/483666/reverse-invert-a-dictionary-mapping) \ No newline at end of file diff --git a/docs/examples/aif.xml b/docs/examples/aif.xml index 602e5dc..ff38747 100644 --- a/docs/examples/aif.xml +++ b/docs/examples/aif.xml @@ -5,13 +5,18 @@ - - - + + + + + + -F 32 -n {label} + + diff --git a/docs/reference/fstypes b/docs/reference/fstypes new file mode 100644 index 0000000..6c75e49 --- /dev/null +++ b/docs/reference/fstypes @@ -0,0 +1,118 @@ +msdos: + 0 Empty 24 NEC DOS 81 Minix / old Lin bf Solaris + 1 FAT12 27 Hidden NTFS Win 82 Linux swap / So c1 DRDOS/sec (FAT- + 2 XENIX root 39 Plan 9 83 Linux c4 DRDOS/sec (FAT- + 3 XENIX usr 3c PartitionMagic 84 OS/2 hidden or c6 DRDOS/sec (FAT- + 4 FAT16 <32M 40 Venix 80286 85 Linux extended c7 Syrinx + 5 Extended 41 PPC PReP Boot 86 NTFS volume set da Non-FS data + 6 FAT16 42 SFS 87 NTFS volume set db CP/M / CTOS / . + 7 HPFS/NTFS/exFAT 4d QNX4.x 88 Linux plaintext de Dell Utility + 8 AIX 4e QNX4.x 2nd part 8e Linux LVM df BootIt + 9 AIX bootable 4f QNX4.x 3rd part 93 Amoeba e1 DOS access + a OS/2 Boot Manag 50 OnTrack DM 94 Amoeba BBT e3 DOS R/O + b W95 FAT32 51 OnTrack DM6 Aux 9f BSD/OS e4 SpeedStor + c W95 FAT32 (LBA) 52 CP/M a0 IBM Thinkpad hi ea Rufus alignment + e W95 FAT16 (LBA) 53 OnTrack DM6 Aux a5 FreeBSD eb BeOS fs + f W95 Ext'd (LBA) 54 OnTrackDM6 a6 OpenBSD ee GPT +10 OPUS 55 EZ-Drive a7 NeXTSTEP ef EFI (FAT-12/16/ +11 Hidden FAT12 56 Golden Bow a8 Darwin UFS f0 Linux/PA-RISC b +12 Compaq diagnost 5c Priam Edisk a9 NetBSD f1 SpeedStor +14 Hidden FAT16 <3 61 SpeedStor ab Darwin boot f4 SpeedStor +16 Hidden FAT16 63 GNU HURD or Sys af HFS / HFS+ f2 DOS secondary +17 Hidden HPFS/NTF 64 Novell Netware b7 BSDI fs fb VMware VMFS +18 AST SmartSleep 65 Novell Netware b8 BSDI swap fc VMware VMKCORE +1b Hidden W95 FAT3 70 DiskSecure Mult bb Boot Wizard hid fd Linux raid auto +1c Hidden W95 FAT3 75 PC/IX bc Acronis FAT32 L fe LANstep +1e Hidden W95 FAT1 80 Old Minix be Solaris boot ff BBT + +gpt: + 1 EFI System C12A7328-F81F-11D2-BA4B-00A0C93EC93B + 2 MBR partition scheme 024DEE41-33E7-11D3-9D69-0008C781F39F + 3 Intel Fast Flash D3BFE2DE-3DAF-11DF-BA40-E3A556D89593 + 4 BIOS boot 21686148-6449-6E6F-744E-656564454649 + 5 Sony boot partition F4019732-066E-4E12-8273-346C5641494F + 6 Lenovo boot partition BFBFAFE7-A34F-448A-9A5B-6213EB736C22 + 7 PowerPC PReP boot 9E1A2D38-C612-4316-AA26-8B49521E5A8B + 8 ONIE boot 7412F7D5-A156-4B13-81DC-867174929325 + 9 ONIE config D4E6E2CD-4469-46F3-B5CB-1BFF57AFC149 + 10 Microsoft reserved E3C9E316-0B5C-4DB8-817D-F92DF00215AE + 11 Microsoft basic data EBD0A0A2-B9E5-4433-87C0-68B6B72699C7 + 12 Microsoft LDM metadata 5808C8AA-7E8F-42E0-85D2-E1E90434CFB3 + 13 Microsoft LDM data AF9B60A0-1431-4F62-BC68-3311714A69AD + 14 Windows recovery environment DE94BBA4-06D1-4D40-A16A-BFD50179D6AC + 15 IBM General Parallel Fs 37AFFC90-EF7D-4E96-91C3-2D7AE055B174 + 16 Microsoft Storage Spaces E75CAF8F-F680-4CEE-AFA3-B001E56EFC2D + 17 HP-UX data 75894C1E-3AEB-11D3-B7C1-7B03A0000000 + 18 HP-UX service E2A1E728-32E3-11D6-A682-7B03A0000000 + 19 Linux swap 0657FD6D-A4AB-43C4-84E5-0933C84B4F4F + 20 Linux filesystem 0FC63DAF-8483-4772-8E79-3D69D8477DE4 + 21 Linux server data 3B8F8425-20E0-4F3B-907F-1A25A76F98E8 + 22 Linux root (x86) 44479540-F297-41B2-9AF7-D131D5F0458A + 23 Linux root (ARM) 69DAD710-2CE4-4E3C-B16C-21A1D49ABED3 + 24 Linux root (x86-64) 4F68BCE3-E8CD-4DB1-96E7-FBCAF984B709 + 25 Linux root (ARM-64) B921B045-1DF0-41C3-AF44-4C6F280D3FAE + 26 Linux root (IA-64) 993D8D3D-F80E-4225-855A-9DAF8ED7EA97 + 27 Linux reserved 8DA63339-0007-60C0-C436-083AC8230908 + 28 Linux home 933AC7E1-2EB4-4F13-B844-0E14E2AEF915 + 29 Linux RAID A19D880F-05FC-4D3B-A006-743F0F84911E + 30 Linux extended boot BC13C2FF-59E6-4262-A352-B275FD6F7172 + 31 Linux LVM E6D6D379-F507-44C2-A23C-238F2A3DF928 + 32 FreeBSD data 516E7CB4-6ECF-11D6-8FF8-00022D09712B + 33 FreeBSD boot 83BD6B9D-7F41-11DC-BE0B-001560B84F0F + 34 FreeBSD swap 516E7CB5-6ECF-11D6-8FF8-00022D09712B + 35 FreeBSD UFS 516E7CB6-6ECF-11D6-8FF8-00022D09712B + 36 FreeBSD ZFS 516E7CBA-6ECF-11D6-8FF8-00022D09712B + 37 FreeBSD Vinum 516E7CB8-6ECF-11D6-8FF8-00022D09712B + 38 Apple HFS/HFS+ 48465300-0000-11AA-AA11-00306543ECAC + 39 Apple UFS 55465300-0000-11AA-AA11-00306543ECAC + 40 Apple RAID 52414944-0000-11AA-AA11-00306543ECAC + 41 Apple RAID offline 52414944-5F4F-11AA-AA11-00306543ECAC + 42 Apple boot 426F6F74-0000-11AA-AA11-00306543ECAC + 43 Apple label 4C616265-6C00-11AA-AA11-00306543ECAC + 44 Apple TV recovery 5265636F-7665-11AA-AA11-00306543ECAC + 45 Apple Core storage 53746F72-6167-11AA-AA11-00306543ECAC + 46 Solaris boot 6A82CB45-1DD2-11B2-99A6-080020736631 + 47 Solaris root 6A85CF4D-1DD2-11B2-99A6-080020736631 + 48 Solaris /usr & Apple ZFS 6A898CC3-1DD2-11B2-99A6-080020736631 + 49 Solaris swap 6A87C46F-1DD2-11B2-99A6-080020736631 + 50 Solaris backup 6A8B642B-1DD2-11B2-99A6-080020736631 + 51 Solaris /var 6A8EF2E9-1DD2-11B2-99A6-080020736631 + 52 Solaris /home 6A90BA39-1DD2-11B2-99A6-080020736631 + 53 Solaris alternate sector 6A9283A5-1DD2-11B2-99A6-080020736631 + 54 Solaris reserved 1 6A945A3B-1DD2-11B2-99A6-080020736631 + 55 Solaris reserved 2 6A9630D1-1DD2-11B2-99A6-080020736631 + 56 Solaris reserved 3 6A980767-1DD2-11B2-99A6-080020736631 + 57 Solaris reserved 4 6A96237F-1DD2-11B2-99A6-080020736631 + 58 Solaris reserved 5 6A8D2AC7-1DD2-11B2-99A6-080020736631 + 59 NetBSD swap 49F48D32-B10E-11DC-B99B-0019D1879648 + 60 NetBSD FFS 49F48D5A-B10E-11DC-B99B-0019D1879648 + 61 NetBSD LFS 49F48D82-B10E-11DC-B99B-0019D1879648 + 62 NetBSD concatenated 2DB519C4-B10E-11DC-B99B-0019D1879648 + 63 NetBSD encrypted 2DB519EC-B10E-11DC-B99B-0019D1879648 + 64 NetBSD RAID 49F48DAA-B10E-11DC-B99B-0019D1879648 + 65 ChromeOS kernel FE3A2A5D-4F32-41A7-B725-ACCC3285A309 + 66 ChromeOS root fs 3CB8E202-3B7E-47DD-8A3C-7FF2A13CFCEC + 67 ChromeOS reserved 2E0A753D-9E48-43B0-8337-B15192CB1B5E + 68 MidnightBSD data 85D5E45A-237C-11E1-B4B3-E89A8F7FC3A7 + 69 MidnightBSD boot 85D5E45E-237C-11E1-B4B3-E89A8F7FC3A7 + 70 MidnightBSD swap 85D5E45B-237C-11E1-B4B3-E89A8F7FC3A7 + 71 MidnightBSD UFS 0394EF8B-237E-11E1-B4B3-E89A8F7FC3A7 + 72 MidnightBSD ZFS 85D5E45D-237C-11E1-B4B3-E89A8F7FC3A7 + 73 MidnightBSD Vinum 85D5E45C-237C-11E1-B4B3-E89A8F7FC3A7 + 74 Ceph Journal 45B0969E-9B03-4F30-B4C6-B4B80CEFF106 + 75 Ceph Encrypted Journal 45B0969E-9B03-4F30-B4C6-5EC00CEFF106 + 76 Ceph OSD 4FBD7E29-9D25-41B8-AFD0-062C0CEFF05D + 77 Ceph crypt OSD 4FBD7E29-9D25-41B8-AFD0-5EC00CEFF05D + 78 Ceph disk in creation 89C57F98-2FE5-4DC0-89C1-F3AD0CEFF2BE + 79 Ceph crypt disk in creation 89C57F98-2FE5-4DC0-89C1-5EC00CEFF2BE + 80 VMware VMFS AA31E02A-400F-11DB-9590-000C2911D1B8 + 81 VMware Diagnostic 9D275380-40AD-11DB-BF97-000C2911D1B8 + 82 VMware Virtual SAN 381CFCCC-7288-11E0-92EE-000C2911D0B2 + 83 VMware Virsto 77719A0C-A4A0-11E3-A47E-000C29745A24 + 84 VMware Reserved 9198EFFC-31C0-11DB-8F78-000C2911D1B8 + 85 OpenBSD data 824CC7A0-36A8-11E3-890A-952519AD3F61 + 86 QNX6 file system CEF5A9AD-73BC-4601-89F3-CDEEEEE321A1 + 87 Plan 9 partition C91818F9-8025-47AF-89D2-F030D7000C2C + 88 HiFive Unleashed FSBL 5B193300-FC78-40CD-8002-E86C45580B47 + 89 HiFive Unleashed BBL 2E54B353-1271-4842-806F-E436D6AF6985 +