r00t2/aif-ng
1
0
Fork 0
Code Issues Pull Requests Projects Releases Activity

cleaned up, etc.

Browse Source
This commit is contained in:
brent s 2019-10-30 03:46:33 -04:00
parent f0d93658d0
commit c4386d55d1
10 changed files with 33 additions and 396 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
aif/constants.py
View File

@ -1,3 +1,12 @@
import os version = '0.2.0'
import re external_deps = ['blkinfo',
'gpg',
'lxml',
'mdstat',
'passlib',
'psutil',
'pyparted',
'pyroute2',
'pytz',
'requests',
'validators']

26
aif/disk/block.py
View File

@ -18,7 +18,7 @@ import blkinfo
import parted # https://www.gnu.org/software/parted/api/index.html import parted # https://www.gnu.org/software/parted/api/index.html
import psutil import psutil
## ##
from aif.utils import xmlBool from aif.utils import xmlBool, size
PARTED_FSTYPES = sorted(list(dict(vars(parted.filesystem))['fileSystemType'].keys())) PARTED_FSTYPES = sorted(list(dict(vars(parted.filesystem))['fileSystemType'].keys()))
@ -26,23 +26,11 @@ PARTED_FLAGS = sorted(list(parted.partition.partitionFlag.values()))
IDX_FLAG = dict(parted.partition.partitionFlag) IDX_FLAG = dict(parted.partition.partitionFlag)
FLAG_IDX = {v: k for k, v in IDX_FLAG.items()} FLAG_IDX = {v: k for k, v in IDX_FLAG.items()}
# parted lib can do SI or IEC (see table to right at https://en.wikipedia.org/wiki/Binary_prefix) # parted lib can do SI or IEC. So can we.
# We bit-shift to do conversions:
# https://stackoverflow.com/a/12912296/733214
# https://stackoverflow.com/a/52684562/733214
_units = {'B': 0,
'kB': 7,
'MB': 17,
'GB': 27,
'TB': 37,
'KiB': 10,
'MiB': 20,
'GiB': 30,
'TiB': 40}
_pos_re = re.compile((r'^(?P<pos_or_neg>-|\+)?\s*' _pos_re = re.compile((r'^(?P<pos_or_neg>-|\+)?\s*'
r'(?P<size>[0-9]+)\s*' r'(?P<size>[0-9]+)\s*'
# empty means size in sectors # empty means size in sectors
r'(?P<pct_unit_or_sct>%|{0}|)\s*$'.format('|'.join(list(_units.keys()))) r'(?P<pct_unit_or_sct>%|{0}|)\s*$'.format('|'.join(size.valid_storage))
)) ))
@ -57,11 +45,11 @@ def convertSizeUnit(pos):
from_beginning = False from_beginning = False
else: else:
from_beginning = pos_or_neg from_beginning = pos_or_neg
size = int(pos.group('size')) _size = int(pos.group('size'))
amt_type = pos.group('pct_unit_or_sct').strip() amt_type = pos.group('pct_unit_or_sct').strip()
else: else:
raise ValueError('Invalid size specified: {0}'.format(orig_pos)) raise ValueError('Invalid size specified: {0}'.format(orig_pos))
return((from_beginning, size, amt_type)) return((from_beginning, _size, amt_type))
class Partition(object): class Partition(object):
@ -105,8 +93,8 @@ class Partition(object):
sectors = x['size'] sectors = x['size']
if x['type'] == '%': if x['type'] == '%':
sectors = int(self.device.getLength() / x['size']) sectors = int(self.device.getLength() / x['size'])
elif x['type'] in _units.keys(): else:
sectors = int(x['size'] << _units[x['type']] / self.device.sectorSize) sectors = int(size.convertStorage(x['size'], x['type'], target = 'B') / self.device.sectorSize)
sizes[s] = (sectors, x['from_bgn']) sizes[s] = (sectors, x['from_bgn'])
if sizes['start'][1] is not None: if sizes['start'][1] is not None:
if sizes['start'][1]: if sizes['start'][1]:

5
aif/disk/filesystem.py
View File

@ -36,8 +36,9 @@ for i in os.listdir(_mod_dir):
if fs_name: if fs_name:
# The kernel *probably* has autoloading enabled, but in case it doesn't... # The kernel *probably* has autoloading enabled, but in case it doesn't...
# TODO: logging! # TODO: logging!
subprocess.run(['modprobe', fs_name]) if os.getuid() == 0:
FS_FSTYPES.append(fs_name) subprocess.run(['modprobe', fs_name])
FS_FSTYPES.append(fs_name)
class FS(object): class FS(object):

2
aif/disk/mdadm.py
View File

@ -11,7 +11,7 @@ from aif.disk.block import Disk
from aif.disk.block import Partition from aif.disk.block import Partition
SUPPORTED_LEVELS = (0, 1, 4, 5, 6) SUPPORTED_LEVELS = (0, 1, 4, 5, 6, 10)
SUPPORTED_METADATA = ('0', '0.90', '1', '1.0', '1.1', '1.2', 'default', 'ddf', 'imsm') SUPPORTED_METADATA = ('0', '0.90', '1', '1.0', '1.1', '1.2', 'default', 'ddf', 'imsm')
SUPPORTED_LAYOUTS = {5: (re.compile(r'^((left|right)-a?symmetric|[lr][as]|' SUPPORTED_LAYOUTS = {5: (re.compile(r'^((left|right)-a?symmetric|[lr][as]|'
r'parity-(fir|la)st|' r'parity-(fir|la)st|'

8
aif/envsetup.py
View File

@ -11,10 +11,8 @@ import subprocess
import sys import sys
import tempfile import tempfile
import venv import venv
##
# TODO: a more consistent way of managing deps? import aif.constants
depmods = ['blkinfo', 'gpg', 'lxml', 'mdstat', 'passlib', 'psutil',
'pyparted', 'pyroute2', 'pytz', 'requests', 'validators']
class EnvBuilder(object): class EnvBuilder(object):
def __init__(self): def __init__(self):
@ -37,7 +35,7 @@ class EnvBuilder(object):
# This is SO. DUMB. WHY DO I HAVE TO CALL PIP FROM A SHELL. IT'S WRITTEN IN PYTHON. # This is SO. DUMB. WHY DO I HAVE TO CALL PIP FROM A SHELL. IT'S WRITTEN IN PYTHON.
# https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program # https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program
# TODO: logging # TODO: logging
for m in depmods: for m in aif.constants.external_deps:
pip_cmd = [os.path.join(self.vdir, pip_cmd = [os.path.join(self.vdir,
'bin', 'bin',
'python3'), 'python3'),

6
aif/utils.py
View File

@ -38,6 +38,8 @@ def collapseValues(d, vallist = None):
class _Sizer(object): class _Sizer(object):
def __init__(self): def __init__(self):
# We use different methods for converting between storage and BW, and different multipliers for each subtype. # We use different methods for converting between storage and BW, and different multipliers for each subtype.
# https://stackoverflow.com/a/12912296/733214
# https://stackoverflow.com/a/52684562/733214
# https://stackoverflow.com/questions/5194057/better-way-to-convert-file-sizes-in-python # https://stackoverflow.com/questions/5194057/better-way-to-convert-file-sizes-in-python
# https://en.wikipedia.org/wiki/Orders_of_magnitude_(data) # https://en.wikipedia.org/wiki/Orders_of_magnitude_(data)
# https://en.wikipedia.org/wiki/Binary_prefix # https://en.wikipedia.org/wiki/Binary_prefix
@ -92,13 +94,13 @@ class _Sizer(object):
for unit_type, convpair in self.storageUnits.items(): for unit_type, convpair in self.storageUnits.items():
for f, l in convpair.items(): for f, l in convpair.items():
for suffix in l: for suffix in l:
if suffix not in self.valid_storage: if suffix not in self.valid_storage and suffix:
self.valid_storage.append(suffix) self.valid_storage.append(suffix)
self.valid_bw = [] self.valid_bw = []
for unit_type, convpair in self.bwUnits.items(): for unit_type, convpair in self.bwUnits.items():
for f, l in convpair.items(): for f, l in convpair.items():
for suffix in l: for suffix in l:
if suffix not in self.valid_bw: if suffix not in self.valid_bw and suffix:
self.valid_bw.append(suffix) self.valid_bw.append(suffix)
def convert(self, n, suffix): def convert(self, n, suffix):

104
extras/bdisk.build.ini
View File

@ -1,104 +0,0 @@
###########################################################
## BUILD.CONF SAMPLE FILE ##
###########################################################
#
# This file is used to define various variables/settings
# used by the build script.
#
# For full (perhaps overly-verbose ;) documentation, please
# see:
# https://bdisk.square-r00t.net/#_the_code_build_ini_code_file
# Or simply refer to the section titled "The build.ini File"
# in the user manual.
[bdisk]
name = AIF
uxname = aif
pname = AIF-NG
ver = 1.00
dev = r00t^2
email = bts@square-r00t.net
desc = See https://aif.square-r00t.net/
uri = https://aif.square-r00t.net/
root_password = BLANK
user = no
[user]
username = ${bdisk:uxname}
name = Default user
password = BLANK
[source_x86_64]
mirror = mirror.us.leaseweb.net
#mirrorproto = https
mirrorproto = http
mirrorpath = /archlinux/iso/latest/
mirrorfile =
mirrorchksum = ${mirrorpath}sha1sums.txt
chksumtype = sha1
mirrorgpgsig = .sig
gpgkey = 4AA4767BBC9C4B1D18AE28B77F2D434B9741E8AC
gpgkeyserver =
[source_i686]
mirror = mirror.us.leaseweb.net
#mirrorproto = https
mirrorproto = http
mirrorpath = /archlinux/iso/latest/
mirrorfile =
mirrorchksum = ${mirrorpath}sha1sums.txt
chksumtype = sha1
mirrorgpgsig = .sig
gpgkey = 7F2D434B9741E8AC
gpgkeyserver =
[build]
gpg = yes
dlpath = /var/tmp/${bdisk:uxname}
chrootdir = /var/tmp/chroots
basedir = /opt/dev/bdisk
isodir = ${dlpath}/iso
srcdir = ${dlpath}/src
prepdir = ${dlpath}/temp
archboot = ${prepdir}/${bdisk:name}
mountpt = /mnt/${bdisk:uxname}
multiarch = 64
sign = yes
ipxe = yes
i_am_a_racecar = yes
[gpg]
mygpgkey = 748231EBCBD808A14F5E85D28C004C2F93481F6B
mygpghome = /root/.gnupg
[sync]
http = yes
tftp = yes
git = no
rsync = no
[http]
path = ${build:dlpath}/http
user = root
group = root
[tftp]
path = ${build:dlpath}/tftpboot
user = root
group = root
[ipxe]
iso = yes
uri = https://aif.square-r00t.net/boot.ipxe
ssldir = ${build:dlpath}/ssl
ssl_ca = ${ssldir}/ca.crt
ssl_cakey = ${ssldir}/ca.key
ssl_crt = ${ssldir}/main.crt
ssl_key = ${ssldir}/main.key
[rsync]
#host = 10.1.1.1
host = bdisk.square-r00t.net
user = root
path = /srv/http/bdisk_ipxe
iso = yes

208
extras/createtest.expect
View File

@ -1,208 +0,0 @@
#!/usr/bin/expect -f
log_file -noappend /tmp/expect.log
set force_conservative 0 ;# set to 1 to force conservative mode even if
;# script wasn't run conservatively originally
if {$force_conservative} {
set send_slow {1 .1}
proc send {ignore arg} {
sleep .1
exp_send -s -- $arg
}
}
#set send_slow {10 .001}
set timeout -1
#spawn ./aif-config.py create -v:r -f /tmp/aif.xml
spawn ./aif-config.py create -v -f /tmp/aif.xml
## disks
send -- "/dev/sda,/dev/sdb\r"
# sda
send -- "gpt\r"
send -- "2\r"
# sda1
send -- "0%\r"
send -- "95%\r"
send -- "8300\r"
# sda2
send -- "95%\r"
send -- "100%\r"
send -- "ef00\r"
# sdb
send -- "gpt\r"
send -- "3\r"
# sdb1
send -- "0%\r"
send -- "47%\r"
send -- "8300\r"
# sdb2
send -- "47%\r"
send -- "95%\r"
send -- "8300\r"
# sdb3
send -- "95%\r"
send -- "100%\r"
send -- "8200\r"
## mounts
send -- "/mnt/aif,/mnt/aif/boot,/mnt/aif/home,/mnt/aif/mnt/data,swap\r"
# /mnt/aif
send -- "/dev/sda1\r"
send -- "1\r"
send -- "ext4\r"
send -- "defaults\r"
# /mnt/aif/boot
send -- "/dev/sda2\r"
send -- "2\r"
send -- "vfat\r"
send -- "rw,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=iso8859-1,shortname=mixed,errors=remount-ro\r"
# /mnt/aif/home
send -- "/dev/sdb1\r"
send -- "3\r"
send -- "ext4\r"
send -- "defaults\r"
# /mnt/aif/mnt/data
send -- "/dev/sdb2\r"
send -- "4\r"
send -- "ext4\r"
send -- "defaults\r"
# swap
send -- "/dev/sdb3\r"
send -- "5\r"
## network
# hostname
send -- "aif.loc.lan\r"
# interface
send -- "ens3\r"
send -- "auto\r"
send -- "ipv4\r"
# add another interface?
send -- "y\r"
# second interface
send -- "ens4\r"
send -- "192.168.1.2/24\r"
send -- "192.168.1.1\r"
send -- "4.2.2.1,4.2.2.2\r"
# add another interface? default is no
send -- "\r"
## system
# timezone (default is UTC)
send -- "\r"
# locale (default is en_US.UTF-8
send -- "\r"
# chroot path
send -- "/mnt/aif\r"
# kbd (default is US)
send -- "\r"
# reboot host after install? default is yes
send -- "\r"
# root password
sleep 2
send -- "test\r"
sleep 2
expect *
# add user?
send -- "y\r"
# user
send -- "aifusr\r"
# sudo access
send -- "y\r"
# password
sleep 2
send -- "test\r"
sleep 2
send -- "A Test User\r"
# uid (default is autogen)
send -- "\r"
# primary group (default is autogen'd based on username)
send -- "\r"
# home dir (default is e.g. /home/username)
send -- "\r"
# add exta groups?
send -- "y\r"
# extra group
send -- "users\r"
# need to be created? default is no
send -- "\r"
# add another extra group? default is no
send -- "\r"
# add more users? default is no
send -- "\r"
# enable/disable services
send -- "y\r"
# service
send -- "sshd\r"
# enable? default is yes
send -- "\r"
# manage another service? default is no
send -- "\r"
# packager (default is pacman)
send -- "\r"
# review default repos? default is yes
send -- "\r"
# edit any of them?
send -- "y\r"
# edit the 6th repo (multilib)
send -- "6\r"
# enabled?
send -- "y\r"
# siglevel (default is unchanged)
send -- "\r"
# mirror URI (default is unchanged)
send -- "\r"
# edit another repo? default is no
send -- "\r"
# add additional repositories? default is no
send -- "\r"
# modify default mirrorlist?
send -- "y\r"
# URI for mirror
send -- "http://mirrors.advancedhosters.com/archlinux/\$repo/os/\$arch\r"
# add another?
send -- "y\r"
send -- "http://mirror.us.leaseweb.net/archlinux/\$repo/os/\$arch\r"
send -- "y\r"
send -- "http://arch.mirror.constant.com/\$repo/os/\$arch\r"
send -- "y\r"
send -- "http://mirror.vtti.vt.edu/archlinux/\$repo/os/\$arch\r"
send -- "y\r"
send -- "http://arch.mirrors.pair.com/\$repo/os/\$arch\r"
send -- "y\r"
send -- "http://mirror.yellowfiber.net/archlinux/\$repo/os/\$arch\r"
send -- "\r"
# install extra software?
send -- "y\r"
# software
send -- "openssh\r"
# repository (optional)
send -- "\r"
# add another package?
send -- "\r"
# bootloader (default is grub)
send -- "\r"
# system supports UEFI? default is yes
send -- "\r"
# ESP/EFI system partition
send -- "/boot\r"
# any hook scripts? default is no
send -- "y\r"
# pre, pkg, or post
send -- "post\r"
# script URI
send -- "https://aif.square-r00t.net/sample-scripts/post/first.sh\r"
# order for the execution run
send -- "1\r"
# auth required?
send -- "y\r"
# basic/digest? default is basic
send -- "digest\r"
# if digest, realm
send -- "realmname\r"
# user
send -- "test\r"
# password
send -- "password\r"
# would you like to add another script? default is no
send -- "\r"
interact
expect eof

49
extras/txttojson.py
View File

@ -1,49 +0,0 @@
#!/usr/bin/env python3
import argparse
import json
import os
import pprint
#import re
try:
import yaml
except:
exit('You need pyYAML.')
def parseArgs():
args = argparse.ArgumentParser()
args.add_argument('-i',
'--in',
dest = 'infile',
required = True,
help = 'The plaintext representation of a python dict')
args.add_argument('-o',
'--out',
dest = 'outfile',
required = True,
help = 'The JSON file to create')
return(args)
def main():
args = vars(parseArgs().parse_args())
infile = os.path.abspath(os.path.normpath(args['infile']))
outfile = os.path.abspath(os.path.normpath(args['outfile']))
if not os.path.lexists(infile):
exit('Input file doesn\'t exist.')
#try:
with open(outfile, 'w') as outgoing:
with open(infile, 'r') as incoming:
#data = re.sub("'", '"', incoming.read())
#outgoing.write(data)
#d = json.dumps(data, ensure_ascii = False)
#d = json.dumps(incoming.read().replace("'", '"'))
d = yaml.load(incoming.read())
pprint.pprint(d)
j = json.dumps(d, indent = 4)
outgoing.write(j)
#except:
#exit('Error when trying to read/write file(s).')
return()
if __name__ == '__main__':
main()

6
setup.py
View File

@ -1,11 +1,12 @@
import setuptools import setuptools
import aif.constants as PROJ_CONST
with open('README', 'r') as fh: with open('README', 'r') as fh:
long_description = fh.read() long_description = fh.read()
setuptools.setup( setuptools.setup(
name = 'aif', name = 'aif',
version = '0.2.0', version = PROJ_CONST.version,
author = 'Brent S.', author = 'Brent S.',
author_email = 'bts@square-r00t.net', author_email = 'bts@square-r00t.net',
description = 'Arch Installation Framework (Next Generation)', description = 'Arch Installation Framework (Next Generation)',
@ -31,6 +32,5 @@ setuptools.setup(
project_urls = {'Documentation': 'https://aif-ng.io/', project_urls = {'Documentation': 'https://aif-ng.io/',
'Source': 'https://git.square-r00t.net/AIF-NG/', 'Source': 'https://git.square-r00t.net/AIF-NG/',
'Tracker': 'https://bugs.square-r00t.net/index.php?project=9'}, 'Tracker': 'https://bugs.square-r00t.net/index.php?project=9'},
install_requires = ['blkinfo', 'gpg', 'lxml', 'mdstat', 'passlib', 'psutil', install_requires = PROJ_CONST.external_deps
'pyparted', 'pyroute2', 'pytz', 'requests', 'validators']
) )