networkmanager almost done; needs auto-dev for wifi/ethernet and handling of auto resolvers i think

aif.xsd

@@ -178,6 +178,15 @@
         </xs:restriction>
     </xs:simpleType>
 
+    <xs:simpleType name="t_gw_ip4">
+        <xs:restriction base="xs:string">
+            <!-- This is a REALLY LAZY regex. Matching IPv4 in regex is ugly as heck, so we do that in-code.
+                 This is just a gatekeeper. -->
+            <xs:pattern value="[0-9.]{7,15}"/>
+            <xs:whiteSpace value="collapse"/>
+        </xs:restriction>
+    </xs:simpleType>
+
     <xs:simpleType name="t_address_ip6">
         <xs:restriction base="xs:string">
             <!-- This is a REALLY LAZY regex. Matching IPv6 in regex is ugly as heck, so we do that in-code.
@@ -187,6 +196,15 @@
         </xs:restriction>
     </xs:simpleType>
 
+    <xs:simpleType name="t_gw_ip6">
+        <xs:restriction base="xs:string">
+            <!-- This is a REALLY LAZY regex. Matching IPv6 in regex is ugly as heck, so we do that in-code.
+                 This is just a gatekeeper. -->
+            <xs:pattern value="[A-Za-z0-9:]+"/>
+            <xs:whiteSpace value="collapse"/>
+        </xs:restriction>
+    </xs:simpleType>
+
     <xs:simpleType name="t_resolver_addr">
         <xs:restriction base="xs:string">
             <!-- This is a REALLY LAZY regex. Matching IPv4/IPv6 in regex is ugly as heck, so we do that in-code.
@@ -224,10 +242,18 @@
                                 <xs:element name="ipv4">
                                     <xs:complexType>
                                         <xs:sequence>
-                                            <xs:element name="address" type="aif:t_address_ip4"
+                                            <xs:element name="address" minOccurs="0" maxOccurs="unbounded">
-                                                        minOccurs="0" maxOccurs="unbounded"/>
+                                                <xs:complexType>
+                                                    <xs:simpleContent>
+                                                        <xs:extension base="aif:t_address_ip4">
+                                                            <xs:attribute name="gateway"
+                                                                          type="aif:t_gw_ip4"
+                                                                          use="optional"/>
+                                                        </xs:extension>
+                                                    </xs:simpleContent>
+                                                </xs:complexType>
+                                            </xs:element>
                                         </xs:sequence>
-                                        <xs:attribute name="gateway" type="aif:t_address_ip4" use="optional"/>
                                     </xs:complexType>
                                     <xs:unique name="uniq_ipv4_addr">
                                         <xs:selector xpath="aif:address"/>
@@ -237,10 +263,18 @@
                                 <xs:element name="ipv6">
                                     <xs:complexType>
                                         <xs:sequence>
-                                            <xs:element name="address" type="aif:t_address_ip6"
+                                            <xs:element name="address" minOccurs="0" maxOccurs="unbounded">
-                                                        minOccurs="0" maxOccurs="unbounded"/>
+                                                <xs:complexType>
+                                                    <xs:simpleContent>
+                                                        <xs:extension base="aif:t_address_ip6">
+                                                            <xs:attribute name="gateway"
+                                                                          type="aif:t_gw_ip6"
+                                                                          use="optional"/>
+                                                        </xs:extension>
+                                                    </xs:simpleContent>
+                                                </xs:complexType>
+                                            </xs:element>
                                         </xs:sequence>
-                                        <xs:attribute name="gateway" type="aif:t_address_ip6" use="optional"/>
                                     </xs:complexType>
                                     <xs:unique name="uniq_ipv6_addr">
                                         <xs:selector xpath="aif:address"/>
@@ -257,12 +291,15 @@
                             <xs:element name="resolver" minOccurs="1" maxOccurs="unbounded"
                                         type="aif:t_resolver_addr"/>
                         </xs:sequence>
+                        <xs:attribute name="noAuto" type="xs:boolean" use="optional" default="false"/>
                     </xs:complexType>
                 </xs:element>
             </xs:choice>
         </xs:sequence>
+        <xs:attribute name="id" type="xs:ID" use="required"/>
         <xs:attribute name="device" type="aif:t_iface_name" use="required"/>
         <xs:attribute name="defroute" type="xs:boolean" use="optional" default="false"/>
+        <xs:attribute name="searchDomain" type="aif:t_nonempty" use="optional"/>
     </xs:complexType>
 
     <xs:simpleType name="t_mac_addr">

aif/network/__init__.py

@@ -1,7 +1,7 @@
-try:
+# try:
-    from . import _common
+#     from . import _common
-except ImportError:
+# except ImportError:
-    pass  # GI isn't supported, so we don't even use a fallback.
+#     pass  # GI isn't supported, so we don't even use a fallback.
 
 from . import netctl
 
@@ -15,10 +15,5 @@ try:
 except ImportError:
     from . import networkd_fallback as networkd
 
-
+from . import networkmanager
-try:
-    from . import networkmanager
-except ImportError:
-    from . import networkmanager_fallback
-
 from . import net

aif/network/_common.py

@@ -1,3 +1,3 @@
-import gi
+# import gi
-gi.require_version('NM', '1.0')
+# gi.require_version('NM', '1.0')
-from gi.repository import GObject, NM, GLib
+# from gi.repository import GObject, NM, GLib

aif/network/net.py

@@ -14,9 +14,11 @@ class Network(object):
         if not self.provider:
             raise RuntimeError('Could not determine handler')
         self.connections = []
+        self._initConns()
 
     def _initConns(self):
         for e in self.xml.xpath('ethernet|wireless'):
+            conn = None
             if e.tag == 'ethernet':
                 conn = self.provider.Ethernet(e)
             elif e.tag == 'wireless':

aif/network/networkmanager.py

@@ -1,47 +1,141 @@
+import configparser
+import datetime
 import ipaddress
+import os
 import uuid
 ##
-from . import _common
+import aif.utils
 
-_NM = _common.NM
+# TODO: auto dev assignment
 
 
 class Connection(object):
     def __init__(self, iface_xml):
         self.xml = iface_xml
+        self.id = self.xml.attrib['id']
+        self.device = self.xml.attrib['device']
+        self.is_defroute = aif.utils.xmlBool(self.xml.attrib.get('defroute', 'false'))
+        self.domain = self.xml.attrib.get('searchDomain', None)
+        self._cfg = None
         self.connection_type = None
         self.provider_type = 'NetworkManager'
-        self.client = _NM.Client.new()
+        self.addrs = {'ipv4': set(),
-        self.addrs = {'ipv4': [],
+                      'ipv6': set()}
-                      'ipv6': []}
         self.resolvers = []
         self.uuid = uuid.uuid4()
         self._initAddrs()
         self._initResolvers()
 
     def _initAddrs(self):
-        for t in ('ipv4', 'ipv6'):
+        # These tuples follow either:
-            for a in self.xml.findall('addresses/{0}/address'.format(t)):
+        #   ('dhcp'/'dhcp6'/'slaac', None, None) for auto configuration
+        #   (ipaddress.IPv4/6Address(IP), CIDR, ipaddress.IPv4/6Address(GW)) for static configuration
+        for addrtype in ('ipv4', 'ipv6'):
+            for a in self.xml.findall('addresses/{0}/address'.format(addrtype)):
                 if a.text in ('dhcp', 'dhcp6', 'slaac'):
-                    addr = net = None
+                    addr = a.text
+                    net = None
+                    gw = None
                 else:
                     components = a.text.split('/')
                     if len(components) > 2:
                         raise ValueError('Invalid IP/CIDR format: {0}'.format(a.text))
                     if len(components) == 1:
                         addr = components[0]
-                        if t == 'ipv4':
+                        if addrtype == 'ipv4':
                             components.append('24')
-                        elif t == 'ipv6':
+                        elif addrtype == 'ipv6':
                             components.append('64')
                     addr = ipaddress.ip_address(components[0])
                     net = ipaddress.ip_network('/'.join(components), strict = False)
-                self.addrs[t].append((addr, net))
+                    gw = ipaddress.ip_address(a.attrib.get('gateway'))
+                self.addrs[addrtype].add((addr, net, gw))
+            self.addrs[addrtype] = list(self.addrs[addrtype])
+        return()
+
+    def _initCfg(self):
+        self._cfg = configparser.ConfigParser()
+        self._cfg.optionxform = str
+        self._cfg['connection'] = {'id': self.id,
+                                   'uuid': self.uuid,
+                                   'type': self.connection_type,
+                                   'interface-name': self.device,
+                                   'permissions': '',
+                                   'timestamp': datetime.datetime.utcnow().timestamp()}
+        # We *theoretically* could do this in _initAddrs() but we do it separately so we can trim out duplicates.
+        for addrtype, addrs in self.addrs.items():
+            self._cfg[addrtype] = {}
+            cidr_gws = {}
+            self._cfg[addrtype]['dns-search'] = (self.domain if self.domain else '')
+            if addrtype == 'ipv6':
+                self._cfg[addrtype]['addr-gen-mode'] = 'stable-privacy'
+            if not addrs:
+                self._cfg[addrtype]['method'] = 'ignore'
+            else:
+                self._cfg[addrtype]['method'] = 'manual'
+                for idx, (ip, cidr, gw) in enumerate(addrs):
+                    if cidr not in cidr_gws.keys():
+                        cidr_gws[cidr] = gw
+                        new_cidr = True
+                    else:
+                        new_cidr = False
+                    if addrtype == 'ipv4':
+                        if ip == 'dhcp':
+                            self._cfg[addrtype]['method'] = 'auto'
+                            continue
+                    elif addrtype == 'ipv6':
+                        if ip == 'dhcp6':
+                            self._cfg[addrtype]['method'] = 'dhcp'
+                            continue
+                        elif ip == 'slaac':
+                            self._cfg[addrtype]['method'] = 'auto'
+                            continue
+                    addrnum = idx + 1
+                    addr_str = '{0}/{1}'.format(str(ip), str(cidr.prefixlen))
+                    if new_cidr:
+                        addr_str = '{0},{1}'.format(addr_str, str(gw))
+                    self._cfg[addrtype]['address{0}'.format(addrnum)] = addr_str
+            for r in self.resolvers:
+                if addrtype == 'ipv{0}'.format(r.version):
+                    if 'dns' not in self._cfg[addrtype]:
+                        self._cfg[addrtype]['dns'] = []
+                    self._cfg[addrtype]['dns'].append(str(r))
+            if 'dns' in self._cfg[addrtype].keys():
+                self._cfg[addrtype]['dns'] = '{0};'.format(';'.join(self._cfg[addrtype]['dns']))
+        self._initConnCfg()
+        return()
+
+    def _initConnCfg(self):
+        # A dummy method; this is overridden by the subclasses.
+        # It's honestly here to make my IDE stop complaining. :)
+        pass
         return()
 
     def _initResolvers(self):
         for r in self.xml.findall('resolvers/resolver'):
-            self.resolvers.append(ipaddress.ip_address(r.text))
+            resolver = ipaddress.ip_address(r.text)
+            if resolver not in self.resolvers:
+                self.resolvers.append(resolver)
+        return()
+
+    def writeConf(self, chroot_base):
+        cfgroot = os.path.join(chroot_base, 'etc', 'NetworkManager')
+        cfgdir = os.path.join(cfgroot, 'system-connections')
+        cfgpath = os.path.join(cfgdir, '{0}.nmconnection'.format(self.id))
+        os.makedirs(cfgdir, exist_ok = True)
+        with open(cfgpath, 'w') as fh:
+            self._cfg.write(fh, space_around_delimiters = False)
+        for root, dirs, files in os.walk(cfgroot):
+            os.chown(root, 0, 0)
+            for d in dirs:
+                dpath = os.path.join(root, d)
+                os.chown(dpath, 0, 0)
+            for f in files:
+                fpath = os.path.join(root, f)
+                os.chown(fpath, 0, 0)
+        os.chmod(cfgroot, 0o0755)
+        os.chmod(cfgdir, 0o0700)
+        os.chmod(cfgpath, 0o0600)
         return()
 
 
@@ -49,9 +143,17 @@ class Ethernet(Connection):
     def __init__(self, iface_xml):
         super().__init__(iface_xml)
         self.connection_type = 'ethernet'
+        self._initCfg()
+
+    def _initConnCfg(self):
+        pass
 
 
 class Wireless(Connection):
     def __init__(self, iface_xml):
         super().__init__(iface_xml)
         self.connection_type = 'wireless'
+        self._initCfg()
+
+    def _initConnCfg(self):
+        pass

aif/network/networkmanager_fallback.py

@@ -1 +0,0 @@
-import subprocess

docs/MANUAL.adoc

@@ -571,6 +571,15 @@ Using `start`/`stop` attributes makes sense for disk partitions because they ope
 
 LVM (LVs, in particular), however, aren't consecutive. There *is* no concept of a "start" and "stop" for an LV; LVM uses chunks called "(physical) extents" rather than sectors, and VGs don't have geometry since they're essentially a pool of blocks. This is also why the modifiers like `-` and `+` aren't allowed for LV sizes - they're position-based.
 
+=== "Why aren't the network settings in <network> being applied during install?"
+Simply put, a logical race condition. In order for probably 90+% of AIF-NG deploys to bootstrap, they fetch their XML configuration via a network URI (rather than a file URI). This means it needs a network connection that pre-exists in the *install environment* (LiveCD, LiveUSB, PXE/iPXE, etc.) before it even knows what network configuration you want the *persistent environment* to have.
+
+Granted, this is a moot point if you're using a *`file://`* URI for the XML configuration, but this is not a very flexible means regardless. If demand increases for this, future releases may include this functionality.
+
+If you desire the configuration to be applied *during* the install, you can do it yourself in an `/aif/scripts/pre/script` or `/aif/scripts/pkg/script` script. The fetched XML file can be found at `/var/tmp/AIF.xml` in the install environment. (Alternatively, configure the network yourself procedurally using one of those scripts).
+
+If you wish to SSH into the install environment to check the status/progress of the install, it is recommended that you set up a static lease (if using DHCP) or use SLAAC (if using IPv6) beforehand and configure your install environment beforehand. Remember, AIF-NG only *installs* Arch Linux; it tries very hard to *not* interact with the install environment.
+
 
 == Bug Reports/Feature Requests
 NOTE: It is possible to submit a bug or feature request without registering in my bugtracker. One of my pet peeves is needing to create an account/register on a bugtracker simply to report a bug! The following links only require an email address to file a bug (which is necessary in case I need any further clarification from you or to keep you updated on the status of the bug/feature request -- so please be sure to use a valid email address).

docs/TODO

@@ -28,9 +28,11 @@
 	tmp on /mnt/aif/tmp type tmpfs (rw,nosuid,nodev)
   OR just use pyalpm
 
-DOCUMENTATION: aif-config.py (and note sample json as well)
+DOCUMENTATION:
+- https://stackoverflow.com/questions/237938/how-to-convert-xsd-to-human-readable-documentation ?
+- (*) https://stackoverflow.com/a/6686367
 
-for network configuration, add in support for using a device's MAC address instead of interface name
+for network configuration, add in support for using a device's MAC address instead of interface name?
 
 also:
 -create boot media with bdisk since default arch doesn't even have python 3
@@ -48,4 +50,4 @@ https://msdn.microsoft.com/en-us/library/dd489258.aspx
 
 if i ever need a list of GPT GUIDs, maybe to do some fancy GUID-to-name-and-back mapping?
 https://en.wikipedia.org/wiki/GUID_Partition_Table#Partition_type_GUIDs
-(mapping can be done via https://stackoverflow.com/questions/483666/reverse-invert-a-dictionary-mapping)
+(mapping can be done via https://stackoverflow.com/questions/483666/reverse-invert-a-dictionary-mapping)

examples/aif.xml

@@ -120,22 +120,25 @@
         </mountPoints>
     </storage>
     <network hostname="aiftest.square-r00t.net" provider="netctl">
-        <ethernet device="auto" defroute="true">
+        <ethernet id="lan" device="auto" defroute="true" searchDomain="domain.tld">
             <addresses>
                 <ipv4>
                     <address>dhcp</address>
-                    <address>192.168.1.5/24</address>
+                    <address gateway="192.168.1.1">192.168.1.5/24</address>
                 </ipv4>
                 <ipv6>
                     <address>slaac</address>
                     <address>fde4:16b9:654b:bbfa::15/64</address>
                 </ipv6>
             </addresses>
-            <resolvers>
+            <resolvers noAuto="true">
+                <resolver>64.6.64.6</resolver>
+                <resolver>4.2.2.1</resolver>
                 <resolver>8.8.8.8</resolver>
             </resolvers>
         </ethernet>
-        <wireless device="wlp2s0" essid="MyWirelessLan" bssid="00-00-5E-00-53-00" defroute="false">
+        <wireless id="wlan" device="wlp2s0" essid="MyWirelessLan"
+                  bssid="00-00-5E-00-53-00" defroute="false" searchDomain="wifi.lan">
             <addresses>
                 <ipv4>
                     <address>dhcp</address>