updating some scripts - fixes, mostly. conf_minify works WAY better now.

aif/scripts/post/sshsecure.py

@@ -4,13 +4,101 @@
 # TODO: check for cryptography module. if it exists, we can do this entirely pythonically
 #       without ever needing to use subprocess/ssh-keygen, i think!
 
+# Thanks to https://stackoverflow.com/a/39126754.
+
+# stdlib
 import datetime
 import glob
 import os
 import pwd
 import re
 import shutil
-import subprocess
+import subprocess  # REMOVE WHEN SWITCHING TO PURE PYTHON
+#### PREP FOR PURE PYTHON IMPLEMENTATION ####
+# # non-stdlib - testing and automatic install if necessary.
+# # TODO #
+# - cryptography module won't generate new-format "openssh-key-v1" keys.
+# - See https://github.com/pts/py_ssh_keygen_ed25519 for possible conversion to python 3
+# - https://github.com/openssh/openssh-portable/blob/master/PROTOCOL.key
+# - https://github.com/pyca/cryptography/issues/3509 and https://github.com/paramiko/paramiko/issues/1136
+# has_crypto = False
+# pure_py = False
+# has_pip = False
+# pipver = None
+# try:
+#     import cryptography
+#     has_crypto = True
+# except ImportError:
+#     # We'll try to install it. We set up the logic below.
+#     try:
+#         import pip
+#         has_pip = True
+#         # We'll use these to create a temporary lib path and remove it when done.
+#         import sys
+#         import tempfile
+#     except ImportError:
+#         # ABSOLUTE LAST fallback, if we got to THIS case, is to use subprocess.
+#         has_pip = False
+#         import subprocess
+#
+# # Try installing it then!
+# if not all((has_crypto, )):
+#     # venv only included after python 3.3.x. We fallback to subprocess if we can't do dis.
+#     if sys.hexversion >= 0x30300f0:
+#         has_ensurepip = False
+#         import venv
+#         if not has_pip and sys.hexversion >= 0x30400f0:
+#             import ensurepip
+#             has_ensurepip = True
+#         temppath = tempfile.mkdtemp('_VENV')
+#         v = venv.create(temppath)
+#         if has_ensurepip and not has_pip:
+#             # This SHOULD be unnecessary, but we want to try really hard.
+#             ensurepip.bootstrap(root = temppath)
+#             import pip
+#             has_pip = True
+#         if has_pip:
+#             pipver = pip.__version__.split('.')
+#             # A thousand people are yelling at me for this.
+#             if int(pipver[0]) >= 10:
+#                 from pip._internal import main as pipinstall
+#             else:
+#                 pipinstall = pip.main
+#             if int(pipver[0]) >= 8:
+#                 pipcmd = ['install',
+#                           '--prefix={0}'.format(temppath),
+#                           '--ignore-installed']
+#             else:
+#                 pipcmd = ['install',
+#                           '--install-option="--prefix={0}"'.format(temppath),
+#                           '--ignore-installed']
+#             # Get the lib path.
+#             libpath = os.path.join(temppath, 'lib')
+#             if os.path.exists('{0}64'.format(libpath)) and not os.path.islink('{0}64'.format(libpath)):
+#                 libpath += '64'
+#             for i in os.listdir(libpath):  # TODO: make this more sane. We cheat a bit here by making assumptions.
+#                 if re.search('python([0-9]+(\.[0-9]+)?)?$', i):
+#                     libpath = os.path.join(libpath, i)
+#                     break
+#             libpath = os.path.join(libpath, 'site-packages')
+#             sys.prefix = temppath
+#             for m in ('cryptography', 'ed25519'):
+#                 pipinstall(['install', 'cryptography'])
+#             sys.path.append(libpath)
+#             try:
+#                 import cryptography
+#                 has_crypto = True
+#             except ImportError:  # All that trouble for nothin'. Shucks.
+#                 pass
+#
+# if all((has_crypto, )):
+#     pure_py = True
+#
+# if pure_py:
+#     from cryptography.hazmat.primitives import serialization as crypto_serialization
+#     from cryptography.hazmat.primitives.asymmetric import rsa
+#     from cryptography.hazmat.backends import default_backend as crypto_default_backend
+#
 
 conf_options = {}
 conf_options['sshd'] = {'KexAlgorithms': 'curve25519-sha256@libssh.org,diffie-hellman-group-exchange-sha256',
@@ -31,12 +119,13 @@ conf_options['ssh'] = {'Host': {'*': {'KexAlgorithms': 'curve25519-sha256@libssh
                                       'PubkeyAuthentication': 'yes',
                                       'HostKeyAlgorithms': 'ssh-ed25519-cert-v01@openssh.com,ssh-rsa-cert-v01@openssh.com,ssh-ed25519,ssh-rsa'}}}
 # Uncomment below if Github still needs diffie-hellman-group-exchange-sha1 sometimes.
+# For what it's worth, it doesn't seem to.
 #conf_options['ssh']['Host']['github.com'] = {'KexAlgorithms': 'curve25519-sha256@libssh.org,diffie-hellman-group-exchange-sha256,' +
 #                                             'diffie-hellman-group-exchange-sha1,diffie-hellman-group14-sha1'}
 
 
 def hostKeys(buildmoduli):
-    # Starting haveged should help lessen the time load, but not much.
+    # Starting haveged should help lessen the time load a non-negligible amount, especially on virtual platforms.
     if os.path.lexists('/usr/bin/haveged'):
         # We could use psutil here, but then that's a python dependency we don't need.
         # We could parse the /proc directory, but that's quite unnecessary. pgrep's installed by default on Arch.

ref/ascii

@@ -1,2 +1,4 @@
 http://plato.asu.edu/MAT420/beginning_perl/3145_AppF.pdf
 http://www.profdavis.net/ASCII_table.pdf
+http://www.aboutmyip.com/AboutMyXApp/AsciiChart.jsp
+http://www.robelle.com/smugbook/ascii.html

ref/progress.py

@@ -11,7 +11,7 @@ def waiter(seconds = 1):
     max = len(anims) - 1
     global is_done
     print('Beginning dhparam gen...')
-    # This is just an example commant that takes a looong time.
+    # This is just an example command that takes a looong time.
     c = subprocess.Popen(['openssl', 'dhparam', '-out', '/tmp/dhpem', '4096'],
     #c = subprocess.Popen(['openssl', 'genrsa', '-out', '/tmp/dhpem', '4096'],
                          stdout = subprocess.PIPE,

ref/python.tips_tricks_and_dirty_hacks

@@ -26,7 +26,7 @@ ref: https://www.python.org/dev/peps/pep-0008/#imports
 
 ###############################################################################
 
-To programmatically install modules via pip if they aren't installed:
+To programmatically install modules via pip if they aren't installed (BROKEN IN RECENT PIP VERSIONS; pip._internal.main() i think):
 
 ____
 import importlib
@@ -109,3 +109,12 @@ log = logger.log(name = 'project.name')
 
 # TODO #
 https://stackoverflow.com/questions/10265193/python-can-a-class-act-like-a-module
+
+
+###############################################################################
+
+To issue an equivalent of "reset" command in linux, assuming console is ANSI-compat,
+
+print('\x1bc', end = '')
+
+###############################################################################

ssl_tls/certparser.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3.6
+#!/usr/bin/env python3
 
 # stdlib
 import argparse

storage/backups/borg/backup.py

@@ -45,6 +45,12 @@ class Backup(object):
         if self.args['oper'] == 'backup':
             for d in (self.args['mysqldir'], self.args['stagedir']):
                 os.makedirs(d, exist_ok = True, mode = 0o700)
+        if self.args['oper'] == 'restore':
+            self.args['target_dir'] = os.path.abspath(os.path.expanduser(
+                                                    self.args['target_dir']))
+            os.makedirs(os.path.dirname(self.args['oper']),
+                        exist_ok = True,
+                        mode = 0o700)
         ### LOGGING ###
         # Thanks to:
         # https://web.archive.org/web/20170726052946/http://www.lexev.org/en/2013/python-logging-every-day/
@@ -255,7 +261,8 @@ class Backup(object):
                                                   self.cfg['config']['host'],
                                                   r,
                                                   self.args['archive']))
-            # TODO: support specific path of extract?
+            _cmd.append(os.path.abspath(self.args['target_dir']))
+            # TODO: support specific path inside archive?
             # if so, append path(s) here.
             _env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password']
             self.logger.debug('VARS: {0}'.format(vars()))

storage/mtree_to_xml.py

@@ -159,6 +159,8 @@ class MtreeXML(object):
             return(out)
         def _unset_parse(unsetline):
             out = {}
+            if unsetline[1] == 'all':
+                return(copy.deepcopy(self._tplitem))
             for i in unsetline:
                 out[i] = self._tplitem[i]
             return(out)

text/conf_minify.py

@@ -31,21 +31,15 @@ class ConfStripper(object):
         if not self.comments:
             if len(self.comment_syms) == 1:
                 if self.inline:
-                    self.regexes.append(re.compile(
+                    self.regexes.append(re.compile('^([^{0}]*){0}.*'.format(self.comment_syms[0])))
-                                                '^(.*){0}.*'.format(
-                                                        self.comment_syms[0])))
                 else:
-                    self.regexes.append(re.compile(
+                    self.regexes.append(re.compile('^(\s*){0}.*'.format(self.comment_syms[0])))
-                                                '^(\s*){0}.*'.format(
-                                                        self.comment_syms[0])))
             else:
                 syms = '|'.join(self.comment_syms)
                 if self.inline:
-                    self.regexes.append(re.compile(
+                    self.regexes.append(re.compile('^(.*)({0}).*'.format(syms)))
-                                                '^(.*)({0}).*'.format(syms)))
                 else:
-                    self.regexes.append(re.compile(
+                    self.regexes.append(re.compile( '^(\s*)({0}).*'.format(syms)))
-                                                '^(\s*)({0}).*'.format(syms)))
         return()
 
     def parse(self, path):
@@ -73,7 +67,7 @@ class ConfStripper(object):
             return(None)
         try:
             with open(path, 'r') as f:
-                conf = [i.strip() for i in f.readlines()]
+                conf = f.readlines()
         except UnicodeDecodeError:  # It's a binary file. Oops.
             if self.cli:
                 print('{0}: Binary file? (is not UTF-8/ASCII)'.format(path))
@@ -89,8 +83,13 @@ class ConfStripper(object):
         # Okay, so now we can actually parse.
         # Comments first.
         for idx, line in enumerate(conf):
+            if line.strip() == '':
+                continue
             for r in self.regexes:
                 conf[idx] = r.sub('\g<1>', conf[idx])
+            if conf[idx].strip() == '':  # The line was "deleted".
+                conf[idx] = None
+        conf = [i for i in conf if i is not None]
         # Then leading spaces...
         if not self.leading:
             for idx, line in enumerate(conf):
@@ -101,7 +100,7 @@ class ConfStripper(object):
                 conf[idx] = conf[idx].rstrip()
         # Lastly, if set, remove blank lines.
         if not self.whitespace:
-            conf = [i for i in conf if i != '']
+            conf = [i for i in conf if i.strip() != '']
         return(conf)
 
     def recurse(self, path):
@@ -139,8 +138,7 @@ class ConfStripper(object):
                 f.write(new_content)
         except PermissionError:
             if self.cli:
-                print('{0}: Cannot write (insufficient permission)'.format(
+                print('{0}: Cannot write (insufficient permission)'.format(path))
-                                                                        path))
             return()
         return()
 
@@ -160,9 +158,7 @@ class ConfStripper(object):
         return(realpaths)
 
 def parseArgs():
-    args = argparse.ArgumentParser(description = ('Remove extraneous ' +
+    args = argparse.ArgumentParser(description = ('Remove extraneous formatting/comments from files'))
-                                                  'formatting/comments from ' +
-                                                  'files'))
     args.add_argument('-c', '--keep-comments',
                       dest = 'comments',
                       action = 'store_true',
@@ -172,15 +168,13 @@ def parseArgs():
                       dest = 'comment_syms',
                       action = 'append',
                       default = [],
-                      help = ('The character(s) to be treated as comments. ' +
+                      help = ('The character(s) to be treated as comments. '
-                              'Can be specified multiple times (one symbol ' +
+                              'Can be specified multiple times (one symbol per flag, please, unless a specific '
-                              'per flag, please, unless a specific sequence ' +
+                              'sequence denotes a comment). Default is just #'))
-                              'denotes a comment). Default is just #'))
     args.add_argument('-i', '--no-inline',
                       dest = 'inline',
                       action = 'store_false',
-                      help = ('If specified, do NOT parse the files as ' +
+                      help = ('If specified, do NOT parse the files as having inline comments (the default is to '
-                              'having inline comments (the default is to ' +
                               'look for inline comments)'))
     args.add_argument('-s', '--keep-whitespace',
                       dest = 'whitespace',
@@ -189,17 +183,15 @@ def parseArgs():
     args.add_argument('-t', '--keep-trailing',
                       dest = 'trailing',
                       action = 'store_true',
-                      help = ('If specified, retain trailing whitespace on ' +
+                      help = ('If specified, retain trailing whitespace on lines'))
-                              'lines'))
     args.add_argument('-l', '--no-leading-whitespace',
                       dest = 'leading',
                       action = 'store_false',
                       help = ('If specified, REMOVE leading whitespace'))
-    args.add_argument('-d', '--dry-run',
+    args.add_argument('-w', '--write',
                       dest = 'dry_run',
-                      action = 'store_true',
+                      action = 'store_false',
-                      help = ('If specified, don\'t actually overwrite the ' +
+                      help = ('If specified, overwrite the file(s) instead of just printing to stdout'))
-                              'file(s) - just print to stdout instead'))
     args.add_argument('-S', '--no-symlinks',
                       dest = 'symlinks',
                       action = 'store_false',
@@ -207,10 +199,8 @@ def parseArgs():
     args.add_argument('paths',
                       metavar = 'PATH/TO/DIR/OR/FILE',
                       nargs = '+',
-                      help = ('The path(s) to the file(s) to strip down. If ' +
+                      help = ('The path(s) to the file(s) to strip down. If a directory is given, files will '
-                              'a directory is given, files will ' +
+                              'recursively be printed (unless -w/--write is specified). Can be specified multiple '
-                              'recursively be modified (unless -d/--dry-run ' +
-                              'is specified). Can be specified multiple ' +
                               'times'))
     return(args)