From 6f450ab68f2cfc4926503c714da59485a3433602 Mon Sep 17 00:00:00 2001 From: brent s Date: Thu, 18 Oct 2018 14:13:34 -0400 Subject: [PATCH] updating some scripts - fixes, mostly. conf_minify works WAY better now. --- aif/scripts/post/sshsecure.py | 93 +++++++++++++++++++++++++- ref/ascii | 4 +- ref/progress.py | 2 +- ref/python.tips_tricks_and_dirty_hacks | 13 +++- ssl_tls/certparser.py | 2 +- storage/backups/borg/backup.py | 9 ++- storage/mtree_to_xml.py | 2 + text/conf_minify.py | 56 +++++++--------- 8 files changed, 140 insertions(+), 41 deletions(-) diff --git a/aif/scripts/post/sshsecure.py b/aif/scripts/post/sshsecure.py index 9dac237..e38e4ea 100644 --- a/aif/scripts/post/sshsecure.py +++ b/aif/scripts/post/sshsecure.py @@ -4,13 +4,101 @@ # TODO: check for cryptography module. if it exists, we can do this entirely pythonically # without ever needing to use subprocess/ssh-keygen, i think! +# Thanks to https://stackoverflow.com/a/39126754. + +# stdlib import datetime import glob import os import pwd import re import shutil -import subprocess +import subprocess # REMOVE WHEN SWITCHING TO PURE PYTHON +#### PREP FOR PURE PYTHON IMPLEMENTATION #### +# # non-stdlib - testing and automatic install if necessary. +# # TODO # +# - cryptography module won't generate new-format "openssh-key-v1" keys. +# - See https://github.com/pts/py_ssh_keygen_ed25519 for possible conversion to python 3 +# - https://github.com/openssh/openssh-portable/blob/master/PROTOCOL.key +# - https://github.com/pyca/cryptography/issues/3509 and https://github.com/paramiko/paramiko/issues/1136 +# has_crypto = False +# pure_py = False +# has_pip = False +# pipver = None +# try: +# import cryptography +# has_crypto = True +# except ImportError: +# # We'll try to install it. We set up the logic below. +# try: +# import pip +# has_pip = True +# # We'll use these to create a temporary lib path and remove it when done. +# import sys +# import tempfile +# except ImportError: +# # ABSOLUTE LAST fallback, if we got to THIS case, is to use subprocess. +# has_pip = False +# import subprocess +# +# # Try installing it then! +# if not all((has_crypto, )): +# # venv only included after python 3.3.x. We fallback to subprocess if we can't do dis. +# if sys.hexversion >= 0x30300f0: +# has_ensurepip = False +# import venv +# if not has_pip and sys.hexversion >= 0x30400f0: +# import ensurepip +# has_ensurepip = True +# temppath = tempfile.mkdtemp('_VENV') +# v = venv.create(temppath) +# if has_ensurepip and not has_pip: +# # This SHOULD be unnecessary, but we want to try really hard. +# ensurepip.bootstrap(root = temppath) +# import pip +# has_pip = True +# if has_pip: +# pipver = pip.__version__.split('.') +# # A thousand people are yelling at me for this. +# if int(pipver[0]) >= 10: +# from pip._internal import main as pipinstall +# else: +# pipinstall = pip.main +# if int(pipver[0]) >= 8: +# pipcmd = ['install', +# '--prefix={0}'.format(temppath), +# '--ignore-installed'] +# else: +# pipcmd = ['install', +# '--install-option="--prefix={0}"'.format(temppath), +# '--ignore-installed'] +# # Get the lib path. +# libpath = os.path.join(temppath, 'lib') +# if os.path.exists('{0}64'.format(libpath)) and not os.path.islink('{0}64'.format(libpath)): +# libpath += '64' +# for i in os.listdir(libpath): # TODO: make this more sane. We cheat a bit here by making assumptions. +# if re.search('python([0-9]+(\.[0-9]+)?)?$', i): +# libpath = os.path.join(libpath, i) +# break +# libpath = os.path.join(libpath, 'site-packages') +# sys.prefix = temppath +# for m in ('cryptography', 'ed25519'): +# pipinstall(['install', 'cryptography']) +# sys.path.append(libpath) +# try: +# import cryptography +# has_crypto = True +# except ImportError: # All that trouble for nothin'. Shucks. +# pass +# +# if all((has_crypto, )): +# pure_py = True +# +# if pure_py: +# from cryptography.hazmat.primitives import serialization as crypto_serialization +# from cryptography.hazmat.primitives.asymmetric import rsa +# from cryptography.hazmat.backends import default_backend as crypto_default_backend +# conf_options = {} conf_options['sshd'] = {'KexAlgorithms': 'curve25519-sha256@libssh.org,diffie-hellman-group-exchange-sha256', @@ -31,12 +119,13 @@ conf_options['ssh'] = {'Host': {'*': {'KexAlgorithms': 'curve25519-sha256@libssh 'PubkeyAuthentication': 'yes', 'HostKeyAlgorithms': 'ssh-ed25519-cert-v01@openssh.com,ssh-rsa-cert-v01@openssh.com,ssh-ed25519,ssh-rsa'}}} # Uncomment below if Github still needs diffie-hellman-group-exchange-sha1 sometimes. +# For what it's worth, it doesn't seem to. #conf_options['ssh']['Host']['github.com'] = {'KexAlgorithms': 'curve25519-sha256@libssh.org,diffie-hellman-group-exchange-sha256,' + # 'diffie-hellman-group-exchange-sha1,diffie-hellman-group14-sha1'} def hostKeys(buildmoduli): - # Starting haveged should help lessen the time load, but not much. + # Starting haveged should help lessen the time load a non-negligible amount, especially on virtual platforms. if os.path.lexists('/usr/bin/haveged'): # We could use psutil here, but then that's a python dependency we don't need. # We could parse the /proc directory, but that's quite unnecessary. pgrep's installed by default on Arch. diff --git a/ref/ascii b/ref/ascii index 8bc4397..56fd129 100644 --- a/ref/ascii +++ b/ref/ascii @@ -1,2 +1,4 @@ http://plato.asu.edu/MAT420/beginning_perl/3145_AppF.pdf -http://www.profdavis.net/ASCII_table.pdf \ No newline at end of file +http://www.profdavis.net/ASCII_table.pdf +http://www.aboutmyip.com/AboutMyXApp/AsciiChart.jsp +http://www.robelle.com/smugbook/ascii.html diff --git a/ref/progress.py b/ref/progress.py index 9e97616..d6392e2 100755 --- a/ref/progress.py +++ b/ref/progress.py @@ -11,7 +11,7 @@ def waiter(seconds = 1): max = len(anims) - 1 global is_done print('Beginning dhparam gen...') - # This is just an example commant that takes a looong time. + # This is just an example command that takes a looong time. c = subprocess.Popen(['openssl', 'dhparam', '-out', '/tmp/dhpem', '4096'], #c = subprocess.Popen(['openssl', 'genrsa', '-out', '/tmp/dhpem', '4096'], stdout = subprocess.PIPE, diff --git a/ref/python.tips_tricks_and_dirty_hacks b/ref/python.tips_tricks_and_dirty_hacks index 4a9a70f..e9ab1e1 100644 --- a/ref/python.tips_tricks_and_dirty_hacks +++ b/ref/python.tips_tricks_and_dirty_hacks @@ -26,7 +26,7 @@ ref: https://www.python.org/dev/peps/pep-0008/#imports ############################################################################### -To programmatically install modules via pip if they aren't installed: +To programmatically install modules via pip if they aren't installed (BROKEN IN RECENT PIP VERSIONS; pip._internal.main() i think): ____ import importlib @@ -108,4 +108,13 @@ log = logger.log(name = 'project.name') ############################################################################### # TODO # -https://stackoverflow.com/questions/10265193/python-can-a-class-act-like-a-module \ No newline at end of file +https://stackoverflow.com/questions/10265193/python-can-a-class-act-like-a-module + + +############################################################################### + +To issue an equivalent of "reset" command in linux, assuming console is ANSI-compat, + +print('\x1bc', end = '') + +############################################################################### diff --git a/ssl_tls/certparser.py b/ssl_tls/certparser.py index 9773814..7372595 100755 --- a/ssl_tls/certparser.py +++ b/ssl_tls/certparser.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.6 +#!/usr/bin/env python3 # stdlib import argparse diff --git a/storage/backups/borg/backup.py b/storage/backups/borg/backup.py index a7e0d00..c8b435c 100755 --- a/storage/backups/borg/backup.py +++ b/storage/backups/borg/backup.py @@ -45,6 +45,12 @@ class Backup(object): if self.args['oper'] == 'backup': for d in (self.args['mysqldir'], self.args['stagedir']): os.makedirs(d, exist_ok = True, mode = 0o700) + if self.args['oper'] == 'restore': + self.args['target_dir'] = os.path.abspath(os.path.expanduser( + self.args['target_dir'])) + os.makedirs(os.path.dirname(self.args['oper']), + exist_ok = True, + mode = 0o700) ### LOGGING ### # Thanks to: # https://web.archive.org/web/20170726052946/http://www.lexev.org/en/2013/python-logging-every-day/ @@ -255,7 +261,8 @@ class Backup(object): self.cfg['config']['host'], r, self.args['archive'])) - # TODO: support specific path of extract? + _cmd.append(os.path.abspath(self.args['target_dir'])) + # TODO: support specific path inside archive? # if so, append path(s) here. _env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password'] self.logger.debug('VARS: {0}'.format(vars())) diff --git a/storage/mtree_to_xml.py b/storage/mtree_to_xml.py index 2be58ab..eddcb94 100755 --- a/storage/mtree_to_xml.py +++ b/storage/mtree_to_xml.py @@ -159,6 +159,8 @@ class MtreeXML(object): return(out) def _unset_parse(unsetline): out = {} + if unsetline[1] == 'all': + return(copy.deepcopy(self._tplitem)) for i in unsetline: out[i] = self._tplitem[i] return(out) diff --git a/text/conf_minify.py b/text/conf_minify.py index 4c68736..fed1b2c 100755 --- a/text/conf_minify.py +++ b/text/conf_minify.py @@ -31,21 +31,15 @@ class ConfStripper(object): if not self.comments: if len(self.comment_syms) == 1: if self.inline: - self.regexes.append(re.compile( - '^(.*){0}.*'.format( - self.comment_syms[0]))) + self.regexes.append(re.compile('^([^{0}]*){0}.*'.format(self.comment_syms[0]))) else: - self.regexes.append(re.compile( - '^(\s*){0}.*'.format( - self.comment_syms[0]))) + self.regexes.append(re.compile('^(\s*){0}.*'.format(self.comment_syms[0]))) else: syms = '|'.join(self.comment_syms) if self.inline: - self.regexes.append(re.compile( - '^(.*)({0}).*'.format(syms))) + self.regexes.append(re.compile('^(.*)({0}).*'.format(syms))) else: - self.regexes.append(re.compile( - '^(\s*)({0}).*'.format(syms))) + self.regexes.append(re.compile( '^(\s*)({0}).*'.format(syms))) return() def parse(self, path): @@ -73,7 +67,7 @@ class ConfStripper(object): return(None) try: with open(path, 'r') as f: - conf = [i.strip() for i in f.readlines()] + conf = f.readlines() except UnicodeDecodeError: # It's a binary file. Oops. if self.cli: print('{0}: Binary file? (is not UTF-8/ASCII)'.format(path)) @@ -89,8 +83,13 @@ class ConfStripper(object): # Okay, so now we can actually parse. # Comments first. for idx, line in enumerate(conf): + if line.strip() == '': + continue for r in self.regexes: conf[idx] = r.sub('\g<1>', conf[idx]) + if conf[idx].strip() == '': # The line was "deleted". + conf[idx] = None + conf = [i for i in conf if i is not None] # Then leading spaces... if not self.leading: for idx, line in enumerate(conf): @@ -101,7 +100,7 @@ class ConfStripper(object): conf[idx] = conf[idx].rstrip() # Lastly, if set, remove blank lines. if not self.whitespace: - conf = [i for i in conf if i != ''] + conf = [i for i in conf if i.strip() != ''] return(conf) def recurse(self, path): @@ -139,8 +138,7 @@ class ConfStripper(object): f.write(new_content) except PermissionError: if self.cli: - print('{0}: Cannot write (insufficient permission)'.format( - path)) + print('{0}: Cannot write (insufficient permission)'.format(path)) return() return() @@ -160,9 +158,7 @@ class ConfStripper(object): return(realpaths) def parseArgs(): - args = argparse.ArgumentParser(description = ('Remove extraneous ' + - 'formatting/comments from ' + - 'files')) + args = argparse.ArgumentParser(description = ('Remove extraneous formatting/comments from files')) args.add_argument('-c', '--keep-comments', dest = 'comments', action = 'store_true', @@ -172,15 +168,13 @@ def parseArgs(): dest = 'comment_syms', action = 'append', default = [], - help = ('The character(s) to be treated as comments. ' + - 'Can be specified multiple times (one symbol ' + - 'per flag, please, unless a specific sequence ' + - 'denotes a comment). Default is just #')) + help = ('The character(s) to be treated as comments. ' + 'Can be specified multiple times (one symbol per flag, please, unless a specific ' + 'sequence denotes a comment). Default is just #')) args.add_argument('-i', '--no-inline', dest = 'inline', action = 'store_false', - help = ('If specified, do NOT parse the files as ' + - 'having inline comments (the default is to ' + + help = ('If specified, do NOT parse the files as having inline comments (the default is to ' 'look for inline comments)')) args.add_argument('-s', '--keep-whitespace', dest = 'whitespace', @@ -189,17 +183,15 @@ def parseArgs(): args.add_argument('-t', '--keep-trailing', dest = 'trailing', action = 'store_true', - help = ('If specified, retain trailing whitespace on ' + - 'lines')) + help = ('If specified, retain trailing whitespace on lines')) args.add_argument('-l', '--no-leading-whitespace', dest = 'leading', action = 'store_false', help = ('If specified, REMOVE leading whitespace')) - args.add_argument('-d', '--dry-run', + args.add_argument('-w', '--write', dest = 'dry_run', - action = 'store_true', - help = ('If specified, don\'t actually overwrite the ' + - 'file(s) - just print to stdout instead')) + action = 'store_false', + help = ('If specified, overwrite the file(s) instead of just printing to stdout')) args.add_argument('-S', '--no-symlinks', dest = 'symlinks', action = 'store_false', @@ -207,10 +199,8 @@ def parseArgs(): args.add_argument('paths', metavar = 'PATH/TO/DIR/OR/FILE', nargs = '+', - help = ('The path(s) to the file(s) to strip down. If ' + - 'a directory is given, files will ' + - 'recursively be modified (unless -d/--dry-run ' + - 'is specified). Can be specified multiple ' + + help = ('The path(s) to the file(s) to strip down. If a directory is given, files will ' + 'recursively be printed (unless -w/--write is specified). Can be specified multiple ' 'times')) return(args)