updating some scripts - fixes, mostly. conf_minify works WAY better now.

This commit is contained in:
brent s 2018-10-18 14:13:34 -04:00
parent d84a98520a
commit 6f450ab68f
8 changed files with 140 additions and 41 deletions

View File

@ -4,13 +4,101 @@
# TODO: check for cryptography module. if it exists, we can do this entirely pythonically # TODO: check for cryptography module. if it exists, we can do this entirely pythonically
# without ever needing to use subprocess/ssh-keygen, i think! # without ever needing to use subprocess/ssh-keygen, i think!


# Thanks to https://stackoverflow.com/a/39126754.

# stdlib
import datetime import datetime
import glob import glob
import os import os
import pwd import pwd
import re import re
import shutil import shutil
import subprocess import subprocess # REMOVE WHEN SWITCHING TO PURE PYTHON
#### PREP FOR PURE PYTHON IMPLEMENTATION ####
# # non-stdlib - testing and automatic install if necessary.
# # TODO #
# - cryptography module won't generate new-format "openssh-key-v1" keys.
# - See https://github.com/pts/py_ssh_keygen_ed25519 for possible conversion to python 3
# - https://github.com/openssh/openssh-portable/blob/master/PROTOCOL.key
# - https://github.com/pyca/cryptography/issues/3509 and https://github.com/paramiko/paramiko/issues/1136
# has_crypto = False
# pure_py = False
# has_pip = False
# pipver = None
# try:
# import cryptography
# has_crypto = True
# except ImportError:
# # We'll try to install it. We set up the logic below.
# try:
# import pip
# has_pip = True
# # We'll use these to create a temporary lib path and remove it when done.
# import sys
# import tempfile
# except ImportError:
# # ABSOLUTE LAST fallback, if we got to THIS case, is to use subprocess.
# has_pip = False
# import subprocess
#
# # Try installing it then!
# if not all((has_crypto, )):
# # venv only included after python 3.3.x. We fallback to subprocess if we can't do dis.
# if sys.hexversion >= 0x30300f0:
# has_ensurepip = False
# import venv
# if not has_pip and sys.hexversion >= 0x30400f0:
# import ensurepip
# has_ensurepip = True
# temppath = tempfile.mkdtemp('_VENV')
# v = venv.create(temppath)
# if has_ensurepip and not has_pip:
# # This SHOULD be unnecessary, but we want to try really hard.
# ensurepip.bootstrap(root = temppath)
# import pip
# has_pip = True
# if has_pip:
# pipver = pip.__version__.split('.')
# # A thousand people are yelling at me for this.
# if int(pipver[0]) >= 10:
# from pip._internal import main as pipinstall
# else:
# pipinstall = pip.main
# if int(pipver[0]) >= 8:
# pipcmd = ['install',
# '--prefix={0}'.format(temppath),
# '--ignore-installed']
# else:
# pipcmd = ['install',
# '--install-option="--prefix={0}"'.format(temppath),
# '--ignore-installed']
# # Get the lib path.
# libpath = os.path.join(temppath, 'lib')
# if os.path.exists('{0}64'.format(libpath)) and not os.path.islink('{0}64'.format(libpath)):
# libpath += '64'
# for i in os.listdir(libpath): # TODO: make this more sane. We cheat a bit here by making assumptions.
# if re.search('python([0-9]+(\.[0-9]+)?)?$', i):
# libpath = os.path.join(libpath, i)
# break
# libpath = os.path.join(libpath, 'site-packages')
# sys.prefix = temppath
# for m in ('cryptography', 'ed25519'):
# pipinstall(['install', 'cryptography'])
# sys.path.append(libpath)
# try:
# import cryptography
# has_crypto = True
# except ImportError: # All that trouble for nothin'. Shucks.
# pass
#
# if all((has_crypto, )):
# pure_py = True
#
# if pure_py:
# from cryptography.hazmat.primitives import serialization as crypto_serialization
# from cryptography.hazmat.primitives.asymmetric import rsa
# from cryptography.hazmat.backends import default_backend as crypto_default_backend
#


conf_options = {} conf_options = {}
conf_options['sshd'] = {'KexAlgorithms': 'curve25519-sha256@libssh.org,diffie-hellman-group-exchange-sha256', conf_options['sshd'] = {'KexAlgorithms': 'curve25519-sha256@libssh.org,diffie-hellman-group-exchange-sha256',
@ -31,12 +119,13 @@ conf_options['ssh'] = {'Host': {'*': {'KexAlgorithms': 'curve25519-sha256@libssh
'PubkeyAuthentication': 'yes', 'PubkeyAuthentication': 'yes',
'HostKeyAlgorithms': 'ssh-ed25519-cert-v01@openssh.com,ssh-rsa-cert-v01@openssh.com,ssh-ed25519,ssh-rsa'}}} 'HostKeyAlgorithms': 'ssh-ed25519-cert-v01@openssh.com,ssh-rsa-cert-v01@openssh.com,ssh-ed25519,ssh-rsa'}}}
# Uncomment below if Github still needs diffie-hellman-group-exchange-sha1 sometimes. # Uncomment below if Github still needs diffie-hellman-group-exchange-sha1 sometimes.
# For what it's worth, it doesn't seem to.
#conf_options['ssh']['Host']['github.com'] = {'KexAlgorithms': 'curve25519-sha256@libssh.org,diffie-hellman-group-exchange-sha256,' + #conf_options['ssh']['Host']['github.com'] = {'KexAlgorithms': 'curve25519-sha256@libssh.org,diffie-hellman-group-exchange-sha256,' +
# 'diffie-hellman-group-exchange-sha1,diffie-hellman-group14-sha1'} # 'diffie-hellman-group-exchange-sha1,diffie-hellman-group14-sha1'}




def hostKeys(buildmoduli): def hostKeys(buildmoduli):
# Starting haveged should help lessen the time load, but not much. # Starting haveged should help lessen the time load a non-negligible amount, especially on virtual platforms.
if os.path.lexists('/usr/bin/haveged'): if os.path.lexists('/usr/bin/haveged'):
# We could use psutil here, but then that's a python dependency we don't need. # We could use psutil here, but then that's a python dependency we don't need.
# We could parse the /proc directory, but that's quite unnecessary. pgrep's installed by default on Arch. # We could parse the /proc directory, but that's quite unnecessary. pgrep's installed by default on Arch.

View File

@ -1,2 +1,4 @@
http://plato.asu.edu/MAT420/beginning_perl/3145_AppF.pdf http://plato.asu.edu/MAT420/beginning_perl/3145_AppF.pdf
http://www.profdavis.net/ASCII_table.pdf http://www.profdavis.net/ASCII_table.pdf
http://www.aboutmyip.com/AboutMyXApp/AsciiChart.jsp
http://www.robelle.com/smugbook/ascii.html

View File

@ -11,7 +11,7 @@ def waiter(seconds = 1):
max = len(anims) - 1 max = len(anims) - 1
global is_done global is_done
print('Beginning dhparam gen...') print('Beginning dhparam gen...')
# This is just an example commant that takes a looong time. # This is just an example command that takes a looong time.
c = subprocess.Popen(['openssl', 'dhparam', '-out', '/tmp/dhpem', '4096'], c = subprocess.Popen(['openssl', 'dhparam', '-out', '/tmp/dhpem', '4096'],
#c = subprocess.Popen(['openssl', 'genrsa', '-out', '/tmp/dhpem', '4096'], #c = subprocess.Popen(['openssl', 'genrsa', '-out', '/tmp/dhpem', '4096'],
stdout = subprocess.PIPE, stdout = subprocess.PIPE,

View File

@ -26,7 +26,7 @@ ref: https://www.python.org/dev/peps/pep-0008/#imports


############################################################################### ###############################################################################


To programmatically install modules via pip if they aren't installed: To programmatically install modules via pip if they aren't installed (BROKEN IN RECENT PIP VERSIONS; pip._internal.main() i think):


____ ____
import importlib import importlib
@ -109,3 +109,12 @@ log = logger.log(name = 'project.name')


# TODO # # TODO #
https://stackoverflow.com/questions/10265193/python-can-a-class-act-like-a-module https://stackoverflow.com/questions/10265193/python-can-a-class-act-like-a-module


###############################################################################

To issue an equivalent of "reset" command in linux, assuming console is ANSI-compat,

print('\x1bc', end = '')

###############################################################################

View File

@ -1,4 +1,4 @@
#!/usr/bin/env python3.6 #!/usr/bin/env python3


# stdlib # stdlib
import argparse import argparse

View File

@ -45,6 +45,12 @@ class Backup(object):
if self.args['oper'] == 'backup': if self.args['oper'] == 'backup':
for d in (self.args['mysqldir'], self.args['stagedir']): for d in (self.args['mysqldir'], self.args['stagedir']):
os.makedirs(d, exist_ok = True, mode = 0o700) os.makedirs(d, exist_ok = True, mode = 0o700)
if self.args['oper'] == 'restore':
self.args['target_dir'] = os.path.abspath(os.path.expanduser(
self.args['target_dir']))
os.makedirs(os.path.dirname(self.args['oper']),
exist_ok = True,
mode = 0o700)
### LOGGING ### ### LOGGING ###
# Thanks to: # Thanks to:
# https://web.archive.org/web/20170726052946/http://www.lexev.org/en/2013/python-logging-every-day/ # https://web.archive.org/web/20170726052946/http://www.lexev.org/en/2013/python-logging-every-day/
@ -255,7 +261,8 @@ class Backup(object):
self.cfg['config']['host'], self.cfg['config']['host'],
r, r,
self.args['archive'])) self.args['archive']))
# TODO: support specific path of extract? _cmd.append(os.path.abspath(self.args['target_dir']))
# TODO: support specific path inside archive?
# if so, append path(s) here. # if so, append path(s) here.
_env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password'] _env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password']
self.logger.debug('VARS: {0}'.format(vars())) self.logger.debug('VARS: {0}'.format(vars()))

View File

@ -159,6 +159,8 @@ class MtreeXML(object):
return(out) return(out)
def _unset_parse(unsetline): def _unset_parse(unsetline):
out = {} out = {}
if unsetline[1] == 'all':
return(copy.deepcopy(self._tplitem))
for i in unsetline: for i in unsetline:
out[i] = self._tplitem[i] out[i] = self._tplitem[i]
return(out) return(out)

View File

@ -31,21 +31,15 @@ class ConfStripper(object):
if not self.comments: if not self.comments:
if len(self.comment_syms) == 1: if len(self.comment_syms) == 1:
if self.inline: if self.inline:
self.regexes.append(re.compile( self.regexes.append(re.compile('^([^{0}]*){0}.*'.format(self.comment_syms[0])))
'^(.*){0}.*'.format(
self.comment_syms[0])))
else: else:
self.regexes.append(re.compile( self.regexes.append(re.compile('^(\s*){0}.*'.format(self.comment_syms[0])))
'^(\s*){0}.*'.format(
self.comment_syms[0])))
else: else:
syms = '|'.join(self.comment_syms) syms = '|'.join(self.comment_syms)
if self.inline: if self.inline:
self.regexes.append(re.compile( self.regexes.append(re.compile('^(.*)({0}).*'.format(syms)))
'^(.*)({0}).*'.format(syms)))
else: else:
self.regexes.append(re.compile( self.regexes.append(re.compile( '^(\s*)({0}).*'.format(syms)))
'^(\s*)({0}).*'.format(syms)))
return() return()


def parse(self, path): def parse(self, path):
@ -73,7 +67,7 @@ class ConfStripper(object):
return(None) return(None)
try: try:
with open(path, 'r') as f: with open(path, 'r') as f:
conf = [i.strip() for i in f.readlines()] conf = f.readlines()
except UnicodeDecodeError: # It's a binary file. Oops. except UnicodeDecodeError: # It's a binary file. Oops.
if self.cli: if self.cli:
print('{0}: Binary file? (is not UTF-8/ASCII)'.format(path)) print('{0}: Binary file? (is not UTF-8/ASCII)'.format(path))
@ -89,8 +83,13 @@ class ConfStripper(object):
# Okay, so now we can actually parse. # Okay, so now we can actually parse.
# Comments first. # Comments first.
for idx, line in enumerate(conf): for idx, line in enumerate(conf):
if line.strip() == '':
continue
for r in self.regexes: for r in self.regexes:
conf[idx] = r.sub('\g<1>', conf[idx]) conf[idx] = r.sub('\g<1>', conf[idx])
if conf[idx].strip() == '': # The line was "deleted".
conf[idx] = None
conf = [i for i in conf if i is not None]
# Then leading spaces... # Then leading spaces...
if not self.leading: if not self.leading:
for idx, line in enumerate(conf): for idx, line in enumerate(conf):
@ -101,7 +100,7 @@ class ConfStripper(object):
conf[idx] = conf[idx].rstrip() conf[idx] = conf[idx].rstrip()
# Lastly, if set, remove blank lines. # Lastly, if set, remove blank lines.
if not self.whitespace: if not self.whitespace:
conf = [i for i in conf if i != ''] conf = [i for i in conf if i.strip() != '']
return(conf) return(conf)


def recurse(self, path): def recurse(self, path):
@ -139,8 +138,7 @@ class ConfStripper(object):
f.write(new_content) f.write(new_content)
except PermissionError: except PermissionError:
if self.cli: if self.cli:
print('{0}: Cannot write (insufficient permission)'.format( print('{0}: Cannot write (insufficient permission)'.format(path))
path))
return() return()
return() return()


@ -160,9 +158,7 @@ class ConfStripper(object):
return(realpaths) return(realpaths)


def parseArgs(): def parseArgs():
args = argparse.ArgumentParser(description = ('Remove extraneous ' + args = argparse.ArgumentParser(description = ('Remove extraneous formatting/comments from files'))
'formatting/comments from ' +
'files'))
args.add_argument('-c', '--keep-comments', args.add_argument('-c', '--keep-comments',
dest = 'comments', dest = 'comments',
action = 'store_true', action = 'store_true',
@ -172,15 +168,13 @@ def parseArgs():
dest = 'comment_syms', dest = 'comment_syms',
action = 'append', action = 'append',
default = [], default = [],
help = ('The character(s) to be treated as comments. ' + help = ('The character(s) to be treated as comments. '
'Can be specified multiple times (one symbol ' + 'Can be specified multiple times (one symbol per flag, please, unless a specific '
'per flag, please, unless a specific sequence ' + 'sequence denotes a comment). Default is just #'))
'denotes a comment). Default is just #'))
args.add_argument('-i', '--no-inline', args.add_argument('-i', '--no-inline',
dest = 'inline', dest = 'inline',
action = 'store_false', action = 'store_false',
help = ('If specified, do NOT parse the files as ' + help = ('If specified, do NOT parse the files as having inline comments (the default is to '
'having inline comments (the default is to ' +
'look for inline comments)')) 'look for inline comments)'))
args.add_argument('-s', '--keep-whitespace', args.add_argument('-s', '--keep-whitespace',
dest = 'whitespace', dest = 'whitespace',
@ -189,17 +183,15 @@ def parseArgs():
args.add_argument('-t', '--keep-trailing', args.add_argument('-t', '--keep-trailing',
dest = 'trailing', dest = 'trailing',
action = 'store_true', action = 'store_true',
help = ('If specified, retain trailing whitespace on ' + help = ('If specified, retain trailing whitespace on lines'))
'lines'))
args.add_argument('-l', '--no-leading-whitespace', args.add_argument('-l', '--no-leading-whitespace',
dest = 'leading', dest = 'leading',
action = 'store_false', action = 'store_false',
help = ('If specified, REMOVE leading whitespace')) help = ('If specified, REMOVE leading whitespace'))
args.add_argument('-d', '--dry-run', args.add_argument('-w', '--write',
dest = 'dry_run', dest = 'dry_run',
action = 'store_true', action = 'store_false',
help = ('If specified, don\'t actually overwrite the ' + help = ('If specified, overwrite the file(s) instead of just printing to stdout'))
'file(s) - just print to stdout instead'))
args.add_argument('-S', '--no-symlinks', args.add_argument('-S', '--no-symlinks',
dest = 'symlinks', dest = 'symlinks',
action = 'store_false', action = 'store_false',
@ -207,10 +199,8 @@ def parseArgs():
args.add_argument('paths', args.add_argument('paths',
metavar = 'PATH/TO/DIR/OR/FILE', metavar = 'PATH/TO/DIR/OR/FILE',
nargs = '+', nargs = '+',
help = ('The path(s) to the file(s) to strip down. If ' + help = ('The path(s) to the file(s) to strip down. If a directory is given, files will '
'a directory is given, files will ' + 'recursively be printed (unless -w/--write is specified). Can be specified multiple '
'recursively be modified (unless -d/--dry-run ' +
'is specified). Can be specified multiple ' +
'times')) 'times'))
return(args) return(args)