adding logger lib and conf_minify

This commit is contained in:
brent s 2018-04-28 07:59:30 -04:00
parent 0836b93fee
commit 4640030373
3 changed files with 359 additions and 0 deletions

109
lib/python/logger.py Executable file
View File

@ -0,0 +1,109 @@
#!/usr/bin/env python3

# The logfile.
dflt_logfile = '/var/log/optools/optools.log'

# The default log level. Can be one of (in increasing levels of output):
# critical
# error
# warning
# info
# debug
# "debug" may log sensitive information! Do *not* use it unless ABSOLUTELY
# NECESSARY.
dflt_loglevel = 'warning'

# stdlib
import datetime
import logging
import logging.handlers
import os

class log(object):
def __init__(self, loglvl = dflt_loglevel, logfile = dflt_logfile,
logname = 'optools'):
# Loglevel mappings.
self.loglvls = {'critical': logging.CRITICAL,
'error': logging.ERROR,
'warning': logging.WARNING,
'info': logging.INFO,
'debug': logging.DEBUG}
self.loglvl = loglvl.lower()
if self.loglvl not in self.loglvls:
raise ValueError(('{0} is not one of: ' +
'{1}').format(loglvl,
', '.join(self.loglvls.keys())))
self.Logger = logging.getLogger(logname)
self.logfile = os.path.abspath(os.path.expanduser(logfile))
try:
os.makedirs(os.path.dirname(self.logfile),
exist_ok = True,
mode = 0o700)
except Exception as e:
# Make this non-fatal since we also log to journal for systemd?
raise e
self.systemd()
self.journald()
self.Logger.setLevel(self.loglvls[self.loglvl])
self.log_handlers()

def systemd(self):
# Add journald support if we're on systemd.
# We probably are since we're most likely on Arch, but we don't want to
# make assumptions.
self.systemd = False
_sysd_chk = ['/run/systemd/system',
'/dev/.run/systemd',
'/dev/.systemd']
for _ in _sysd_chk:
if os.path.isdir(_):
self.systemd = True
break
return()

def journald(self):
if not self.systemd:
return()
try:
from systemd import journal
except ImportError:
try:
import pip
pip.main(['install', '--user', 'systemd'])
from systemd import journal
except Exception as e:
# Build failed. Missing gcc, disk too full, whatever.
self.systemd = False
return()

def log_handlers(self):
# Log formats
if self.systemd:
_jrnlfmt = logging.Formatter(fmt = ('{levelname}: {message} ' +
'({filename}:{lineno})'),
style = '{',
datefmt = '%Y-%m-%d %H:%M:%S')
_logfmt = logging.Formatter(fmt = ('{asctime}:{levelname}: {message} (' +
'{filename}:{lineno})'),
style = '{',
datefmt = '%Y-%m-%d %H:%M:%S')
# Add handlers
_dflthandler = logging.handlers.RotatingFileHandler(self.logfile,
encoding = 'utf8',
# 1GB
maxBytes = 1073741824,
backupCount = 5)
_dflthandler.setFormatter(_logfmt)
_dflthandler.setLevel(self.loglvls[self.loglvl])
if self.systemd:
from systemd import journal
try:
h = journal.JournaldLogHandler()
except AttributeError: # Uses the other version
h = journal.JournalHandler()
h.setFormatter(_jrnlfmt)
h.setLevel(self.loglvls[self.loglvl])
self.Logger.addHandler(h)
self.Logger.addHandler(_dflthandler)
self.Logger.info('Logging initialized')
return()

View File

@ -82,5 +82,30 @@ class ClassName(object):
for i in kwargs.keys(): for i in kwargs.keys():
setattr(self, i, kwargs[i]) setattr(self, i, kwargs[i])
---- ----

############################################################################### ###############################################################################


To store stdout and stderr to different files in a subprocess call:
----
with open('/tmp/test.o', 'w') as out, open('/tmp/test.e', 'w') as err:
subprocess.run(['command'], stdout = out, stderr = err)
----
###############################################################################

To use optools logging lib (or other "shared" modules):
----
import os
import re
import importlib
spec = importlib.util.spec_from_file_location(
'logger',
'/opt/dev/optools/lib/python/logger.py')
logger = importlib.util.module_from_spec(spec)
spec.loader.exec_module(logger)
log = logger.log(name = 'project.name')
----

###############################################################################

# TODO #
https://stackoverflow.com/questions/10265193/python-can-a-class-act-like-a-module

225
text/conf_minify.py Executable file
View File

@ -0,0 +1,225 @@
#!/usr/bin/env python3.6

import argparse
import os
import re
import stat

class ConfStripper(object):
def __init__(self, paths, comments = False, comment_syms = '#',
inline = True, whitespace = False, leading = True,
trailing = False, dry_run = False, symlinks = True):
if __name__ == '__main__':
# We're being run as a CLI utility, not an import.
self.cli = True
else:
self.cli = False
self.paths = self.paths_parser(paths)
self.comments = comments
self.comment_syms = comment_syms
self.inline = inline
self.whitespace = whitespace
self.leading = leading
self.trailing = trailing
self.dry_run = dry_run
self.symlinks = symlinks
self.prep()

def prep(self):
self.regexes = []
# In self.regexes, we group what we *keep* into group #1.
if not self.comments:
if len(self.comment_syms) == 1:
if self.inline:
self.regexes.append(re.compile(
'^(.*){0}.*'.format(
self.comment_syms[0])))
else:
self.regexes.append(re.compile(
'^(\s*){0}.*'.format(
self.comment_syms[0])))
else:
syms = '|'.join(self.comment_syms)
if self.inline:
self.regexes.append(re.compile(
'^(.*)({0}).*'.format(syms)))
else:
self.regexes.append(re.compile(
'^(\s*)({0}).*'.format(syms)))
return()

def parse(self, path):
if os.path.islink(path):
if self.symlinks:
# Check for a broken symlink
try:
os.stat(path)
except FileNotFoundError:
if self.cli:
print('{0}: Broken symlink'.format(path))
return(None)
else:
# We don't even WANT to follow symlinks.
if self.cli:
print('{0}: Symlink'.format(path))
return(None)
if stat.S_ISSOCK(os.stat(path).st_mode): # It's a socket
if self.cli:
print('{0}: Socket'.format(path))
return(None)
if stat.S_ISFIFO(os.stat(path).st_mode): # It's a named pipe
if self.cli:
print('{0}: Named pipe'.format(path))
return(None)
try:
with open(path, 'r') as f:
conf = [i.strip() for i in f.readlines()]
except UnicodeDecodeError: # It's a binary file. Oops.
if self.cli:
print('{0}: Binary file? (is not UTF-8/ASCII)'.format(path))
return(None)
except PermissionError:
if self.cli:
print('{0}: Insufficient permission'.format(path))
return(None)
except Exception as e:
if self.cli:
print('{0}: {1}'.format(path, e))
return(None)
# Okay, so now we can actually parse.
# Comments first.
for idx, line in enumerate(conf):
for r in self.regexes:
conf[idx] = r.sub('\g<1>', conf[idx])
# Then leading spaces...
if not self.leading:
for idx, line in enumerate(conf):
conf[idx] = conf[idx].lstrip()
# Then trailing spaces...
if not self.trailing:
for idx, line in enumerate(conf):
conf[idx] = conf[idx].rstrip()
# Lastly, if set, remove blank lines.
if not self.whitespace:
conf = [i for i in conf if i != '']
return(conf)

def recurse(self, path):
files = []
for r, d, f in os.walk(path):
for i in f:
files.append(os.path.join(r, i))
return(files)

def main(self):
# Handle the files first.
for p in self.paths['files']:
try:
new_content = '\n'.join(self.parse(p))
except TypeError: # Binary file, etc.
continue
self.writer(p, new_content)
# Then the directories...
for d in self.paths['dirs']:
for f in self.recurse(d):
try:
new_content = '\n'.join(self.parse(f))
except TypeError: # Binary file, etc.
continue
self.writer(f, new_content)
return()

def writer(self, path, new_content):
if self.dry_run:
print('\n== {0} =='.format(path))
print(new_content, end = '\n\n')
return()
try:
with open(path, 'w') as f:
f.write(new_content)
except PermissionError:
if self.cli:
print('{0}: Cannot write (insufficient permission)'.format(
path))
return()
return()

def paths_parser(self, paths):
realpaths = {'files': [],
'dirs': []}
for p in paths:
path = os.path.abspath(os.path.expanduser(p))
if not os.path.exists(path):
if self.cli:
print('{0} does not exist; skipping...'.format(path))
continue
if os.path.isfile(path):
realpaths['files'].append(path)
elif os.path.isdir(path):
realpaths['dirs'].append(path)
return(realpaths)

def parseArgs():
args = argparse.ArgumentParser(description = ('Remove extraneous ' +
'formatting/comments from ' +
'files'))
args.add_argument('-c', '--keep-comments',
dest = 'comments',
action = 'store_true',
help = ('If specified, retain all comments'))
args.add_argument('-C', '--comment-symbol',
metavar = 'SYMBOL',
dest = 'comment_syms',
action = 'append',
default = [],
help = ('The character(s) to be treated as comments. ' +
'Can be specified multiple times (one symbol ' +
'per flag, please, unless a specific sequence ' +
'denotes a comment). Default is just #'))
args.add_argument('-i', '--no-inline',
dest = 'inline',
action = 'store_false',
help = ('If specified, do NOT parse the files as ' +
'having inline comments (the default is to ' +
'look for inline comments)'))
args.add_argument('-s', '--keep-whitespace',
dest = 'whitespace',
action = 'store_true',
help = ('If specified, retain whitespace'))
args.add_argument('-t', '--keep-trailing',
dest = 'trailing',
action = 'store_true',
help = ('If specified, retain trailing whitespace on ' +
'lines'))
args.add_argument('-l', '--no-leading-whitespace',
dest = 'leading',
action = 'store_false',
help = ('If specified, REMOVE leading whitespace'))
args.add_argument('-d', '--dry-run',
dest = 'dry_run',
action = 'store_true',
help = ('If specified, don\'t actually overwrite the ' +
'file(s) - just print to stdout instead'))
args.add_argument('-S', '--no-symlinks',
dest = 'symlinks',
action = 'store_false',
help = ('If specified, don\'t follow symlinks'))
args.add_argument('paths',
metavar = 'PATH/TO/DIR/OR/FILE',
nargs = '+',
help = ('The path(s) to the file(s) to strip down. If ' +
'a directory is given, files will ' +
'recursively be modified (unless -d/--dry-run ' +
'is specified). Can be specified multiple ' +
'times'))
return(args)

def main():
args = vars(parseArgs().parse_args())
if not args['comment_syms']:
args['comment_syms'].append('#')
c = ConfStripper(**args)
c.main()

if __name__ == '__main__':
main()