adding logger lib and conf_minify
This commit is contained in:
brent s
parent
0836b93fee
commit
4640030373
109
lib/python/logger.py
Executable file
109
lib/python/logger.py
Executable file
@ -0,0 +1,109 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
# The logfile.
|
||||||
|
dflt_logfile = '/var/log/optools/optools.log'
|
||||||
|
|
||||||
|
# The default log level. Can be one of (in increasing levels of output):
|
||||||
|
# critical
|
||||||
|
# error
|
||||||
|
# warning
|
||||||
|
# info
|
||||||
|
# debug
|
||||||
|
# "debug" may log sensitive information! Do *not* use it unless ABSOLUTELY
|
||||||
|
# NECESSARY.
|
||||||
|
dflt_loglevel = 'warning'
|
||||||
|
|
||||||
|
# stdlib
|
||||||
|
import datetime
|
||||||
|
import logging
|
||||||
|
import logging.handlers
|
||||||
|
import os
|
||||||
|
|
||||||
|
class log(object):
|
||||||
|
def __init__(self, loglvl = dflt_loglevel, logfile = dflt_logfile,
|
||||||
|
logname = 'optools'):
|
||||||
|
# Loglevel mappings.
|
||||||
|
self.loglvls = {'critical': logging.CRITICAL,
|
||||||
|
'error': logging.ERROR,
|
||||||
|
'warning': logging.WARNING,
|
||||||
|
'info': logging.INFO,
|
||||||
|
'debug': logging.DEBUG}
|
||||||
|
self.loglvl = loglvl.lower()
|
||||||
|
if self.loglvl not in self.loglvls:
|
||||||
|
raise ValueError(('{0} is not one of: ' +
|
||||||
|
'{1}').format(loglvl,
|
||||||
|
', '.join(self.loglvls.keys())))
|
||||||
|
self.Logger = logging.getLogger(logname)
|
||||||
|
self.logfile = os.path.abspath(os.path.expanduser(logfile))
|
||||||
|
try:
|
||||||
|
os.makedirs(os.path.dirname(self.logfile),
|
||||||
|
exist_ok = True,
|
||||||
|
mode = 0o700)
|
||||||
|
except Exception as e:
|
||||||
|
# Make this non-fatal since we also log to journal for systemd?
|
||||||
|
raise e
|
||||||
|
self.systemd()
|
||||||
|
self.journald()
|
||||||
|
self.Logger.setLevel(self.loglvls[self.loglvl])
|
||||||
|
self.log_handlers()
|
||||||
|
|
||||||
|
def systemd(self):
|
||||||
|
# Add journald support if we're on systemd.
|
||||||
|
# We probably are since we're most likely on Arch, but we don't want to
|
||||||
|
# make assumptions.
|
||||||
|
self.systemd = False
|
||||||
|
_sysd_chk = ['/run/systemd/system',
|
||||||
|
'/dev/.run/systemd',
|
||||||
|
'/dev/.systemd']
|
||||||
|
for _ in _sysd_chk:
|
||||||
|
if os.path.isdir(_):
|
||||||
|
self.systemd = True
|
||||||
|
break
|
||||||
|
return()
|
||||||
|
|
||||||
|
def journald(self):
|
||||||
|
if not self.systemd:
|
||||||
|
return()
|
||||||
|
try:
|
||||||
|
from systemd import journal
|
||||||
|
except ImportError:
|
||||||
|
try:
|
||||||
|
import pip
|
||||||
|
pip.main(['install', '--user', 'systemd'])
|
||||||
|
from systemd import journal
|
||||||
|
except Exception as e:
|
||||||
|
# Build failed. Missing gcc, disk too full, whatever.
|
||||||
|
self.systemd = False
|
||||||
|
return()
|
||||||
|
|
||||||
|
def log_handlers(self):
|
||||||
|
# Log formats
|
||||||
|
if self.systemd:
|
||||||
|
_jrnlfmt = logging.Formatter(fmt = ('{levelname}: {message} ' +
|
||||||
|
'({filename}:{lineno})'),
|
||||||
|
style = '{',
|
||||||
|
datefmt = '%Y-%m-%d %H:%M:%S')
|
||||||
|
_logfmt = logging.Formatter(fmt = ('{asctime}:{levelname}: {message} (' +
|
||||||
|
'{filename}:{lineno})'),
|
||||||
|
style = '{',
|
||||||
|
datefmt = '%Y-%m-%d %H:%M:%S')
|
||||||
|
# Add handlers
|
||||||
|
_dflthandler = logging.handlers.RotatingFileHandler(self.logfile,
|
||||||
|
encoding = 'utf8',
|
||||||
|
# 1GB
|
||||||
|
maxBytes = 1073741824,
|
||||||
|
backupCount = 5)
|
||||||
|
_dflthandler.setFormatter(_logfmt)
|
||||||
|
_dflthandler.setLevel(self.loglvls[self.loglvl])
|
||||||
|
if self.systemd:
|
||||||
|
from systemd import journal
|
||||||
|
try:
|
||||||
|
h = journal.JournaldLogHandler()
|
||||||
|
except AttributeError: # Uses the other version
|
||||||
|
h = journal.JournalHandler()
|
||||||
|
h.setFormatter(_jrnlfmt)
|
||||||
|
h.setLevel(self.loglvls[self.loglvl])
|
||||||
|
self.Logger.addHandler(h)
|
||||||
|
self.Logger.addHandler(_dflthandler)
|
||||||
|
self.Logger.info('Logging initialized')
|
||||||
|
return()
|
||||||
@ -82,5 +82,30 @@ class ClassName(object):
|
|||||||
for i in kwargs.keys():
|
for i in kwargs.keys():
|
||||||
setattr(self, i, kwargs[i])
|
setattr(self, i, kwargs[i])
|
||||||
----
|
----
|
||||||
|
|
||||||
###############################################################################
|
###############################################################################
|
||||||
|
|
||||||
|
To store stdout and stderr to different files in a subprocess call:
|
||||||
|
----
|
||||||
|
with open('/tmp/test.o', 'w') as out, open('/tmp/test.e', 'w') as err:
|
||||||
|
subprocess.run(['command'], stdout = out, stderr = err)
|
||||||
|
----
|
||||||
|
###############################################################################
|
||||||
|
|
||||||
|
To use optools logging lib (or other "shared" modules):
|
||||||
|
----
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import importlib
|
||||||
|
spec = importlib.util.spec_from_file_location(
|
||||||
|
'logger',
|
||||||
|
'/opt/dev/optools/lib/python/logger.py')
|
||||||
|
logger = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(logger)
|
||||||
|
log = logger.log(name = 'project.name')
|
||||||
|
----
|
||||||
|
|
||||||
|
###############################################################################
|
||||||
|
|
||||||
|
# TODO #
|
||||||
|
https://stackoverflow.com/questions/10265193/python-can-a-class-act-like-a-module
|
||||||
225
text/conf_minify.py
Executable file
225
text/conf_minify.py
Executable file
@ -0,0 +1,225 @@
|
|||||||
|
#!/usr/bin/env python3.6
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import stat
|
||||||
|
|
||||||
|
class ConfStripper(object):
|
||||||
|
def __init__(self, paths, comments = False, comment_syms = '#',
|
||||||
|
inline = True, whitespace = False, leading = True,
|
||||||
|
trailing = False, dry_run = False, symlinks = True):
|
||||||
|
if __name__ == '__main__':
|
||||||
|
# We're being run as a CLI utility, not an import.
|
||||||
|
self.cli = True
|
||||||
|
else:
|
||||||
|
self.cli = False
|
||||||
|
self.paths = self.paths_parser(paths)
|
||||||
|
self.comments = comments
|
||||||
|
self.comment_syms = comment_syms
|
||||||
|
self.inline = inline
|
||||||
|
self.whitespace = whitespace
|
||||||
|
self.leading = leading
|
||||||
|
self.trailing = trailing
|
||||||
|
self.dry_run = dry_run
|
||||||
|
self.symlinks = symlinks
|
||||||
|
self.prep()
|
||||||
|
|
||||||
|
def prep(self):
|
||||||
|
self.regexes = []
|
||||||
|
# In self.regexes, we group what we *keep* into group #1.
|
||||||
|
if not self.comments:
|
||||||
|
if len(self.comment_syms) == 1:
|
||||||
|
if self.inline:
|
||||||
|
self.regexes.append(re.compile(
|
||||||
|
'^(.*){0}.*'.format(
|
||||||
|
self.comment_syms[0])))
|
||||||
|
else:
|
||||||
|
self.regexes.append(re.compile(
|
||||||
|
'^(\s*){0}.*'.format(
|
||||||
|
self.comment_syms[0])))
|
||||||
|
else:
|
||||||
|
syms = '|'.join(self.comment_syms)
|
||||||
|
if self.inline:
|
||||||
|
self.regexes.append(re.compile(
|
||||||
|
'^(.*)({0}).*'.format(syms)))
|
||||||
|
else:
|
||||||
|
self.regexes.append(re.compile(
|
||||||
|
'^(\s*)({0}).*'.format(syms)))
|
||||||
|
return()
|
||||||
|
|
||||||
|
def parse(self, path):
|
||||||
|
if os.path.islink(path):
|
||||||
|
if self.symlinks:
|
||||||
|
# Check for a broken symlink
|
||||||
|
try:
|
||||||
|
os.stat(path)
|
||||||
|
except FileNotFoundError:
|
||||||
|
if self.cli:
|
||||||
|
print('{0}: Broken symlink'.format(path))
|
||||||
|
return(None)
|
||||||
|
else:
|
||||||
|
# We don't even WANT to follow symlinks.
|
||||||
|
if self.cli:
|
||||||
|
print('{0}: Symlink'.format(path))
|
||||||
|
return(None)
|
||||||
|
if stat.S_ISSOCK(os.stat(path).st_mode): # It's a socket
|
||||||
|
if self.cli:
|
||||||
|
print('{0}: Socket'.format(path))
|
||||||
|
return(None)
|
||||||
|
if stat.S_ISFIFO(os.stat(path).st_mode): # It's a named pipe
|
||||||
|
if self.cli:
|
||||||
|
print('{0}: Named pipe'.format(path))
|
||||||
|
return(None)
|
||||||
|
try:
|
||||||
|
with open(path, 'r') as f:
|
||||||
|
conf = [i.strip() for i in f.readlines()]
|
||||||
|
except UnicodeDecodeError: # It's a binary file. Oops.
|
||||||
|
if self.cli:
|
||||||
|
print('{0}: Binary file? (is not UTF-8/ASCII)'.format(path))
|
||||||
|
return(None)
|
||||||
|
except PermissionError:
|
||||||
|
if self.cli:
|
||||||
|
print('{0}: Insufficient permission'.format(path))
|
||||||
|
return(None)
|
||||||
|
except Exception as e:
|
||||||
|
if self.cli:
|
||||||
|
print('{0}: {1}'.format(path, e))
|
||||||
|
return(None)
|
||||||
|
# Okay, so now we can actually parse.
|
||||||
|
# Comments first.
|
||||||
|
for idx, line in enumerate(conf):
|
||||||
|
for r in self.regexes:
|
||||||
|
conf[idx] = r.sub('\g<1>', conf[idx])
|
||||||
|
# Then leading spaces...
|
||||||
|
if not self.leading:
|
||||||
|
for idx, line in enumerate(conf):
|
||||||
|
conf[idx] = conf[idx].lstrip()
|
||||||
|
# Then trailing spaces...
|
||||||
|
if not self.trailing:
|
||||||
|
for idx, line in enumerate(conf):
|
||||||
|
conf[idx] = conf[idx].rstrip()
|
||||||
|
# Lastly, if set, remove blank lines.
|
||||||
|
if not self.whitespace:
|
||||||
|
conf = [i for i in conf if i != '']
|
||||||
|
return(conf)
|
||||||
|
|
||||||
|
def recurse(self, path):
|
||||||
|
files = []
|
||||||
|
for r, d, f in os.walk(path):
|
||||||
|
for i in f:
|
||||||
|
files.append(os.path.join(r, i))
|
||||||
|
return(files)
|
||||||
|
|
||||||
|
def main(self):
|
||||||
|
# Handle the files first.
|
||||||
|
for p in self.paths['files']:
|
||||||
|
try:
|
||||||
|
new_content = '\n'.join(self.parse(p))
|
||||||
|
except TypeError: # Binary file, etc.
|
||||||
|
continue
|
||||||
|
self.writer(p, new_content)
|
||||||
|
# Then the directories...
|
||||||
|
for d in self.paths['dirs']:
|
||||||
|
for f in self.recurse(d):
|
||||||
|
try:
|
||||||
|
new_content = '\n'.join(self.parse(f))
|
||||||
|
except TypeError: # Binary file, etc.
|
||||||
|
continue
|
||||||
|
self.writer(f, new_content)
|
||||||
|
return()
|
||||||
|
|
||||||
|
def writer(self, path, new_content):
|
||||||
|
if self.dry_run:
|
||||||
|
print('\n== {0} =='.format(path))
|
||||||
|
print(new_content, end = '\n\n')
|
||||||
|
return()
|
||||||
|
try:
|
||||||
|
with open(path, 'w') as f:
|
||||||
|
f.write(new_content)
|
||||||
|
except PermissionError:
|
||||||
|
if self.cli:
|
||||||
|
print('{0}: Cannot write (insufficient permission)'.format(
|
||||||
|
path))
|
||||||
|
return()
|
||||||
|
return()
|
||||||
|
|
||||||
|
def paths_parser(self, paths):
|
||||||
|
realpaths = {'files': [],
|
||||||
|
'dirs': []}
|
||||||
|
for p in paths:
|
||||||
|
path = os.path.abspath(os.path.expanduser(p))
|
||||||
|
if not os.path.exists(path):
|
||||||
|
if self.cli:
|
||||||
|
print('{0} does not exist; skipping...'.format(path))
|
||||||
|
continue
|
||||||
|
if os.path.isfile(path):
|
||||||
|
realpaths['files'].append(path)
|
||||||
|
elif os.path.isdir(path):
|
||||||
|
realpaths['dirs'].append(path)
|
||||||
|
return(realpaths)
|
||||||
|
|
||||||
|
def parseArgs():
|
||||||
|
args = argparse.ArgumentParser(description = ('Remove extraneous ' +
|
||||||
|
'formatting/comments from ' +
|
||||||
|
'files'))
|
||||||
|
args.add_argument('-c', '--keep-comments',
|
||||||
|
dest = 'comments',
|
||||||
|
action = 'store_true',
|
||||||
|
help = ('If specified, retain all comments'))
|
||||||
|
args.add_argument('-C', '--comment-symbol',
|
||||||
|
metavar = 'SYMBOL',
|
||||||
|
dest = 'comment_syms',
|
||||||
|
action = 'append',
|
||||||
|
default = [],
|
||||||
|
help = ('The character(s) to be treated as comments. ' +
|
||||||
|
'Can be specified multiple times (one symbol ' +
|
||||||
|
'per flag, please, unless a specific sequence ' +
|
||||||
|
'denotes a comment). Default is just #'))
|
||||||
|
args.add_argument('-i', '--no-inline',
|
||||||
|
dest = 'inline',
|
||||||
|
action = 'store_false',
|
||||||
|
help = ('If specified, do NOT parse the files as ' +
|
||||||
|
'having inline comments (the default is to ' +
|
||||||
|
'look for inline comments)'))
|
||||||
|
args.add_argument('-s', '--keep-whitespace',
|
||||||
|
dest = 'whitespace',
|
||||||
|
action = 'store_true',
|
||||||
|
help = ('If specified, retain whitespace'))
|
||||||
|
args.add_argument('-t', '--keep-trailing',
|
||||||
|
dest = 'trailing',
|
||||||
|
action = 'store_true',
|
||||||
|
help = ('If specified, retain trailing whitespace on ' +
|
||||||
|
'lines'))
|
||||||
|
args.add_argument('-l', '--no-leading-whitespace',
|
||||||
|
dest = 'leading',
|
||||||
|
action = 'store_false',
|
||||||
|
help = ('If specified, REMOVE leading whitespace'))
|
||||||
|
args.add_argument('-d', '--dry-run',
|
||||||
|
dest = 'dry_run',
|
||||||
|
action = 'store_true',
|
||||||
|
help = ('If specified, don\'t actually overwrite the ' +
|
||||||
|
'file(s) - just print to stdout instead'))
|
||||||
|
args.add_argument('-S', '--no-symlinks',
|
||||||
|
dest = 'symlinks',
|
||||||
|
action = 'store_false',
|
||||||
|
help = ('If specified, don\'t follow symlinks'))
|
||||||
|
args.add_argument('paths',
|
||||||
|
metavar = 'PATH/TO/DIR/OR/FILE',
|
||||||
|
nargs = '+',
|
||||||
|
help = ('The path(s) to the file(s) to strip down. If ' +
|
||||||
|
'a directory is given, files will ' +
|
||||||
|
'recursively be modified (unless -d/--dry-run ' +
|
||||||
|
'is specified). Can be specified multiple ' +
|
||||||
|
'times'))
|
||||||
|
return(args)
|
||||||
|
|
||||||
|
def main():
|
||||||
|
args = vars(parseArgs().parse_args())
|
||||||
|
if not args['comment_syms']:
|
||||||
|
args['comment_syms'].append('#')
|
||||||
|
c = ConfStripper(**args)
|
||||||
|
c.main()
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
Loading…
Reference in New Issue
Block a user