adding logger lib and conf_minify
This commit is contained in:
brent s
parent
0836b93fee
commit
4640030373
109
lib/python/logger.py
Executable file
109
lib/python/logger.py
Executable file
@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# The logfile.
|
||||
dflt_logfile = '/var/log/optools/optools.log'
|
||||
|
||||
# The default log level. Can be one of (in increasing levels of output):
|
||||
# critical
|
||||
# error
|
||||
# warning
|
||||
# info
|
||||
# debug
|
||||
# "debug" may log sensitive information! Do *not* use it unless ABSOLUTELY
|
||||
# NECESSARY.
|
||||
dflt_loglevel = 'warning'
|
||||
|
||||
# stdlib
|
||||
import datetime
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
|
||||
class log(object):
|
||||
def __init__(self, loglvl = dflt_loglevel, logfile = dflt_logfile,
|
||||
logname = 'optools'):
|
||||
# Loglevel mappings.
|
||||
self.loglvls = {'critical': logging.CRITICAL,
|
||||
'error': logging.ERROR,
|
||||
'warning': logging.WARNING,
|
||||
'info': logging.INFO,
|
||||
'debug': logging.DEBUG}
|
||||
self.loglvl = loglvl.lower()
|
||||
if self.loglvl not in self.loglvls:
|
||||
raise ValueError(('{0} is not one of: ' +
|
||||
'{1}').format(loglvl,
|
||||
', '.join(self.loglvls.keys())))
|
||||
self.Logger = logging.getLogger(logname)
|
||||
self.logfile = os.path.abspath(os.path.expanduser(logfile))
|
||||
try:
|
||||
os.makedirs(os.path.dirname(self.logfile),
|
||||
exist_ok = True,
|
||||
mode = 0o700)
|
||||
except Exception as e:
|
||||
# Make this non-fatal since we also log to journal for systemd?
|
||||
raise e
|
||||
self.systemd()
|
||||
self.journald()
|
||||
self.Logger.setLevel(self.loglvls[self.loglvl])
|
||||
self.log_handlers()
|
||||
|
||||
def systemd(self):
|
||||
# Add journald support if we're on systemd.
|
||||
# We probably are since we're most likely on Arch, but we don't want to
|
||||
# make assumptions.
|
||||
self.systemd = False
|
||||
_sysd_chk = ['/run/systemd/system',
|
||||
'/dev/.run/systemd',
|
||||
'/dev/.systemd']
|
||||
for _ in _sysd_chk:
|
||||
if os.path.isdir(_):
|
||||
self.systemd = True
|
||||
break
|
||||
return()
|
||||
|
||||
def journald(self):
|
||||
if not self.systemd:
|
||||
return()
|
||||
try:
|
||||
from systemd import journal
|
||||
except ImportError:
|
||||
try:
|
||||
import pip
|
||||
pip.main(['install', '--user', 'systemd'])
|
||||
from systemd import journal
|
||||
except Exception as e:
|
||||
# Build failed. Missing gcc, disk too full, whatever.
|
||||
self.systemd = False
|
||||
return()
|
||||
|
||||
def log_handlers(self):
|
||||
# Log formats
|
||||
if self.systemd:
|
||||
_jrnlfmt = logging.Formatter(fmt = ('{levelname}: {message} ' +
|
||||
'({filename}:{lineno})'),
|
||||
style = '{',
|
||||
datefmt = '%Y-%m-%d %H:%M:%S')
|
||||
_logfmt = logging.Formatter(fmt = ('{asctime}:{levelname}: {message} (' +
|
||||
'{filename}:{lineno})'),
|
||||
style = '{',
|
||||
datefmt = '%Y-%m-%d %H:%M:%S')
|
||||
# Add handlers
|
||||
_dflthandler = logging.handlers.RotatingFileHandler(self.logfile,
|
||||
encoding = 'utf8',
|
||||
# 1GB
|
||||
maxBytes = 1073741824,
|
||||
backupCount = 5)
|
||||
_dflthandler.setFormatter(_logfmt)
|
||||
_dflthandler.setLevel(self.loglvls[self.loglvl])
|
||||
if self.systemd:
|
||||
from systemd import journal
|
||||
try:
|
||||
h = journal.JournaldLogHandler()
|
||||
except AttributeError: # Uses the other version
|
||||
h = journal.JournalHandler()
|
||||
h.setFormatter(_jrnlfmt)
|
||||
h.setLevel(self.loglvls[self.loglvl])
|
||||
self.Logger.addHandler(h)
|
||||
self.Logger.addHandler(_dflthandler)
|
||||
self.Logger.info('Logging initialized')
|
||||
return()
|
||||
@ -82,5 +82,30 @@ class ClassName(object):
|
||||
for i in kwargs.keys():
|
||||
setattr(self, i, kwargs[i])
|
||||
----
|
||||
|
||||
###############################################################################
|
||||
|
||||
To store stdout and stderr to different files in a subprocess call:
|
||||
----
|
||||
with open('/tmp/test.o', 'w') as out, open('/tmp/test.e', 'w') as err:
|
||||
subprocess.run(['command'], stdout = out, stderr = err)
|
||||
----
|
||||
###############################################################################
|
||||
|
||||
To use optools logging lib (or other "shared" modules):
|
||||
----
|
||||
import os
|
||||
import re
|
||||
import importlib
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
'logger',
|
||||
'/opt/dev/optools/lib/python/logger.py')
|
||||
logger = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(logger)
|
||||
log = logger.log(name = 'project.name')
|
||||
----
|
||||
|
||||
###############################################################################
|
||||
|
||||
# TODO #
|
||||
https://stackoverflow.com/questions/10265193/python-can-a-class-act-like-a-module
|
||||
225
text/conf_minify.py
Executable file
225
text/conf_minify.py
Executable file
@ -0,0 +1,225 @@
|
||||
#!/usr/bin/env python3.6
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
|
||||
class ConfStripper(object):
|
||||
def __init__(self, paths, comments = False, comment_syms = '#',
|
||||
inline = True, whitespace = False, leading = True,
|
||||
trailing = False, dry_run = False, symlinks = True):
|
||||
if __name__ == '__main__':
|
||||
# We're being run as a CLI utility, not an import.
|
||||
self.cli = True
|
||||
else:
|
||||
self.cli = False
|
||||
self.paths = self.paths_parser(paths)
|
||||
self.comments = comments
|
||||
self.comment_syms = comment_syms
|
||||
self.inline = inline
|
||||
self.whitespace = whitespace
|
||||
self.leading = leading
|
||||
self.trailing = trailing
|
||||
self.dry_run = dry_run
|
||||
self.symlinks = symlinks
|
||||
self.prep()
|
||||
|
||||
def prep(self):
|
||||
self.regexes = []
|
||||
# In self.regexes, we group what we *keep* into group #1.
|
||||
if not self.comments:
|
||||
if len(self.comment_syms) == 1:
|
||||
if self.inline:
|
||||
self.regexes.append(re.compile(
|
||||
'^(.*){0}.*'.format(
|
||||
self.comment_syms[0])))
|
||||
else:
|
||||
self.regexes.append(re.compile(
|
||||
'^(\s*){0}.*'.format(
|
||||
self.comment_syms[0])))
|
||||
else:
|
||||
syms = '|'.join(self.comment_syms)
|
||||
if self.inline:
|
||||
self.regexes.append(re.compile(
|
||||
'^(.*)({0}).*'.format(syms)))
|
||||
else:
|
||||
self.regexes.append(re.compile(
|
||||
'^(\s*)({0}).*'.format(syms)))
|
||||
return()
|
||||
|
||||
def parse(self, path):
|
||||
if os.path.islink(path):
|
||||
if self.symlinks:
|
||||
# Check for a broken symlink
|
||||
try:
|
||||
os.stat(path)
|
||||
except FileNotFoundError:
|
||||
if self.cli:
|
||||
print('{0}: Broken symlink'.format(path))
|
||||
return(None)
|
||||
else:
|
||||
# We don't even WANT to follow symlinks.
|
||||
if self.cli:
|
||||
print('{0}: Symlink'.format(path))
|
||||
return(None)
|
||||
if stat.S_ISSOCK(os.stat(path).st_mode): # It's a socket
|
||||
if self.cli:
|
||||
print('{0}: Socket'.format(path))
|
||||
return(None)
|
||||
if stat.S_ISFIFO(os.stat(path).st_mode): # It's a named pipe
|
||||
if self.cli:
|
||||
print('{0}: Named pipe'.format(path))
|
||||
return(None)
|
||||
try:
|
||||
with open(path, 'r') as f:
|
||||
conf = [i.strip() for i in f.readlines()]
|
||||
except UnicodeDecodeError: # It's a binary file. Oops.
|
||||
if self.cli:
|
||||
print('{0}: Binary file? (is not UTF-8/ASCII)'.format(path))
|
||||
return(None)
|
||||
except PermissionError:
|
||||
if self.cli:
|
||||
print('{0}: Insufficient permission'.format(path))
|
||||
return(None)
|
||||
except Exception as e:
|
||||
if self.cli:
|
||||
print('{0}: {1}'.format(path, e))
|
||||
return(None)
|
||||
# Okay, so now we can actually parse.
|
||||
# Comments first.
|
||||
for idx, line in enumerate(conf):
|
||||
for r in self.regexes:
|
||||
conf[idx] = r.sub('\g<1>', conf[idx])
|
||||
# Then leading spaces...
|
||||
if not self.leading:
|
||||
for idx, line in enumerate(conf):
|
||||
conf[idx] = conf[idx].lstrip()
|
||||
# Then trailing spaces...
|
||||
if not self.trailing:
|
||||
for idx, line in enumerate(conf):
|
||||
conf[idx] = conf[idx].rstrip()
|
||||
# Lastly, if set, remove blank lines.
|
||||
if not self.whitespace:
|
||||
conf = [i for i in conf if i != '']
|
||||
return(conf)
|
||||
|
||||
def recurse(self, path):
|
||||
files = []
|
||||
for r, d, f in os.walk(path):
|
||||
for i in f:
|
||||
files.append(os.path.join(r, i))
|
||||
return(files)
|
||||
|
||||
def main(self):
|
||||
# Handle the files first.
|
||||
for p in self.paths['files']:
|
||||
try:
|
||||
new_content = '\n'.join(self.parse(p))
|
||||
except TypeError: # Binary file, etc.
|
||||
continue
|
||||
self.writer(p, new_content)
|
||||
# Then the directories...
|
||||
for d in self.paths['dirs']:
|
||||
for f in self.recurse(d):
|
||||
try:
|
||||
new_content = '\n'.join(self.parse(f))
|
||||
except TypeError: # Binary file, etc.
|
||||
continue
|
||||
self.writer(f, new_content)
|
||||
return()
|
||||
|
||||
def writer(self, path, new_content):
|
||||
if self.dry_run:
|
||||
print('\n== {0} =='.format(path))
|
||||
print(new_content, end = '\n\n')
|
||||
return()
|
||||
try:
|
||||
with open(path, 'w') as f:
|
||||
f.write(new_content)
|
||||
except PermissionError:
|
||||
if self.cli:
|
||||
print('{0}: Cannot write (insufficient permission)'.format(
|
||||
path))
|
||||
return()
|
||||
return()
|
||||
|
||||
def paths_parser(self, paths):
|
||||
realpaths = {'files': [],
|
||||
'dirs': []}
|
||||
for p in paths:
|
||||
path = os.path.abspath(os.path.expanduser(p))
|
||||
if not os.path.exists(path):
|
||||
if self.cli:
|
||||
print('{0} does not exist; skipping...'.format(path))
|
||||
continue
|
||||
if os.path.isfile(path):
|
||||
realpaths['files'].append(path)
|
||||
elif os.path.isdir(path):
|
||||
realpaths['dirs'].append(path)
|
||||
return(realpaths)
|
||||
|
||||
def parseArgs():
|
||||
args = argparse.ArgumentParser(description = ('Remove extraneous ' +
|
||||
'formatting/comments from ' +
|
||||
'files'))
|
||||
args.add_argument('-c', '--keep-comments',
|
||||
dest = 'comments',
|
||||
action = 'store_true',
|
||||
help = ('If specified, retain all comments'))
|
||||
args.add_argument('-C', '--comment-symbol',
|
||||
metavar = 'SYMBOL',
|
||||
dest = 'comment_syms',
|
||||
action = 'append',
|
||||
default = [],
|
||||
help = ('The character(s) to be treated as comments. ' +
|
||||
'Can be specified multiple times (one symbol ' +
|
||||
'per flag, please, unless a specific sequence ' +
|
||||
'denotes a comment). Default is just #'))
|
||||
args.add_argument('-i', '--no-inline',
|
||||
dest = 'inline',
|
||||
action = 'store_false',
|
||||
help = ('If specified, do NOT parse the files as ' +
|
||||
'having inline comments (the default is to ' +
|
||||
'look for inline comments)'))
|
||||
args.add_argument('-s', '--keep-whitespace',
|
||||
dest = 'whitespace',
|
||||
action = 'store_true',
|
||||
help = ('If specified, retain whitespace'))
|
||||
args.add_argument('-t', '--keep-trailing',
|
||||
dest = 'trailing',
|
||||
action = 'store_true',
|
||||
help = ('If specified, retain trailing whitespace on ' +
|
||||
'lines'))
|
||||
args.add_argument('-l', '--no-leading-whitespace',
|
||||
dest = 'leading',
|
||||
action = 'store_false',
|
||||
help = ('If specified, REMOVE leading whitespace'))
|
||||
args.add_argument('-d', '--dry-run',
|
||||
dest = 'dry_run',
|
||||
action = 'store_true',
|
||||
help = ('If specified, don\'t actually overwrite the ' +
|
||||
'file(s) - just print to stdout instead'))
|
||||
args.add_argument('-S', '--no-symlinks',
|
||||
dest = 'symlinks',
|
||||
action = 'store_false',
|
||||
help = ('If specified, don\'t follow symlinks'))
|
||||
args.add_argument('paths',
|
||||
metavar = 'PATH/TO/DIR/OR/FILE',
|
||||
nargs = '+',
|
||||
help = ('The path(s) to the file(s) to strip down. If ' +
|
||||
'a directory is given, files will ' +
|
||||
'recursively be modified (unless -d/--dry-run ' +
|
||||
'is specified). Can be specified multiple ' +
|
||||
'times'))
|
||||
return(args)
|
||||
|
||||
def main():
|
||||
args = vars(parseArgs().parse_args())
|
||||
if not args['comment_syms']:
|
||||
args['comment_syms'].append('#')
|
||||
c = ConfStripper(**args)
|
||||
c.main()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
Reference in New Issue
Block a user