optools/net/irc/irssilogparse.py
2018-08-02 07:30:43 -04:00

364 lines
15 KiB
Python
Executable File

#!/usr/bin/env python3
## REFERENCE ##
# https://github.com/myano/jenni/wiki/IRC-String-Formatting
# https://www.mirc.com/colors.html
# https://en.wikipedia.org/wiki/ANSI_escape_code
# http://invisible-island.net/xterm/ctlseqs/ctlseqs.html
# https://github.com/shabble/irssi-docs/wiki/Formats#Colourising-Text-in-IRC-Messages
# https://askubuntu.com/a/528938
# Sorry for the copious comments, but the majority of the Irssi log stuff isn't
# really documented... anywhere. Just the color codes and \x03.
# And the log2ansi.pl script is... well, perl, so minus points for that, but
# the names are obtuse and perl's ugly af.
import argparse
import curses
import os
import pprint
import re
import sys
try:
import magic
has_magic = True
except ImportError:
print('Warning: you do not have the magic module installed (you can '
'install it via "pip3 install --user file-magic"). Automatic log '
'decompression will not work.')
has_magic = False
# This is a map to determine which module to use to decompress,
# if we should.
cmprsn_map = {'text/plain': None, # Plain ol' text
# Sometimes the formatting with color gives this
'application/octet-stream': None,
'application/x-bzip2': 'bz2', # Bzip2
'application/x-gzip': 'gzip', # Gzip
'application/x-xz': 'lzma'} # XZ
# irssi/mIRC to ANSI
# Split into 3 maps (truecolor will be populated later, currently uses 8-bit):
# - 8 (3/4 bit color values, 8 colors)
# - 256 (8-bit, 256 colors)
# - 'truecolor' (24-bit, ISO-8613-3, 16777216 colors)
# Keys are the mIRC color value. Reference the links above for reference on
# what the values map to.
# Values are:
# - 8: tuple for ANSI fg and bg values
# - 256: single value (same number is used for fg and bg)
# - 'truecolor': tuple of (R#, G#, B#) (same number is used for fg and bg)
# In addition, all three have the following:
# - ansi_wrap: the string formatter.
# fg: foreground color
# bg: background color (if present)
# They are concatted together in that order.
## https://en.wikipedia.org/wiki/ANSI_escape_code#3/4_bit
colormap = {8: {'0': ('97', '107'),
'1': ('30', '40'),
'2': ('34', '44'),
'3': ('32', '42'),
'4': ('91', '101'),
'5': ('31', '41'),
'6': ('35', '45'),
'7': ('33', '43'),
'8': ('93', '103'),
'9': ('92', '102'),
'10': ('36', '46'),
'11': ('96', '106'),
'12': ('94', '104'),
'13': ('95', '105'),
'14': ('90', '100'),
'15': ('37', '47'),
'ansi_wrap': {'fg': '\x1b[{0}',
'bg': ';{0}m'}},
## https://en.wikipedia.org/wiki/ANSI_escape_code#8-bit
256: {'0': '15',
'1': '0',
'2': '19',
'3': '34',
'4': '196',
'5': '52',
'6': '90',
'7': '208',
'8': '226',
'9': '82',
'10': '37',
'11': '51',
'12': '21',
'13': '199',
'14': '241',
'15': '252',
'ansi_wrap': {'fg': '\x1b[38;5;{0}m',
'bg': '\x1b[48;5;{0}m'}},
## https://en.wikipedia.org/wiki/ANSI_escape_code#24-bit
# (can just use mIRC's R,G,B)
'truecolor': {'0': ('255', '255', '255'),
'1': ('0', '0', '0'),
'2': ('0', '0', '127'),
'3': ('0', '147', '0'),
'4': ('255', '0', '0'),
'5': ('127', '0', '0'),
'6': ('156', '0', '156'),
'7': ('252', '127', '0'),
'8': ('255', '255', '0'),
'9': ('0', '252', '0'),
'10': ('0', '147', '147'),
'11': ('0', '255', '255'),
'12': ('0', '0', '252'),
'13': ('255', '0', '255'),
'14': ('127', '127', '127'),
'15': ('210', '210', '210'),
'ansi_wrap': {'fg': '\x1b[38;2;'
'{0[0]};{0[1]};{0[2]}m',
'bg': '\x1b[48;2;'
'{0[0]};{0[1]};{0[2]}m'}}}
# These are special "control characters" Irssi uses.
reset_char = '\x1b[0m'
# Used for Irssi-specific escapes/controls.
irssi_ctrl = {'a': '\x1b[5m', # Blink
'b': '\x1b[4m', # Underline
'c': '\x1b[1m', # Bold
'd': '\x1b[7m', # Reverse
'e': '\t', # Indent
'f': None, # "f" is an indent func, so no-op
'g': reset_char, # Reset
'h': None} # Monospace (no-op)
def get_palette():
# Return 8, 256, or 'truecolor'
colorterm = os.getenv('COLORTERM', None)
if colorterm in ('truecolor', '24bit'):
# TODO: 24-bit support (16777216 colors) instead of 8-bit.
# See note above.
#return('truecolor')
return(256)
else:
curses.initscr()
curses.start_color()
c = curses.COLORS
curses.endwin()
return(c)
def color_converter(data_in, palette_map):
# Only used if logParser().args['color'] = True
# Convert mIRC/Irssi color coding to ANSI color codes.
# A sub-function that generates the replacement characters.
fg = '39'
bg = '49'
# the value to reset the foreground text
def_fg = '\x1b[39m'
# the value to reset the background text
def_bg = '\x1b[49m'
_colors = colormap[palette_map]
def _repl(ch_in):
nonlocal bg
nonlocal fg
ch_out = ''
_ch = {'stripped': re.sub('^[\x00-\x7f]', '', ch_in.group()),
'ctrl': re.sub('^\x04([a-h]|[0-9]|;|>)/?.*$', '\g<1>',
ch_in.group())}
# We assign this separately as we use an existing dict entry.
# This is the "color code" (if it exists).
_ch['c'] = [re.sub('^0?([0-9])',
'\g<1>',
i.strip()) for i in _ch['stripped'].split(',', 1)]
pprint.pprint(_ch)
pprint.pprint(ch_in.group())
# Color-handling
if _ch['ctrl'].startswith('\x03'):
if len(_ch['c']) == 1:
fg_only = True
elif len(_ch['c']) == 2:
fg_only = False
else:
raise RuntimeError('Parsing error! "{0}"'.format(
ch_in.group()))
ch_out = _colors['ansi_wrap']['fg'].format(_colors[_ch['c'][0]])
if not fg_only:
ch_out += _colors['ansi_wrap']['bg'].format(
_colors[_ch['c'][1]])
# Control-character handling
elif _ch['ctrl'].startswith('\x04'):
if _ch['ctrl'] in irssi_ctrl:
ch_out = irssi_ctrl[_ch['ctrl']]
return(ch_out)
#color_ptrn = re.compile('\x03[0-9]{1,2}(,[0-9]{1,2})?')
catch = re.compile('(\x03[0-9](,[0-9]{1, 2})?|'
'\x04([a-h]|;|[0-9]/?))')
# These get replaced in the order they're defined here.
regex = {'nick': re.compile('\x04[89]/'),
'bold': re.compile('(\x02|\x04(;|[a-h]|>))/?'),
## Doublecheck this. what is the significance of \x04(.*) chars?
'control': re.compile('\x04(g|c|[389;]/?|e|>)?/?'),
'color_clear': re.compile('\x0f(g|c|[389;]/?|e)?/?')}
data = data_in.splitlines()
for idx, line in enumerate(data[:]):
# Get some preliminary replacements out of the way.
line = regex['nick'].sub(' ', line, 1)
#line = regex['bold'].sub(bold, line)
#line = regex['control'].sub(reset_char, line)
#line = regex['color_clear'].sub(def_fg + def_bg, line)
# This is like 90% of the magic, honestly.
line = catch.sub(_repl, line)
if not line.endswith(reset_char):
line += reset_char
data[idx] = line
return('\n'.join(data))
def plain_stripper(data_in):
# Strip to plaintext only.
data = data_in.splitlines()
ptrns = [re.compile('\x04(g|c|[389;]/?|e|>)/?'),
re.compile('((\x03)\d\d?,\d\d?|(\x03)\d\d?|[\x01-\x1F])')]
for idx, line in enumerate(data[:]):
# This cleans the nick field
l = re.sub('\x04[89]/', ' ', line, 1)
# And these clean the actual chat messages
for p in ptrns:
l = p.sub('', l)
data[idx] = l
return('\n'.join(data))
class irssiLogParser(object):
def __init__(self, args, data = None):
# We'll need these accessible across the entire class.
self.args = args
# If specified, self.data takes precedence over self.args['logfile']
# (if it was specified).
self.data = data
self.raw = data
self.has_html = False
self.decompress = None
if 'color' in self.args and self.args['color']:
if not self.args['html']:
# Ensure that we support color output.
curses.initscr()
self.args['color'] = curses.can_change_color()
curses.endwin()
if not self.args['color'] and not self.args['raw']:
raise RuntimeError('You have specified ANSI colorized '
'output but your terminal does not '
'support it. Use -fc/--force-color '
'to force.')
elif not self.args['color'] and self.args['raw']:
self.args['color'] = True # Force the output anyways.
if self.args['color']:
if not self.args['raw']:
self.colors = get_palette()
else:
self.colors = 8 # Best play it safe for maximum compatibility.
# The full, interpreted path.
if ('logfile' in self.args.keys() and
self.args['logfile'] is not None):
self.args['logfile'] = os.path.abspath(
os.path.expanduser(
self.args['logfile']))
if not self.data:
self.getlog()
else:
# Conform everything to bytes.
if not isinstance(self.data, bytes):
self.data = self.data.encode('utf-8')
self.decompressor()
self.parser()
def getlog(self):
# A filepath was specified
if self.args['logfile']:
if not os.path.isfile(self.args['logfile']):
raise FileNotFoundError('{0} does not exist'.format(
self.args['logfile']))
with open(self.args['logfile'], 'rb') as f:
self.data = f.read()
# Try to get it from stdin
else:
if not sys.stdin.isatty():
self.data = sys.stdin.buffer.read()
else:
raise ValueError('Either a path to a logfile must be '
'specified or you must pipe a log in from '
'stdin.')
self.raw = self.data
return()
def decompressor(self):
# TODO: use mime module as fallback?
# https://docs.python.org/3/library/mimetypes.html
# VERY less-than-ideal since it won't work without self.args['logfile']
# (and has iffy detection at best, since it relies on file extensions).
# Determine what decompressor to use, if we need to.
if has_magic:
_mime = magic.detect_from_content(self.data).mime_type
self.decompress = cmprsn_map[_mime]
if self.decompress:
import importlib
decmp = importlib.import_module(self.decompress)
self.raw = decmp.decompress(self.data)
else:
# Assume that it's text and that it isn't compressed.
# We'll get a UnicodeDecodeError exception if it isn't.
pass
try:
self.raw = self.data.decode('utf-8')
except UnicodeDecodeError:
pass
self.data = self.raw
return()
def parser(self):
if 'color' not in self.args or not self.args['color']:
self.data = plain_stripper(self.data)
else:
self.data = color_converter(self.data, self.colors)
# Just in case...
self.data += '\x1b[0m'
return()
def parseArgs():
args = argparse.ArgumentParser()
args.add_argument('-c', '--color',
dest = 'color',
action = 'store_true',
help = ('Print the log with converted colors (ANSI)'))
args.add_argument('-r', '--raw',
dest = 'raw',
action = 'store_true',
help = ('Use this switch if your terminal is detected '
'as not supporting color output but wish to '
'force it anyways. A string representation of '
'the ANSI output will be produced instead ('
'suitable for pasting elsewhere). Only used if '
'-c/--color is enabled (ignored with '
'-H/--html)'))
args.add_argument('-H', '--html',
dest = 'html',
action = 'store_true',
help = ('Render HTML output (requires ansi2html)'))
args.add_argument(dest = 'logfile',
default = None,
nargs = '?',
metavar = 'path/to/logfile',
help = ('The path to the log file. It can be uncompressed ' +
'or compressed with XZ/LZMA, Gzip, or Bzip2. '
'If not specified, read from stdin'))
return(args)
if __name__ == '__main__':
args = vars(parseArgs().parse_args())
l = irssiLogParser(args)
import shutil
cols = shutil.get_terminal_size().columns
#pprint.pprint(l.args, width = cols)
pprint.pprint(l.raw, width = cols)
with open('/tmp/freenode.formatted', 'r') as f:
print(f.read())
#pprint.pprint(l.data, width = cols)
#pprint.pprint(repr(l.data).split('\\n'))
print(l.data)
with open('/tmp/log.raw', 'w') as f:
for line in repr(l.data).split('\\n'):
f.write(line + '\n')
# l.parseLog()
# print(l.data.decode('utf-8'))