r00t2/optools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Activity

rewriting backup script; config and plugins done, just need to parse it and execute

Browse Source
This commit is contained in:
brent s 2019-05-23 02:46:05 -04:00
parent 36061cccb5
commit 130746fa00
7 changed files with 1203 additions and 226 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

330
storage/backups/borg/backup.py
View File

@ -5,18 +5,27 @@
# we import it for various prep stuff anyways. # we import it for various prep stuff anyways.
# TODO: change loglevel of borg itself in subprocess to match the argparse? # TODO: change loglevel of borg itself in subprocess to match the argparse?
# --debug, --info (same as -v/--verbose), --warning, --error, --critical # --debug, --info (same as -v/--verbose), --warning, --error, --critical
# TODO: switch to XML-based cfg instead of JSON. add automatic converter.
# TODO: modify config to add repo to cfg for init? or add new operation, "add" # TODO: modify config to add repo to cfg for init? or add new operation, "add"


import argparse import argparse
import configparser import configparser
import datetime import datetime
import json import json
import getpass
import logging import logging
import logging.handlers import logging.handlers
import os import os
import re
# TODO: use borg module directly?
import subprocess import subprocess
import sys import sys
# TODO: virtual env?
try:
from lxml import etree
has_lxml = True
except ImportError:
import xml.etree.ElementTree as etree # https://docs.python.org/3/library/xml.etree.elementtree.html
has_lxml = False


try: try:
import pymysql # not stdlib; "python-pymysql" in Arch's AUR import pymysql # not stdlib; "python-pymysql" in Arch's AUR
@ -31,8 +40,7 @@ except ImportError:
has_systemd = False has_systemd = False


### LOG LEVEL MAPPINGS ### ### LOG LEVEL MAPPINGS ###
loglvls = { loglvls = {'critical': logging.CRITICAL,
'critical': logging.CRITICAL,
'error': logging.ERROR, 'error': logging.ERROR,
'warning': logging.WARNING, 'warning': logging.WARNING,
'info': logging.INFO, 'info': logging.INFO,
@ -129,7 +137,8 @@ class Backup(object):
if self.args['dryrun']: if self.args['dryrun']:
return () # no-op return () # no-op
if stdoutfh: if stdoutfh:
_cmd = subprocess.run(cmd, stdout = stdoutfh, _cmd = subprocess.run(cmd,
stdout = stdoutfh,
stderr = subprocess.PIPE) stderr = subprocess.PIPE)
else: else:
_cmd = subprocess.run(cmd, _cmd = subprocess.run(cmd,
@ -297,87 +306,13 @@ class Backup(object):
self.logger.info('END: restore') self.logger.info('END: restore')
return() return()


def miscBak(self, pkgr):
self.logger.info('BEGIN: miscBak()')
_cmd = None
for p in os.environ['PATH'].split(':'):
d = os.path.expanduser(p)
if os.path.isfile(os.path.join(d, pkgr)):
_pkgr = pkgr
self.logger.debug('Package tool found at {0}'.format(_pkgr))
else:
_pkgr = 'pacman'
self.logger.debug('Using {0} as package tool'.format(_pkgr))
with open(os.path.join(self.args['stagedir'], 'pkg.lst'), 'w') as f:
_cmd = [_pkgr,
'-Qet',
'--color',
'never']
self.cmdExec(_cmd, stdoutfh = f)
self.logger.info('END: miscBak()')
return()

def mysqlBak(self):
self.logger.info('BEGIN: mysqlBak()')
if not has_mysql:
self.logger.error(
'You need to install the PyMySQL module to back up MySQL databases. Skipping.')
return ()
# These are mysqldump options shared by ALL databases
_mysqlopts = ['--routines',
'--add-drop-database',
'--add-drop-table',
'--allow-keywords',
'--complete-insert',
'--create-options',
'--extended-insert']
_DBs = []
_mycnf = os.path.expanduser(os.path.join('~', '.my.cnf'))
if not os.path.isfile(_mycnf):
exit(
'{0}: ERROR: Cannot get credentials for MySQL (cannot find ~/.my.cnf)!')
_mycfg = configparser.ConfigParser()
_mycfg._interpolation = configparser.ExtendedInterpolation()
_mycfg.read(_mycnf)
_sqlcfg = {s: dict(_mycfg.items(s)) for s in _mycfg.sections()}
if 'host' not in _sqlcfg.keys():
_socketpath = '/var/run/mysqld/mysqld.sock' # correct for Arch, YMMV.
_mysql = pymysql.connect(unix_socket = _socketpath,
user = _sqlcfg['client']['user'],
passwd = _sqlcfg['client']['password'])
else:
_mysql = pymysql.connect(host = _sqlcfg['client']['host'],
user = _sqlcfg['client']['user'],
port = _sqlcfg['client']['port'],
passwd = _sqlcfg['client']['password'])
_cur = _mysql.cursor()
_cur.execute('SHOW DATABASES')
for row in _cur.fetchall():
_DBs.append(row[0])
self.logger.debug('Databases: {0}'.format(', '.join(_DBs)))
for db in _DBs:
_cmd = ['mysqldump',
'--result-file={0}.sql'.format(
os.path.join(self.args['mysqldir'], db))]
# These are database-specific options
if db in ('information_schema', 'performance_schema'):
_cmd.append('--skip-lock-tables')
elif db == 'mysql':
_cmd.append('--flush-privileges')
_cmd.extend(_mysqlopts)
_cmd.append(db)
self.cmdExec(_cmd)
self.logger.info('END: mysqlBak()')
return()

def listRepos(self): def listRepos(self):
print('\n\033[1mCurrently configured repositories are:\033[0m\n') print('\n\033[1mCurrently configured repositories are:\033[0m\n')
print('\t{0}\n'.format(', '.join(self.cfg['repos'].keys()))) print('\t{0}\n'.format(', '.join(self.cfg['repos'].keys())))
if self.args['verbose']: if self.args['verbose']:
print('\033[1mDETAILS:\033[0m\n') print('\033[1mDETAILS:\033[0m\n')
for r in self.args['repo']: for r in self.args['repo']:
print( print('\t\033[1m{0}:\033[0m\n\t\t\033[1mPath(s):\033[0m\t'.format(
'\t\033[1m{0}:\033[0m\n\t\t\033[1mPath(s):\033[0m\t'.format(
r.upper()), end = '') r.upper()), end = '')
for p in self.cfg['repos'][r]['paths']: for p in self.cfg['repos'][r]['paths']:
print(p, end = ' ') print(p, end = ' ')
@ -406,14 +341,11 @@ class Backup(object):
print() print()
else: # It's a listing inside an archive else: # It's a listing inside an archive
if self.args['verbose']: if self.args['verbose']:
_fields = ['REPO:', 'PERMS:', 'OWNERSHIP:', 'SIZE:', _fields = ['REPO:', 'PERMS:', 'OWNERSHIP:', 'SIZE:', 'TIMESTAMP:', 'PATH:']
'TIMESTAMP:', 'PATH:']
for r in _results.keys(): for r in _results.keys():
print('\033[1m{0}\t{1}\033[0m'.format(_fields[0], r)) print('\033[1m{0}\t{1}\033[0m'.format(_fields[0], r))
# https://docs.python.org/3/library/string.html#formatspec # https://docs.python.org/3/library/string.html#formatspec
print( print('{0[1]:<15}\t{0[2]:<15}\t{0[3]:<15}\t{0[4]:<24}\t{0[5]:<15}'.format(_fields))
'{0[1]:<15}\t{0[2]:<15}\t{0[3]:<15}\t{0[4]:<24}\t{0[5]:<15}'.format(
_fields))
for line in _results[r]: for line in _results[r]:
_fline = line.split() _fline = line.split()
_perms = _fline[0] _perms = _fline[0]
@ -421,9 +353,7 @@ class Backup(object):
_size = _fline[3] _size = _fline[3]
_time = ' '.join(_fline[4:7]) _time = ' '.join(_fline[4:7])
_path = ' '.join(_fline[7:]) _path = ' '.join(_fline[7:])
print( print('{0:<15}\t{1:<15}\t{2:<15}\t{3:<24}\t{4:<15}'.format(_perms,
'{0:<15}\t{1:<15}\t{2:<15}\t{3:<24}\t{4:<15}'.format(
_perms,
_ownership, _ownership,
_size, _size,
_time, _time,
@ -463,31 +393,26 @@ class Backup(object):
env = _env, env = _env,
stdout = subprocess.PIPE, stdout = subprocess.PIPE,
stderr = subprocess.PIPE) stderr = subprocess.PIPE)
_stdout = [i.strip() for i in _stdout = [i.strip() for i in _out.stdout.decode('utf-8').splitlines()]
_out.stdout.decode('utf-8').splitlines()]
_stderr = _out.stderr.decode('utf-8').strip() _stderr = _out.stderr.decode('utf-8').strip()
_returncode = _out.returncode _returncode = _out.returncode
output[r] = _stdout output[r] = _stdout
self.logger.debug('[{0}]: (RESULT) {1}'.format(r, self.logger.debug('[{0}]: (RESULT) {1}'.format(r,
'\n'.join( '\n'.join(_stdout)))
_stdout)))
if _returncode != 0: if _returncode != 0:
self.logger.error('[{0}]: STDERR: ({2}) ({1})'.format(r, self.logger.error('[{0}]: STDERR: ({2}) ({1})'.format(r,
_stderr, _stderr,
' '.join( ' '.join(_cmd)))
_cmd)))
if _stderr != '' and self.cron and _returncode != 0: if _stderr != '' and self.cron and _returncode != 0:
self.logger.warning( self.logger.warning(
'Command {0} failed: {1}'.format(' '.join(cmd), 'Command {0} failed: {1}'.format(' '.join(cmd), _err))
_err))
del(_env['BORG_PASSPHRASE']) del(_env['BORG_PASSPHRASE'])
if not self.args['archive']: if not self.args['archive']:
if self.args['numlimit'] > 0: if self.args['numlimit'] > 0:
if self.args['old']: if self.args['old']:
output[r] = output[r][:self.args['numlimit']] output[r] = output[r][:self.args['numlimit']]
else: else:
output[r] = list(reversed(output[r]))[ output[r] = list(reversed(output[r]))[:self.args['numlimit']]
:self.args['numlimit']]
if self.args['invert']: if self.args['invert']:
output[r] = reversed(output[r]) output[r] = reversed(output[r])
self.logger.debug('END: lister') self.logger.debug('END: lister')
@ -495,24 +420,19 @@ class Backup(object):




def printMoarHelp(): def printMoarHelp():
_helpstr = ( _helpstr = ('\n\tNOTE: Sorting only applies to listing archives, NOT the contents!\n\n'
'\n\tNOTE: Sorting only applies to listing archives, NOT the ' 'In order to efficiently display results, there are several options to handle it. '
'contents!\n\n' 'Namely, these are:\n\n\t\t'
'In order to efficiently display results, there are several options '
'to handle it. Namely, these are:\n\n\t\t'
'-s/--sort [direction]\n\t\t' '-s/--sort [direction]\n\t\t'
'-l/--limit [number]\n\t\t' '-l/--limit [number]\n\t\t'
'-x/--invert\n\n' '-x/--invert\n\n'
'For example, if you want to list the 5 most recently *taken* ' 'For example, if you want to list the 5 most recently *taken* snapshots, you would use:\n\n\t\t'
'snapshots, you would use:\n\n\t\t'
'-l 5\n\n' '-l 5\n\n'
'If you would want those SAME results SORTED in the reverse order ' 'If you would want those SAME results SORTED in the reverse order (i.e. the 5 most recently '
'(i.e. the 5 most recently taken snapshots sorted from newest to ' 'taken snapshots sorted from newest to oldest), then it would be: \n\n\t\t'
'oldest), then it would be: \n\n\t\t'
'-l 5 -x\n\n' '-l 5 -x\n\n'
'Lastly, if you wanted to list the 7 OLDEST TAKEN snapshots in ' 'Lastly, if you wanted to list the 7 OLDEST TAKEN snapshots in reverse order '
'reverse order (that is, sorted from newest to oldest), that\'d be: ' '(that is, sorted from newest to oldest), that\'d be:\n\n\t\t'
'\n\n\t\t'
'-o -l 7 -x\n') '-o -l 7 -x\n')
print(_helpstr) print(_helpstr)
exit(0) exit(0)
@ -534,13 +454,12 @@ def parseArgs():
os.path.join(os.path.expanduser('~'), os.path.join(os.path.expanduser('~'),
'.config', '.config',
'optools', 'optools',
'backup.json')) 'backup.xml'))
_defloglvl = 'info' _defloglvl = 'info'
###### ######
args = argparse.ArgumentParser(description = 'Backups manager', args = argparse.ArgumentParser(description = 'Backups manager',
epilog = ('TIP: this program has ' epilog = ('TIP: this program has context-specific help. '
'context-specific help. e.g. ' 'e.g. try "%(prog)s list --help"'))
'try "%(prog)s list --help"'))
args.add_argument('-c', '--config', args.add_argument('-c', '--config',
dest = 'cfgfile', dest = 'cfgfile',
default = _cfgfile, default = _cfgfile,
@ -552,58 +471,45 @@ def parseArgs():
default = _defloglvl, default = _defloglvl,
choices = list(loglvls.keys()), choices = list(loglvls.keys()),
help = ( help = (
'The level of logging to perform. ' 'The level of logging to perform. \033[1mWARNING:\033[0m \033[1mdebug\033[0m will '
'\033[1mWARNING:\033[0m \033[1mdebug\033[0m will '
'log VERY sensitive information such as passwords! ' 'log VERY sensitive information such as passwords! '
'Default: \033[1m{0}\033[0m'.format(_defloglvl))) 'Default: \033[1m{0}\033[0m'.format(_defloglvl)))
args.add_argument('-Ld', '--log-to-disk', args.add_argument('-Ld', '--log-to-disk',
dest = 'disklog', dest = 'disklog',
action = 'store_true', action = 'store_true',
help = ( help = (
'If specified, log to a specific file ' 'If specified, log to a specific file (-Lf/--logfile) instead of the system logger.'))
'(-Lf/--logfile) instead of the system logger.'))
args.add_argument('-Lf', '--logfile', args.add_argument('-Lf', '--logfile',
dest = 'logfile', dest = 'logfile',
default = _logfile, default = _logfile,
help = ( help = (
'The path to the logfile, only used if ' 'The path to the logfile, only used if -Ld/--log-to-disk is specified. '
'-Ld/--log-to-disk is specified. ' 'Default: \033[1m{0}\033[0m (dynamic)').format(_logfile))
'Default: \033[1m{0}\033[0m (dynamic)').format(
_logfile))
args.add_argument('-v', '--verbose', args.add_argument('-v', '--verbose',
dest = 'verbose', dest = 'verbose',
action = 'store_true', action = 'store_true',
help = ( help = ('If specified, log messages will be printed to STDERR in addition to the other '
'If specified, log messages will be printed to ' 'configured log system(s), and verbosity for printing functions is increased. '
'STDERR in addition to the other configured log ' '\033[1mWARNING:\033[0m This may display VERY sensitive information such as passwords!'))
'system(s), and verbosity for printing '
'functions is increased. '
'\033[1mWARNING:\033[0m This may display VERY '
'sensitive information such as passwords!'))
### ARGS FOR ALL OPERATIONS ### ### ARGS FOR ALL OPERATIONS ###
commonargs = argparse.ArgumentParser(add_help = False) commonargs = argparse.ArgumentParser(add_help = False)
commonargs.add_argument('-r', '--repo', commonargs.add_argument('-r', '--repo',
dest = 'repo', dest = 'repo',
default = 'all', default = 'all',
help = ( help = ('The repository to perform the operation for. '
'The repository to perform the operation for. ' 'The default is \033[1mall\033[0m, a special value that specifies all known '
'The default is \033[1mall\033[0m, a special ' 'repositories. Can also accept a comma-separated list.'))
'value that specifies all known repositories. '
'Can also accept a comma-separated list.'))
fileargs = argparse.ArgumentParser(add_help = False) fileargs = argparse.ArgumentParser(add_help = False)
fileargs.add_argument('-a', '--archive', fileargs.add_argument('-a', '--archive',
default = _date, default = _date,
dest = 'archive', dest = 'archive',
help = ( help = ('The name of the archive/snapshot. '
'The name of the archive/snapshot. ' 'Default: \033[1m{0}\033[0m (dynamic)').format(_date))
'Default: \033[1m{0}\033[0m (dynamic)').format(
_date))
remoteargs = argparse.ArgumentParser(add_help = False) remoteargs = argparse.ArgumentParser(add_help = False)
remoteargs.add_argument('-d', '--dry-run', remoteargs.add_argument('-d', '--dry-run',
dest = 'dryrun', dest = 'dryrun',
action = 'store_true', action = 'store_true',
help = ('Act as if we are performing tasks, but ' help = ('Act as if we are performing tasks, but none will actually be executed '
'none will actually be executed '
'(useful for testing logging)')) '(useful for testing logging)'))
### OPERATIONS ### ### OPERATIONS ###
subparsers = args.add_subparsers(help = 'Operation to perform', subparsers = args.add_subparsers(help = 'Operation to perform',
@ -617,72 +523,56 @@ def parseArgs():
help = 'List available backups.', help = 'List available backups.',
parents = [commonargs, remoteargs]) parents = [commonargs, remoteargs])
listrepoargs = subparsers.add_parser('listrepos', listrepoargs = subparsers.add_parser('listrepos',
help = ('List availabile/configured ' help = ('List availabile/configured repositories.'),
'repositories.'),
parents = [commonargs]) parents = [commonargs])
initargs = subparsers.add_parser('init', initargs = subparsers.add_parser('init',
help = 'Initialise a repository.', help = 'Initialise a repository.',
parents = [commonargs, remoteargs]) parents = [commonargs, remoteargs])
rstrargs = subparsers.add_parser('restore', rstrargs = subparsers.add_parser('restore',
help = ('Restore ("extract") an ' help = ('Restore ("extract") an archive.'),
'archive.'),
parents = [commonargs, parents = [commonargs,
remoteargs, remoteargs,
fileargs]) fileargs])
cvrtargs = subparsers.add_parser('convert',
help = ('Convert the legacy JSON format to the new XML format and quit'))
### OPERATION-SPECIFIC OPTIONS ### ### OPERATION-SPECIFIC OPTIONS ###
# CREATE ("backup") # # CREATE ("backup") #
backupargs.add_argument('-s', '--stagedir', backupargs.add_argument('-s', '--stagedir',
default = _stagedir, default = _stagedir,
dest = 'stagedir', dest = 'stagedir',
help = ( help = ('The directory used for staging temporary files, if necessary. '
'The directory used for staging ' 'Default: \033[1m{0}\033[0m').format(_stagedir))
'temporary files, if necessary. '
'Default: \033[1m{0}\033[0m').format(
_stagedir))
backupargs.add_argument('-m', '--mysqldir', backupargs.add_argument('-m', '--mysqldir',
default = _mysqldir, default = _mysqldir,
dest = 'mysqldir', dest = 'mysqldir',
help = ( help = ('The path to where MySQL dumps should go. '
'The path to where MySQL dumps should go. ' 'Default: \033[1m{0}\033[0m').format(_mysqldir))
'Default: \033[1m{0}\033[0m').format(
_mysqldir))
# DISPLAY/OUTPUT ("list") # # DISPLAY/OUTPUT ("list") #
listargs.add_argument('-a', '--archive', listargs.add_argument('-a', '--archive',
dest = 'archive', dest = 'archive',
default = False, default = False,
help = 'If specified, will list the *contents* of ' help = 'If specified, will list the *contents* of the given archive name.')
'the given archive name.')
listargs.add_argument('-l', '--limit', listargs.add_argument('-l', '--limit',
dest = 'numlimit', dest = 'numlimit',
type = int, type = int,
default = '5', default = '5',
help = ( help = ('If specified, constrain the outout to this number of results each repo. '
'If specified, constrain the outout to this ' 'Default is \033[1m5\033[0m, use 0 for unlimited. See \033[1m-H/--list-help\033[0m'))
'number of results each repo. '
'Default is \033[1m5\033[0m, use 0 for '
'unlimited. See '
'\033[1m-H/--list-help\033[0m'))
listargs.add_argument('-s', '--sort', listargs.add_argument('-s', '--sort',
dest = 'sortby', dest = 'sortby',
choices = ['newest', 'oldest'], choices = ['newest', 'oldest'],
default = 'oldest', default = 'oldest',
help = ( help = ('The order to sort the results by. See \033[1m-H/--list-help\033[0m. '
'The order to sort the results by. '
'See \033[1m-H/--list-help\033[0m. '
'Default: \033[1moldest\033[0m')) 'Default: \033[1moldest\033[0m'))
listargs.add_argument('-x', '--invert', listargs.add_argument('-x', '--invert',
dest = 'invert', dest = 'invert',
action = 'store_true', action = 'store_true',
help = 'Invert the order of results. ' help = 'Invert the order of results. See \033[1m-H/--list-help\033[0m.')
'See \033[1m-H/--list-help\033[0m.')
listargs.add_argument('-o', '--old', listargs.add_argument('-o', '--old',
dest = 'old', dest = 'old',
action = 'store_true', action = 'store_true',
help = ( help = ('Instead of grabbing the latest results, grab the earliest results. This differs '
'Instead of grabbing the latest results, ' 'from \033[1m-s/--sort\033[0m. See \033[1m-H/--list-help\033[0m.'))
'grab the earliest results. '
'This differs from \033[1m-s/--sort\033[0m. '
'See \033[1m-H/--list-help\033[0m.'))
listargs.add_argument('-H', '--list-help', listargs.add_argument('-H', '--list-help',
dest = 'moarhelp', dest = 'moarhelp',
action = 'store_true', action = 'store_true',
@ -692,12 +582,91 @@ def parseArgs():
rstrargs.add_argument('-t', '--target', rstrargs.add_argument('-t', '--target',
required = True, required = True,
dest = 'target_dir', dest = 'target_dir',
help = ('The path to the directory where the ' help = ('The path to the directory where the restore should be dumped to. It is '
'restore should be dumped to. It is ' 'recommended to NOT restore to the same directory that the archive is taken from.'))
'recommended to NOT restore to the same '
'directory that the archive is taken from.'))
return (args) return (args)


def convertConf(cfgfile):
try:
with open(cfgfile, 'r') as f:
oldcfg = json.load(f)
except json.decoder.JSONDecodeError:
# It's not JSON. It's either already XML or invalid config.
return(cfgfile)
# Switched from JSON to XML, so we need to do some basic conversion.
newfname = re.sub('(\.json)?$', '.xml', os.path.basename(cfgfile))
newcfg = os.path.join(os.path.dirname(cfgfile),
newfname)
if os.path.exists(newcfg):
# Do nothing. We don't want to overwrite an existing config
# and we'll assume it's an already-done conversion.
return(newcfg)
print(('It appears that you are still using the legacy JSON format. '
'We will attempt to convert it to the new XML format ({0}) but it may '
'require modifications, especially if you are using any prep functions as those are not '
'converted automatically. See sample.config.xml for an example of this.').format(newcfg))
cfg = etree.Element('borg')
# The old format only supported one server.
server = etree.Element('server')
server.attrib['target'] = oldcfg['config']['host']
server.attrib['rsh'] = oldcfg['config']['ctx']
server.attrib['user'] = oldcfg['config']['user']
for r in oldcfg['repos']:
repo = etree.Element('repo')
repo.attrib['name'] = r
repo.attrib['password'] = oldcfg['repos'][r]['password']
for p in oldcfg['repos'][r]['paths']:
path = etree.Element('path')
server.append(repo)
# Build the full XML spec.
namespaces = {'borg': 'http://git.square-r00t.net/OpTools/tree/storage/backups/borg/',
'xsi': 'http://www.w3.org/2001/XMLSchema-instance'}
xsi = {('{http://www.w3.org/2001/'
'XMLSchema-instance}schemaLocation'): ('http://git.square-r00t.net/OpTools/plain/'
'storage/backups/borg/config.xsd')}
if has_lxml:
genname = 'LXML (http://lxml.de/)'
root = etree.Element('borg', nsmap = namespaces, attrib = xsi)
else:
genname = 'Python stdlib "xml" module'
for ns in namespaces.keys():
etree.register_namespace(ns, namespaces[ns])
root = etree.Element('borg')
fromstr = cfgfile
root.append(etree.Comment(
('Generated by {0} on {1} from {2} via {3}').format(sys.argv[0],
datetime.datetime.now(),
fromstr,
genname)))
root.append(etree.Comment('THIS FILE CONTAINS SENSITIVE INFORMATION. SHARE/SCRUB WISELY.'))
for x in cfg:
root.append(x)
# Write out the file to disk.
if has_lxml:
xml = etree.ElementTree(root)
with open(newcfg, 'wb') as f:
xml.write(f,
xml_declaration = True,
encoding = 'utf-8',
pretty_print = True)
else:
import xml.dom.minidom
xmlstr = etree.tostring(root, encoding = 'utf-8')
# holy cats, the xml module sucks.
nsstr = ''
for ns in namespaces.keys():
nsstr += ' xmlns:{0}="{1}"'.format(ns, namespaces[ns])
for x in xsi.keys():
xsiname = x.split('}')[1]
nsstr += ' xsi:{0}="{1}"'.format(xsiname, xsi[x])
outstr = xml.dom.minidom.parseString(xmlstr).toprettyxml(indent = ' ').splitlines()
outstr[0] = '<?xml version=\'1.0\' encoding=\'utf-8\'?>'
outstr[1] = '<borg{0}>'.format(nsstr)
with open(newcfg, 'w') as f:
f.write('\n'.join(outstr))
# Return the new config's path.
return(newcfg)



def main(): def main():
rawargs = parseArgs() rawargs = parseArgs()
@ -709,6 +678,17 @@ def main():
exit(0) exit(0)
if 'moarhelp' in args.keys() and args['moarhelp']: if 'moarhelp' in args.keys() and args['moarhelp']:
printMoarHelp() printMoarHelp()
if args['oper'] == 'convert':
convertConf(args['cfgfile'])
return()
else:
try:
with open(args['cfgfile'], 'r') as f:
json.load(f)
args['cfgfile'] = convertConf(args['cfgfile'])
except json.decoder.JSONDecodeError:
# It's not JSON. It's either already XML or invalid config.
pass
# The "Do stuff" part # The "Do stuff" part
bak = Backup(args) bak = Backup(args)
if args['oper'] == 'list': if args['oper'] == 'list':

775
storage/backups/borg/backup.py.old Executable file
View File

@ -0,0 +1,775 @@
#!/usr/bin/env python3

# TODO: https://borgbackup.readthedocs.io/en/latest/internals/frontends.html
# will they EVER release a public API? for now we'll just use subprocess since
# we import it for various prep stuff anyways.
# TODO: change loglevel of borg itself in subprocess to match the argparse?
# --debug, --info (same as -v/--verbose), --warning, --error, --critical
# TODO: modify config to add repo to cfg for init? or add new operation, "add"

import argparse
import configparser
import datetime
import json
import getpass
import logging
import logging.handlers
import os
import re
import subprocess
import sys
# TODO: virtual env?
try:
from lxml import etree
has_lxml = True
except ImportError:
import xml.etree.ElementTree as etree # https://docs.python.org/3/library/xml.etree.elementtree.html
has_lxml = False

try:
import pymysql # not stdlib; "python-pymysql" in Arch's AUR
has_mysql = True
except ImportError:
has_mysql = False
try:
# https://www.freedesktop.org/software/systemd/python-systemd/journal.html#journalhandler-class
from systemd import journal
has_systemd = True
except ImportError:
has_systemd = False

### LOG LEVEL MAPPINGS ###
loglvls = {
'critical': logging.CRITICAL,
'error': logging.ERROR,
'warning': logging.WARNING,
'info': logging.INFO,
'debug': logging.DEBUG}


### THE GUTS ###
class Backup(object):
def __init__(self, args):
self.args = args
### DIRECTORIES ###
if self.args['oper'] == 'backup':
for d in (self.args['mysqldir'], self.args['stagedir']):
os.makedirs(d, exist_ok = True, mode = 0o700)
if self.args['oper'] == 'restore':
self.args['target_dir'] = os.path.abspath(os.path.expanduser(
self.args['target_dir']))
os.makedirs(os.path.dirname(self.args['oper']),
exist_ok = True,
mode = 0o700)
### LOGGING ###
# Thanks to:
# https://web.archive.org/web/20170726052946/http://www.lexev.org/en/2013/python-logging-every-day/
# https://stackoverflow.com/a/42604392
# https://plumberjack.blogspot.com/2010/10/supporting-alternative-formatting.html
# and user K900_ on r/python for entertaining my very silly question.
self.logger = logging.getLogger(__name__)
self.logger.setLevel(loglvls[self.args['loglevel']])
_logfmt = logging.Formatter(
fmt = ('{levelname}:{name}: {message} ({asctime}; '
'{filename}:{lineno})'),
style = '{',
datefmt = '%Y-%m-%d %H:%M:%S')
_journalfmt = logging.Formatter(
fmt = '{levelname}:{name}: {message} ({filename}:{lineno})',
style = '{',
datefmt = '%Y-%m-%d %H:%M:%S')
handlers = []
if self.args['disklog']:
os.makedirs(os.path.dirname(self.args['logfile']),
exist_ok = True,
mode = 0o700)
# TODO: make the constraints for rotation in config?
handlers.append(
logging.handlers.RotatingFileHandler(self.args['logfile'],
encoding = 'utf8',
maxBytes = 100000,
backupCount = 1))
if self.args['verbose']:
handlers.append(logging.StreamHandler())
if has_systemd:
h = journal.JournalHandler()
h.setFormatter(_journalfmt)
h.setLevel(loglvls[self.args['loglevel']])
self.logger.addHandler(h)
for h in handlers:
h.setFormatter(_logfmt)
h.setLevel(loglvls[self.args['loglevel']])
self.logger.addHandler(h)
self.logger.debug('BEGIN INITIALIZATION')
### CONFIG ###
if not os.path.isfile(self.args['cfgfile']):
self.logger.error(
'{0} does not exist'.format(self.args['cfgfile']))
exit(1)
with open(self.args['cfgfile'], 'r') as f:
self.cfg = json.loads(f.read())
### END LOGGING ###
### ARGS CLEANUP ###
self.logger.debug('VARS (before args cleanup): {0}'.format(vars()))
self.args['repo'] = [i.strip() for i in self.args['repo'].split(',')]
if 'all' in self.args['repo']:
self.args['repo'] = list(self.cfg['repos'].keys())
for r in self.args['repo'][:]:
if r == 'all':
self.args['repo'].remove(r)
elif r not in self.cfg['repos'].keys():
self.logger.warning(
'Repository {0} is not configured; skipping.'.format(
r))
self.args['repo'].remove(r)
self.logger.debug('VARS (after args cleanup): {0}'.format(vars()))
self.logger.debug('END INITIALIZATION')
### CHECK ENVIRONMENT ###
# If we're running from cron, we want to print errors to stdout.
if os.isatty(sys.stdin.fileno()):
self.cron = False
else:
self.cron = True
### END INIT ###

def cmdExec(self, cmd, stdoutfh = None):
self.logger.debug('Running command: {0}'.format(' '.join(cmd)))
if self.args['dryrun']:
return () # no-op
if stdoutfh:
_cmd = subprocess.run(cmd, stdout = stdoutfh,
stderr = subprocess.PIPE)
else:
_cmd = subprocess.run(cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
_out = _cmd.stdout.decode('utf-8').strip()
_err = _cmd.stderr.decode('utf-8').strip()
_returncode = _cmd.returncode
if _returncode != 0:
self.logger.error('STDERR: ({1})\n{0}'.format(_err, ' '.join(cmd)))
if _err != '' and self.cron:
self.logger.warning(
'Command {0} failed: {1}'.format(' '.join(cmd), _err))
return()

def createRepo(self):
_env = os.environ.copy()
_env['BORG_RSH'] = self.cfg['config']['ctx']
for r in self.args['repo']:
self.logger.info('[{0}]: BEGIN INITIALIZATION'.format(r))
_cmd = ['borg',
'init',
'-v',
'{0}@{1}:{2}'.format(self.cfg['config']['user'],
self.cfg['config']['host'],
r)]
_env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password']
# We don't use self.cmdExec() here either because
# again, custom env, etc.
self.logger.debug('VARS: {0}'.format(vars()))
if not self.args['dryrun']:
_out = subprocess.run(_cmd,
env = _env,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
_stdout = _out.stdout.decode('utf-8').strip()
_stderr = _out.stderr.decode('utf-8').strip()
_returncode = _out.returncode
self.logger.debug('[{0}]: (RESULT) {1}'.format(r, _stdout))
# sigh. borg uses stderr for verbose output.
self.logger.debug('[{0}]: STDERR: ({2})\n{1}'.format(r,
_stderr,
' '.join(
_cmd)))
if _returncode != 0:
self.logger.error(
'[{0}]: FAILED: {1}'.format(r, ' '.join(_cmd)))
if _err != '' and self.cron and _returncode != 0:
self.logger.warning(
'Command {0} failed: {1}'.format(' '.join(cmd),
_err))
del (_env['BORG_PASSPHRASE'])
self.logger.info('[{0}]: END INITIALIZATION'.format(r))
return()

def create(self):
# TODO: support "--strip-components N"?
_env = os.environ.copy()
_env['BORG_RSH'] = self.cfg['config']['ctx']
self.logger.info('START: backup')
for r in self.args['repo']:
self.logger.info('[{0}]: BEGIN BACKUP'.format(r))
if 'prep' in self.cfg['repos'][r].keys():
for prep in self.cfg['repos'][r]['prep']:
self.logger.info(
'[{0}]: Running prepfunc {1}'.format(r, prep))
eval('self.{0}'.format(
prep)) # I KNOW, IT'S TERRIBLE. so sue me.
self.logger.info(
'[{0}]: Finished prepfunc {1}'.format(r, prep))
_cmd = ['borg',
'create',
'-v', '--stats',
'--compression', 'lzma,9']
if 'excludes' in self.cfg['repos'][r].keys():
for e in self.cfg['repos'][r]['excludes']:
_cmd.extend(['--exclude', e])
_cmd.append('{0}@{1}:{2}::{3}'.format(self.cfg['config']['user'],
self.cfg['config']['host'],
r,
self.args['archive']))
for p in self.cfg['repos'][r]['paths']:
_cmd.append(p)
_env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password']
self.logger.debug('VARS: {0}'.format(vars()))
# We don't use self.cmdExec() here because we want to explicitly
# pass the env and format the log line differently.
self.logger.debug(
'[{0}]: Running command: {1}'.format(r, ' '.join(_cmd)))
if not self.args['dryrun']:
_out = subprocess.run(_cmd,
env = _env,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
_stdout = _out.stdout.decode('utf-8').strip()
_stderr = _out.stderr.decode('utf-8').strip()
_returncode = _out.returncode
self.logger.debug('[{0}]: (RESULT) {1}'.format(r, _stdout))
self.logger.error('[{0}]: STDERR: ({2})\n{1}'.format(r,
_stderr,
' '.join(
_cmd)))
if _returncode != 0:
self.logger.error(
'[{0}]: FAILED: {1}'.format(r, ' '.join(_cmd)))
if _stderr != '' and self.cron and _returncode != 0:
self.logger.warning(
'Command {0} failed: {1}'.format(' '.join(_cmd),
_stderr))
del (_env['BORG_PASSPHRASE'])
self.logger.info('[{0}]: END BACKUP'.format(r))
self.logger.info('END: backup')
return()

def restore(self):
# TODO: support "--strip-components N"?
# TODO: support add'l args?
# https://borgbackup.readthedocs.io/en/stable/usage/extract.html
_env = os.environ.copy()
_env['BORG_RSH'] = self.cfg['config']['ctx']
self.logger.info('START: restore')
for r in self.args['repo']:
self.logger.info('[{0}]: BEGIN RESTORE'.format(r))
_cmd = ['borg',
'extract',
'-v']
# if 'excludes' in self.cfg['repos'][r].keys():
# for e in self.cfg['repos'][r]['excludes']:
# _cmd.extend(['--exclude', e])
_cmd.append('{0}@{1}:{2}::{3}'.format(self.cfg['config']['user'],
self.cfg['config']['host'],
r,
self.args['archive']))
_cmd.append(os.path.abspath(self.args['target_dir']))
# TODO: support specific path inside archive?
# if so, append path(s) here.
_env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password']
self.logger.debug('VARS: {0}'.format(vars()))
# We don't use self.cmdExec() here because we want to explicitly
# pass the env and format the log line differently.
self.logger.debug(
'[{0}]: Running command: {1}'.format(r, ' '.join(_cmd)))
if not self.args['dryrun']:
_out = subprocess.run(_cmd,
env = _env,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
_stdout = _out.stdout.decode('utf-8').strip()
_stderr = _out.stderr.decode('utf-8').strip()
_returncode = _out.returncode
self.logger.debug('[{0}]: (RESULT) {1}'.format(r, _stdout))
self.logger.error('[{0}]: STDERR: ({2})\n{1}'.format(r,
_stderr,
' '.join(
_cmd)))
if _returncode != 0:
self.logger.error(
'[{0}]: FAILED: {1}'.format(r, ' '.join(_cmd)))
if _stderr != '' and self.cron and _returncode != 0:
self.logger.warning(
'Command {0} failed: {1}'.format(' '.join(_cmd),
_stderr))
del (_env['BORG_PASSPHRASE'])
self.logger.info('[{0}]: END RESTORE'.format(r))
self.logger.info('END: restore')
return()

def miscBak(self, pkgr):
self.logger.info('BEGIN: miscBak()')
_cmd = None
for p in os.environ['PATH'].split(':'):
d = os.path.expanduser(p)
if os.path.isfile(os.path.join(d, pkgr)):
_pkgr = pkgr
self.logger.debug('Package tool found at {0}'.format(_pkgr))
else:
_pkgr = 'pacman'
self.logger.debug('Using {0} as package tool'.format(_pkgr))
with open(os.path.join(self.args['stagedir'], 'pkg.lst'), 'w') as f:
_cmd = [_pkgr,
'-Qet',
'--color',
'never']
self.cmdExec(_cmd, stdoutfh = f)
self.logger.info('END: miscBak()')
return()

def mysqlBak(self):
self.logger.info('BEGIN: mysqlBak()')
if not has_mysql:
self.logger.error(
'You need to install the PyMySQL module to back up MySQL databases. Skipping.')
return ()
# These are mysqldump options shared by ALL databases
_mysqlopts = ['--routines',
'--add-drop-database',
'--add-drop-table',
'--allow-keywords',
'--complete-insert',
'--create-options',
'--extended-insert']
_DBs = []
_mycnf = os.path.expanduser(os.path.join('~', '.my.cnf'))
if not os.path.isfile(_mycnf):
exit(
'{0}: ERROR: Cannot get credentials for MySQL (cannot find ~/.my.cnf)!')
_mycfg = configparser.ConfigParser()
_mycfg._interpolation = configparser.ExtendedInterpolation()
_mycfg.read(_mycnf)
_sqlcfg = {s: dict(_mycfg.items(s)) for s in _mycfg.sections()}
if 'host' not in _sqlcfg.keys():
_socketpath = '/var/run/mysqld/mysqld.sock' # correct for Arch, YMMV.
_mysql = pymysql.connect(unix_socket = _socketpath,
user = _sqlcfg['client']['user'],
passwd = _sqlcfg['client']['password'])
else:
_mysql = pymysql.connect(host = _sqlcfg['client']['host'],
user = _sqlcfg['client']['user'],
port = _sqlcfg['client']['port'],
passwd = _sqlcfg['client']['password'])
_cur = _mysql.cursor()
_cur.execute('SHOW DATABASES')
for row in _cur.fetchall():
_DBs.append(row[0])
self.logger.debug('Databases: {0}'.format(', '.join(_DBs)))
for db in _DBs:
_cmd = ['mysqldump',
'--result-file={0}.sql'.format(
os.path.join(self.args['mysqldir'], db))]
# These are database-specific options
if db in ('information_schema', 'performance_schema'):
_cmd.append('--skip-lock-tables')
elif db == 'mysql':
_cmd.append('--flush-privileges')
_cmd.extend(_mysqlopts)
_cmd.append(db)
self.cmdExec(_cmd)
self.logger.info('END: mysqlBak()')
return()

def listRepos(self):
print('\n\033[1mCurrently configured repositories are:\033[0m\n')
print('\t{0}\n'.format(', '.join(self.cfg['repos'].keys())))
if self.args['verbose']:
print('\033[1mDETAILS:\033[0m\n')
for r in self.args['repo']:
print('\t\033[1m{0}:\033[0m\n\t\t\033[1mPath(s):\033[0m\t'.format(
r.upper()), end = '')
for p in self.cfg['repos'][r]['paths']:
print(p, end = ' ')
if 'prep' in self.cfg['repos'][r].keys():
print('\n\t\t\033[1mPrep:\033[0m\t\t', end = '')
for p in self.cfg['repos'][r]['prep']:
print(p, end = ' ')
if 'excludes' in self.cfg['repos'][r].keys():
print('\n\t\t\033[1mExclude(s):\033[0m\t', end = '')
for p in self.cfg['repos'][r]['excludes']:
print(p, end = ' ')
print('\n')
return()

def printer(self):
# TODO: better alignment. https://stackoverflow.com/a/5676884
_results = self.lister()
if not self.args['archive']: # It's a listing of archives
print('\033[1mREPO:\tSNAPSHOT:\t\tTIMESTAMP:\033[0m\n')
for r in _results.keys():
print(r, end = '')
for line in _results[r]:
_snapshot = line.split()
print('\t{0}\t\t{1}'.format(_snapshot[0],
' '.join(_snapshot[1:])))
print()
else: # It's a listing inside an archive
if self.args['verbose']:
_fields = ['REPO:', 'PERMS:', 'OWNERSHIP:', 'SIZE:', 'TIMESTAMP:', 'PATH:']
for r in _results.keys():
print('\033[1m{0}\t{1}\033[0m'.format(_fields[0], r))
# https://docs.python.org/3/library/string.html#formatspec
print('{0[1]:<15}\t{0[2]:<15}\t{0[3]:<15}\t{0[4]:<24}\t{0[5]:<15}'.format(_fields))
for line in _results[r]:
_fline = line.split()
_perms = _fline[0]
_ownership = '{0}:{1}'.format(_fline[1], _fline[2])
_size = _fline[3]
_time = ' '.join(_fline[4:7])
_path = ' '.join(_fline[7:])
print('{0:<15}\t{1:<15}\t{2:<15}\t{3:<24}\t{4:<15}'.format(_perms,
_ownership,
_size,
_time,
_path))
else:
print('\033[1mREPO:\tPATH:\033[0m\n')
for r in _results.keys():
print(r, end = '')
for line in _results[r]:
_fline = line.split()
print('\t{0}'.format(' '.join(_fline[7:])))
return()

def lister(self):
output = {}
_env = os.environ.copy()
self.logger.debug('START: lister')
_env['BORG_RSH'] = self.cfg['config']['ctx']
for r in self.args['repo']:
if self.args['archive']:
_cmd = ['borg',
'list',
'{0}@{1}:{2}::{3}'.format(self.cfg['config']['user'],
self.cfg['config']['host'],
r,
self.args['archive'])]
else:
_cmd = ['borg',
'list',
'{0}@{1}:{2}'.format(self.cfg['config']['user'],
self.cfg['config']['host'],
r)]
_env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password']
if not self.args['dryrun']:

_out = subprocess.run(_cmd,
env = _env,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
_stdout = [i.strip() for i in _out.stdout.decode('utf-8').splitlines()]
_stderr = _out.stderr.decode('utf-8').strip()
_returncode = _out.returncode
output[r] = _stdout
self.logger.debug('[{0}]: (RESULT) {1}'.format(r,
'\n'.join(_stdout)))
if _returncode != 0:
self.logger.error('[{0}]: STDERR: ({2}) ({1})'.format(r,
_stderr,
' '.join(_cmd)))
if _stderr != '' and self.cron and _returncode != 0:
self.logger.warning(
'Command {0} failed: {1}'.format(' '.join(cmd), _err))
del(_env['BORG_PASSPHRASE'])
if not self.args['archive']:
if self.args['numlimit'] > 0:
if self.args['old']:
output[r] = output[r][:self.args['numlimit']]
else:
output[r] = list(reversed(output[r]))[:self.args['numlimit']]
if self.args['invert']:
output[r] = reversed(output[r])
self.logger.debug('END: lister')
return(output)


def printMoarHelp():
_helpstr = ('\n\tNOTE: Sorting only applies to listing archives, NOT the contents!\n\n'
'In order to efficiently display results, there are several options to handle it. '
'Namely, these are:\n\n\t\t'
'-s/--sort [direction]\n\t\t'
'-l/--limit [number]\n\t\t'
'-x/--invert\n\n'
'For example, if you want to list the 5 most recently *taken* snapshots, you would use:\n\n\t\t'
'-l 5\n\n'
'If you would want those SAME results SORTED in the reverse order (i.e. the 5 most recently '
'taken snapshots sorted from newest to oldest), then it would be: \n\n\t\t'
'-l 5 -x\n\n'
'Lastly, if you wanted to list the 7 OLDEST TAKEN snapshots in reverse order '
'(that is, sorted from newest to oldest), that\'d be:\n\n\t\t'
'-o -l 7 -x\n')
print(_helpstr)
exit(0)


def parseArgs():
### DEFAULTS ###
_date = datetime.datetime.now().strftime("%Y_%m_%d.%H_%M")
_logfile = '/var/log/borg/{0}'.format(_date)
_mysqldir = os.path.abspath(
os.path.join(os.path.expanduser('~'),
'.bak',
'mysql'))
_stagedir = os.path.abspath(
os.path.join(os.path.expanduser('~'),
'.bak',
'misc'))
_cfgfile = os.path.abspath(
os.path.join(os.path.expanduser('~'),
'.config',
'optools',
'backup.xml'))
_defloglvl = 'info'
######
args = argparse.ArgumentParser(description = 'Backups manager',
epilog = ('TIP: this program has context-specific help. '
'e.g. try "%(prog)s list --help"'))
args.add_argument('-c', '--config',
dest = 'cfgfile',
default = _cfgfile,
help = (
'The path to the config file. '
'Default: \033[1m{0}\033[0m'.format(_cfgfile)))
args.add_argument('-Ll', '--loglevel',
dest = 'loglevel',
default = _defloglvl,
choices = list(loglvls.keys()),
help = (
'The level of logging to perform. \033[1mWARNING:\033[0m \033[1mdebug\033[0m will '
'log VERY sensitive information such as passwords! '
'Default: \033[1m{0}\033[0m'.format(_defloglvl)))
args.add_argument('-Ld', '--log-to-disk',
dest = 'disklog',
action = 'store_true',
help = (
'If specified, log to a specific file (-Lf/--logfile) instead of the system logger.'))
args.add_argument('-Lf', '--logfile',
dest = 'logfile',
default = _logfile,
help = (
'The path to the logfile, only used if -Ld/--log-to-disk is specified. '
'Default: \033[1m{0}\033[0m (dynamic)').format(_logfile))
args.add_argument('-v', '--verbose',
dest = 'verbose',
action = 'store_true',
help = ('If specified, log messages will be printed to STDERR in addition to the other '
'configured log system(s), and verbosity for printing functions is increased. '
'\033[1mWARNING:\033[0m This may display VERY sensitive information such as passwords!'))
### ARGS FOR ALL OPERATIONS ###
commonargs = argparse.ArgumentParser(add_help = False)
commonargs.add_argument('-r', '--repo',
dest = 'repo',
default = 'all',
help = ('The repository to perform the operation for. '
'The default is \033[1mall\033[0m, a special value that specifies all known '
'repositories. Can also accept a comma-separated list.'))
fileargs = argparse.ArgumentParser(add_help = False)
fileargs.add_argument('-a', '--archive',
default = _date,
dest = 'archive',
help = ('The name of the archive/snapshot. '
'Default: \033[1m{0}\033[0m (dynamic)').format(_date))
remoteargs = argparse.ArgumentParser(add_help = False)
remoteargs.add_argument('-d', '--dry-run',
dest = 'dryrun',
action = 'store_true',
help = ('Act as if we are performing tasks, but none will actually be executed '
'(useful for testing logging)'))
### OPERATIONS ###
subparsers = args.add_subparsers(help = 'Operation to perform',
dest = 'oper')
backupargs = subparsers.add_parser('backup',
help = 'Perform a backup.',
parents = [commonargs,
remoteargs,
fileargs])
listargs = subparsers.add_parser('list',
help = 'List available backups.',
parents = [commonargs, remoteargs])
listrepoargs = subparsers.add_parser('listrepos',
help = ('List availabile/configured repositories.'),
parents = [commonargs])
initargs = subparsers.add_parser('init',
help = 'Initialise a repository.',
parents = [commonargs, remoteargs])
rstrargs = subparsers.add_parser('restore',
help = ('Restore ("extract") an archive.'),
parents = [commonargs,
remoteargs,
fileargs])
cvrtargs = subparsers.add_parser('convert',
help = ('Convert the legacy JSON format to the new XML format and quit'))
### OPERATION-SPECIFIC OPTIONS ###
# CREATE ("backup") #
backupargs.add_argument('-s', '--stagedir',
default = _stagedir,
dest = 'stagedir',
help = ('The directory used for staging temporary files, if necessary. '
'Default: \033[1m{0}\033[0m').format(_stagedir))
backupargs.add_argument('-m', '--mysqldir',
default = _mysqldir,
dest = 'mysqldir',
help = ('The path to where MySQL dumps should go. '
'Default: \033[1m{0}\033[0m').format(_mysqldir))
# DISPLAY/OUTPUT ("list") #
listargs.add_argument('-a', '--archive',
dest = 'archive',
default = False,
help = 'If specified, will list the *contents* of the given archive name.')
listargs.add_argument('-l', '--limit',
dest = 'numlimit',
type = int,
default = '5',
help = ('If specified, constrain the outout to this number of results each repo. '
'Default is \033[1m5\033[0m, use 0 for unlimited. See \033[1m-H/--list-help\033[0m'))
listargs.add_argument('-s', '--sort',
dest = 'sortby',
choices = ['newest', 'oldest'],
default = 'oldest',
help = ('The order to sort the results by. See \033[1m-H/--list-help\033[0m. '
'Default: \033[1moldest\033[0m'))
listargs.add_argument('-x', '--invert',
dest = 'invert',
action = 'store_true',
help = 'Invert the order of results. See \033[1m-H/--list-help\033[0m.')
listargs.add_argument('-o', '--old',
dest = 'old',
action = 'store_true',
help = ('Instead of grabbing the latest results, grab the earliest results. This differs '
'from \033[1m-s/--sort\033[0m. See \033[1m-H/--list-help\033[0m.'))
listargs.add_argument('-H', '--list-help',
dest = 'moarhelp',
action = 'store_true',
help = ('Print extended information about how to '
'manage the output of listing and exit.'))
## EXTRACT ("restore")
rstrargs.add_argument('-t', '--target',
required = True,
dest = 'target_dir',
help = ('The path to the directory where the restore should be dumped to. It is '
'recommended to NOT restore to the same directory that the archive is taken from.'))
return (args)

def convertConf(cfgfile):
import json
try:
with open(cfgfile, 'r') as f:
oldcfg = json.load(f)
except json.decoder.JSONDecodeError:
# It's not JSON. It's either already XML or invalid config.
return(cfgfile)
# Switched from JSON to XML, so we need to do some basic conversion.
newfname = re.sub('(\.json)?$', '.xml', os.path.basename(cfgfile))
newcfg = os.path.join(os.path.dirname(cfgfile),
newfname)
if os.path.exists(newcfg):
# Do nothing. We don't want to overwrite an existing config
# and we'll assume it's an already-done conversion.
return(newcfg)
print(('It appears that you are still using the legacy JSON format. '
'We will attempt to convert it to the new XML format ({0}) but it may '
'require modifications, especially if you are using any prep functions as those are not '
'converted automatically. See sample.config.xml for an example of this.').format(newcfg))
cfg = etree.Element('borg')
# The old format only supported one server.
server = etree.Element('server')
server.attrib['target'] = oldcfg['config']['host']
server.attrib['rsh'] = oldcfg['config']['ctx']
server.attrib['user'] = oldcfg['config']['user']
for r in oldcfg['repos']:
repo = etree.Element('repo')
repo.attrib['name'] = r
repo.attrib['password'] = oldcfg['repos'][r]['password']
for p in oldcfg['repos'][r]['paths']:
path = etree.Element('path')
server.append(repo)
# Build the full XML spec.
namespaces = {'borg': 'http://git.square-r00t.net/OpTools/tree/storage/backups/borg/',
'xsi': 'http://www.w3.org/2001/XMLSchema-instance'}
xsi = {('{http://www.w3.org/2001/'
'XMLSchema-instance}schemaLocation'): ('http://git.square-r00t.net/OpTools/plain/'
'storage/backups/borg/config.xsd')}
if has_lxml:
genname = 'LXML (http://lxml.de/)'
root = etree.Element('borg', nsmap = namespaces, attrib = xsi)
else:
genname = 'Python stdlib "xml" module'
for ns in namespaces.keys():
etree.register_namespace(ns, namespaces[ns])
root = etree.Element('borg')
fromstr = cfgfile
root.append(etree.Comment(
('Generated by {0} on {1} from {2} via {3}').format(sys.argv[0],
datetime.datetime.now(),
fromstr,
genname)))
root.append(etree.Comment('THIS FILE CONTAINS SENSITIVE INFORMATION. SHARE/SCRUB WISELY.'))
for x in cfg:
root.append(x)
# Write out the file to disk.
if has_lxml:
xml = etree.ElementTree(root)
with open(newcfg, 'wb') as f:
xml.write(f,
xml_declaration = True,
encoding = 'utf-8',
pretty_print = True)
else:
import xml.dom.minidom
xmlstr = etree.tostring(root, encoding = 'utf-8')
# holy cats, the xml module sucks.
nsstr = ''
for ns in namespaces.keys():
nsstr += ' xmlns:{0}="{1}"'.format(ns, namespaces[ns])
for x in xsi.keys():
xsiname = x.split('}')[1]
nsstr += ' xsi:{0}="{1}"'.format(xsiname, xsi[x])
outstr = xml.dom.minidom.parseString(xmlstr).toprettyxml(indent = ' ').splitlines()
outstr[0] = '<?xml version=\'1.0\' encoding=\'utf-8\'?>'
outstr[1] = '<borg{0}>'.format(nsstr)
with open(newcfg, 'w') as f:
f.write('\n'.join(outstr))
# Return the new config's path.
return(newcfg)


def main():
rawargs = parseArgs()
parsedargs = rawargs.parse_args()
args = vars(parsedargs)
args['cfgfile'] = os.path.abspath(os.path.expanduser(args['cfgfile']))
if not args['oper']:
rawargs.print_help()
exit(0)
if args['oper'] == 'convert':
convertConf(args['cfgfile'])
return()
else:
args['cfgfile'] = convertConf(args['cfgfile'])
if 'moarhelp' in args.keys() and args['moarhelp']:
printMoarHelp()
# The "Do stuff" part
bak = Backup(args)
if args['oper'] == 'list':
bak.printer()
elif args['oper'] == 'listrepos':
bak.listRepos()
elif args['oper'] == 'backup':
bak.create()
elif args['oper'] == 'init':
bak.createRepo()
elif args['oper'] == 'restore':
bak.restore()
return()


if __name__ == '__main__':
main()

39
storage/backups/borg/config.xsd
View File

@ -30,10 +30,15 @@
<xs:element name="repo" minOccurs="1" maxOccurs="unbounded"> <xs:element name="repo" minOccurs="1" maxOccurs="unbounded">
<xs:complexType> <xs:complexType>
<xs:choice minOccurs="1" maxOccurs="unbounded"> <xs:choice minOccurs="1" maxOccurs="unbounded">
<!-- START PATH -->
<xs:element name="path" minOccurs="1" <xs:element name="path" minOccurs="1"
maxOccurs="unbounded" type="xs:anyURI"/> maxOccurs="unbounded" type="xs:anyURI"/>
<!-- END PATH -->
<!-- START EXCLUDE -->
<xs:element name="exclude" minOccurs="0" <xs:element name="exclude" minOccurs="0"
maxOccurs="unbounded" type="xs:anyURI"/> maxOccurs="unbounded" type="xs:anyURI"/>
<!-- END EXCLUDE -->
<!-- START PREP -->
<!-- This gets messy. We essentially preserve whitespace, allowing <!-- This gets messy. We essentially preserve whitespace, allowing
either an inline script to be executed (written to a temp file) or either an inline script to be executed (written to a temp file) or
a path to an external script/command to be specified. --> a path to an external script/command to be specified. -->
@ -48,6 +53,40 @@
</xs:simpleContent> </xs:simpleContent>
</xs:complexType> </xs:complexType>
</xs:element> </xs:element>
<!-- END PREP -->
<!-- START PLUGIN -->
<xs:element name="plugins" minOccurs="0"
maxOccurs="1">
<xs:complexType>
<xs:sequence>
<xs:element name="plugin" minOccurs="1" maxOccurs="unbounded">
<xs:complexType>
<xs:sequence>
<xs:element name="param" minOccurs="0"
maxOccurs="unbounded">
<xs:complexType>
<xs:simpleContent>
<xs:extension base="borg:blocktext">
<xs:attribute name="key"
type="xs:token"
use="required"/>
<xs:attribute name="json"
type="xs:boolean"
default="0"
use="optional"/>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
</xs:element>
</xs:sequence>
<xs:attribute name="name" type="xs:string" use="required"/>
<xs:attribute name="path" type="xs:anyURI" use="optional"/>
</xs:complexType>
</xs:element>
</xs:sequence>
</xs:complexType>
</xs:element>
<!-- END PLUGIN -->
</xs:choice> </xs:choice>
<xs:attribute name="name" type="xs:token" use="required"/> <xs:attribute name="name" type="xs:token" use="required"/>
<!-- Optional. If not specified, the password will <!-- Optional. If not specified, the password will

96
storage/backups/borg/plugins/ldap.py Normal file
View File

@ -0,0 +1,96 @@
import os
# TODO: virtual env?
import ldap
import ldif


# Designed for use with OpenLDAP in an OLC configuration.


class Backup(object):
def __init__(self,
server = 'ldap://sub.domain.tld',
port = 389,
basedn = 'dc=domain,dc=tld',
sasl = False,
starttls = True,
binddn = 'cn=Manager,dc=domain,dc=tld',
password_file = '~/.ldap.pass',
password = None,
outdir = '~/.cache/backup/ldap',
splitldifs = True):
self.server = server
self.port = port
self.basedn = basedn
self.sasl = sasl
self.binddn = binddn
self.outdir = os.path.abspath(os.path.expanduser(outdir))
os.makedirs(self.outdir, exist_ok = True)
self.splitldifs = splitldifs
self.starttls = starttls
if password_file and not password:
with open(os.path.abspath(os.path.expanduser(password_file)), 'r') as f:
self.password = f.read().strip()
else:
self.password = password
# Human readability, yay.
# A note, SSLv3 is 0x300. But StartTLS can only be done with TLS, not SSL, I *think*?
# PRESUMABLY, now that it's finalized, TLS 1.3 will be 0x304.
# See https://tools.ietf.org/html/rfc5246#appendix-E
self._tlsmap = {'1.0': int(0x301), # 769
'1.1': int(0x302), # 770
'1.2': int(0x303)} # 771
self._minimum_tls_ver = '1.2'
if self.sasl:
self.server = 'ldapi:///'
self.cxn = None
self.connect()
self.dump()
self.close()

def connect(self):
self.cxn = ldap.initialize(self.server)
self.cxn.set_option(ldap.OPT_REFERRALS, 0)
self.cxn.set_option(ldap.OPT_PROTOCOL_VERSION, 3)
if not self.sasl:
if self.starttls:
self.cxn.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_NEVER)
self.cxn.set_option(ldap.OPT_X_TLS, ldap.OPT_X_TLS_DEMAND)
self.cxn.set_option(ldap.OPT_X_TLS_DEMAND, True)
self.cxn.set_option(ldap.OPT_X_TLS_PROTOCOL_MIN, self._tlsmap[self._minimum_tls_ver])
if self.sasl:
self.cxn.sasl_external_bind_s()
else:
if self.starttls:
self.cxn.start_tls_s()
self.cxn.bind_s(self.binddn, self.password)
return()

def dump(self):
dumps = {'schema': 'cn=config',
'data': self.basedn}
with open(os.path.join(self.outdir, ('ldap-config.ldif' if self.splitldifs else 'ldap.ldif')), 'w') as f:
l = ldif.LDIFWriter(f)
rslts = self.cxn.search_s(dumps['schema'],
ldap.SCOPE_SUBTREE,
filterstr = '(objectClass=*)',
attrlist = ['*', '+'])
for r in rslts:
l.unparse(r[0], r[1])
if self.splitldifs:
f = open(os.path.join(self.outdir, 'ldap-data.ldif'), 'w')
else:
f = open(os.path.join(self.outdir, 'ldap.ldif'), 'a')
rslts = self.cxn.search_s(dumps['data'],
ldap.SCOPE_SUBTREE,
filterstr = '(objectClass=*)',
attrlist = ['*', '+'])
l = ldif.LDIFWriter(f)
for r in rslts:
l.unparse(r[0], r[1])
f.close()

def close(self):
if self.cxn:
self.cxn.unbind_s()
return()

72
storage/backups/borg/plugins/mysql.py Normal file
View File

@ -0,0 +1,72 @@
import copy
import os
import re
import subprocess
import warnings

_mysql_ssl_re = re.compile('^ssl-(.*)$')

# TODO: is it possible to do a pure-python dump via PyMySQL?

class Backup(object):
def __init__(self, dbs = None,
cfg = '~/.my.cnf',
cfgsuffix = '',
splitdumps = True,
dumpopts = None,
mysqlbin = 'mysql',
mysqldumpbin = 'mysqldump',
outdir = '~/.cache/backup/mysql'):
# If dbs is None, we dump ALL databases.
self.dbs = dbs
self.cfgsuffix = cfgsuffix
self.splitdumps = splitdumps
self.mysqlbin = mysqlbin
self.mysqldumpbin = mysqldumpbin
self.outdir = os.path.abspath(os.path.expanduser(outdir))
self.cfg = os.path.abspath(os.path.expanduser(cfg))
os.makedirs(self.outdir, exist_ok = True)
if not os.path.isfile(self.cfg):
raise OSError(('{0} does not exist!').format(self.cfg))
if not dumpopts:
self.dumpopts = ['--routines',
'--add-drop-database',
'--add-drop-table',
'--allow-keywords',
'--complete-insert',
'--create-options',
'--extended-insert']
else:
self.dumpopts = dumpopts
self.getDBs()
self.dump()

def getDBs(self):
if not self.dbs:
_out = subprocess.run([self.mysqlbin, '-BNne', 'SHOW DATABASES'],
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
if _out.returncode != 0:
raise RuntimeError(('Could not successfully list databases: '
'{0}').format(_out.stderr.decode('utf-8')))
self.dbs = _out.stdout.decode('utf-8').strip().splitlines()
return()

def dump(self):
for db in self.dbs:
args = copy.deepcopy(self.dumpopts)
outfile = os.path.join(self.outdir, '{0}.sql'.format(db))
if db in ('information_schema', 'performance_schema'):
args.append('--skip-lock-tables')
elif db == 'mysql':
args.append('--flush-privileges')
out = subprocess.run([self.mysqldumpbin,
'--result-file={0}'.format(outfile),
args,
db],
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
if out.returncode != 0:
warn = ('Error dumping {0}: {1}').format(db, out.stderr.decode('utf-8').strip())
warnings.warn(warn)
return()

35
storage/backups/borg/sample.config.json
View File

@ -1,35 +0,0 @@
# A normal config would be placed in the current user's home directory,
# typically under ~/.config/optools/backup.json

{
"repos": {
"web": {
"paths": ["/srv/http"],
"password": "PASSWORD1", # OBVIOUSLY, change all these.
"excludes": ["/srv/http/sks/dumps"] # you can specify paths to exclude.
},
"etc": {
"paths": ["/etc"],
"password": "PASSWORD2"
},
"home": {
"paths": ["/home"],
"password": "PASSWORD3"
},
"misc": {
"paths": ["/usr/local/bin", "/var/lib/awstats", "/var/spool/anacron", "/var/spool/cron", "/usr/local/bin", "/root/.bak/misc"],
"prep": ["miscBak('apacman')"], # i'm... thinking of ways to make this more flexible, but for now it supports pacman/apacman
"password": "PASSWORD4"
},
"root": {
"paths": ["/root"],
"password": "PASSWORD5",
"excludes": ["/root/.cache", "/root/.bak", "*.sql*"]
}
},
"config": {
"user": "username", # the username on your backup box
"host": "backups.domain.tld", # where the borg server is
"ctx": "ssh -p 1234" # if you run a borg-specific SSHD on an alternate port
}
}

50
storage/backups/borg/sample.config.xml
View File

@ -2,13 +2,63 @@
<borg xmlns="http://git.square-r00t.net/OpTools/tree/storage/backups/borg/" <borg xmlns="http://git.square-r00t.net/OpTools/tree/storage/backups/borg/"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://git.square-r00t.net/OpTools/plain/storage/backups/borg/config.xsd"> xsi:schemaLocation="http://git.square-r00t.net/OpTools/plain/storage/backups/borg/config.xsd">
<!-- "target" = either the local filesystem (absolute) path or the remote host
"rsh" = (remote host only) the ssh command to use. The default is given below.
"user" = (remote host only) the ssh user to use. -->
<server target=" fq.dn.tld" rsh="ssh -p 22" user="root"> <server target=" fq.dn.tld" rsh="ssh -p 22" user="root">
<!-- "name" = the repositoriy name.
"password" = the repository's password for the key. If not specified, you will be prompted
to enter it interactively and securely. -->
<repo name="testrepo" password="SuperSecretPassword"> <repo name="testrepo" password="SuperSecretPassword">
<!-- Each path entry is a path to back up.
See https://borgbackup.readthedocs.io/en/stable/usage/create.html for examples of globbing, etc. -->
<path>/a</path> <path>/a</path>
<!-- Each exclude entry should be a subdirectory of a <path> (otherwise it wouldn't match, obviously).
See https://borgbackup.readthedocs.io/en/stable/usage/create.html for examples of globbing etc. -->
<exclude>/a/b</exclude> <exclude>/a/b</exclude>
<!-- Prep items are executed in non-guaranteed order (but are likely to be performed in order given).
If you require them to be in a specific order, you should use a wrapper script and
use that as a prep item. -->
<!-- "inline" = if true/1, the provided text will be temporarily written to disk, executed, and deleted.
if false/0, the provided text is assumed to be a single-shot command/path to a script. -->
<!-- If using inline especially, take note of and use XML escape characters:
" = &quot;
' = &apos;
< = &lt;
> = &gt;
& = &amp; -->
<prep inline="1">#!/bin/bash <prep inline="1">#!/bin/bash
# this is block text # this is block text
</prep> </prep>
<prep inline="0">/usr/local/bin/someprep.sh</prep>
<!-- Plugins are direct Python modules, and are alternatives to prep items.
They must:
- be in the Python's path environment (or a path must be provided, either absolute or relative to
execution)
- contain a class called <module>.Backup() (which will execute all tasks on initialization)
See plugins/ directory for examples and below for example of invocation. -->
<plugins>
<!-- Each plugin item MUST define a "name" attribute. This is the name of the module to import.
"path" = (optional) the directory containing the plugin module; it must end in .py -->
<plugin name="mysql" path="./plugins">
<!-- Param elements are optional. Each param element MUST define a "key" attribute; this is
the name of the parameter. (For positional attributes, this should match the name used
by the <module>.Backup().init() parameter name.)
If you want a parameter to be provided but with a None value, make it self-enclosed
(e.g. '<param key="someparam"/>').
If you need to serialize pythonic objects (lists, dicts, booleans),
then set the "json" attribute to 1/true and provide the data in minified JSON format. -->
<param key="dbs" json="true">["db1", "db2"]</param>
<param key="splitdumps" json="true">true</param>
<param key="dumpopts" json="true">["--routines", "--add-drop-database", "--add-drop-table", "--allow-keywords", "--complete-insert", "--create-options", "--extended-insert"]</param>
</plugin>
<plugin name="ldap" path="./plugins">
<param key="server">ldap://my.server.tld</param>
<param key="binddn">cn=Manager,dc=server,dc=tld</param>
<param key="password">SuperSecretPassword</param>
<param key="splitldifs" json="true">false</param>
</plugin>
</plugins>
</repo> </repo>
</server> </server>
</borg> </borg>