r00t2/optools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Activity

untested, but pretty sure it's done

Browse Source
This commit is contained in:
brent s 2019-05-24 13:37:40 -04:00
parent 130746fa00
commit 431b4e3425
7 changed files with 537 additions and 1129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

675
storage/backups/borg/backup.py
View File

@ -8,25 +8,20 @@
# TODO: modify config to add repo to cfg for init? or add new operation, "add" # TODO: modify config to add repo to cfg for init? or add new operation, "add"


import argparse import argparse
import configparser
import datetime import datetime
import json import json
import getpass import getpass
import logging import logging
import logging.handlers import logging.handlers
import os import os
import pwd
import re import re
# TODO: use borg module directly? # TODO: use borg module directly instead of subprocess?
import subprocess import subprocess
import sys import sys
import tempfile
# TODO: virtual env? # TODO: virtual env?
try: from lxml import etree # A lot safer and easier to use than the stdlib xml module.
from lxml import etree
has_lxml = True
except ImportError:
import xml.etree.ElementTree as etree # https://docs.python.org/3/library/xml.etree.elementtree.html
has_lxml = False

try: try:
import pymysql # not stdlib; "python-pymysql" in Arch's AUR import pymysql # not stdlib; "python-pymysql" in Arch's AUR
has_mysql = True has_mysql = True
@ -46,21 +41,21 @@ loglvls = {'critical': logging.CRITICAL,
'info': logging.INFO, 'info': logging.INFO,
'debug': logging.DEBUG} 'debug': logging.DEBUG}


### DEFAULT NAMESPACE ###
dflt_ns = 'http://git.square-r00t.net/OpTools/tree/storage/backups/borg/'



### THE GUTS ### ### THE GUTS ###
class Backup(object): class Backup(object):
def __init__(self, args): def __init__(self, args):
self.args = args self.args = args
### DIRECTORIES ### self.ns = '{{{0}}}'.format(dflt_ns)
if self.args['oper'] == 'backup':
for d in (self.args['mysqldir'], self.args['stagedir']):
os.makedirs(d, exist_ok = True, mode = 0o700)
if self.args['oper'] == 'restore': if self.args['oper'] == 'restore':
self.args['target_dir'] = os.path.abspath(os.path.expanduser( self.args['target_dir'] = os.path.abspath(os.path.expanduser(self.args['target_dir']))
self.args['target_dir'])) os.makedirs(self.args['target_dir'],
os.makedirs(os.path.dirname(self.args['oper']),
exist_ok = True, exist_ok = True,
mode = 0o700) mode = 0o700)
self.repos = {}
### LOGGING ### ### LOGGING ###
# Thanks to: # Thanks to:
# https://web.archive.org/web/20170726052946/http://www.lexev.org/en/2013/python-logging-every-day/ # https://web.archive.org/web/20170726052946/http://www.lexev.org/en/2013/python-logging-every-day/
@ -69,13 +64,10 @@ class Backup(object):
# and user K900_ on r/python for entertaining my very silly question. # and user K900_ on r/python for entertaining my very silly question.
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
self.logger.setLevel(loglvls[self.args['loglevel']]) self.logger.setLevel(loglvls[self.args['loglevel']])
_logfmt = logging.Formatter( _logfmt = logging.Formatter(fmt = ('{levelname}:{name}: {message} ({asctime}; {filename}:{lineno})'),
fmt = ('{levelname}:{name}: {message} ({asctime}; '
'{filename}:{lineno})'),
style = '{', style = '{',
datefmt = '%Y-%m-%d %H:%M:%S') datefmt = '%Y-%m-%d %H:%M:%S')
_journalfmt = logging.Formatter( _journalfmt = logging.Formatter(fmt = '{levelname}:{name}: {message} ({filename}:{lineno})',
fmt = '{levelname}:{name}: {message} ({filename}:{lineno})',
style = '{', style = '{',
datefmt = '%Y-%m-%d %H:%M:%S') datefmt = '%Y-%m-%d %H:%M:%S')
handlers = [] handlers = []
@ -84,8 +76,7 @@ class Backup(object):
exist_ok = True, exist_ok = True,
mode = 0o700) mode = 0o700)
# TODO: make the constraints for rotation in config? # TODO: make the constraints for rotation in config?
handlers.append( handlers.append(logging.handlers.RotatingFileHandler(self.args['logfile'],
logging.handlers.RotatingFileHandler(self.args['logfile'],
encoding = 'utf8', encoding = 'utf8',
maxBytes = 100000, maxBytes = 100000,
backupCount = 1)) backupCount = 1))
@ -100,156 +91,241 @@ class Backup(object):
h.setFormatter(_logfmt) h.setFormatter(_logfmt)
h.setLevel(loglvls[self.args['loglevel']]) h.setLevel(loglvls[self.args['loglevel']])
self.logger.addHandler(h) self.logger.addHandler(h)
### END LOGGING ###
self.logger.debug('BEGIN INITIALIZATION') self.logger.debug('BEGIN INITIALIZATION')
### CONFIG ### ### CONFIG ###
if not os.path.isfile(self.args['cfgfile']): if not os.path.isfile(self.args['cfgfile']):
self.logger.error( self.logger.error('{0} does not exist'.format(self.args['cfgfile']))
'{0} does not exist'.format(self.args['cfgfile']))
exit(1) exit(1)
with open(self.args['cfgfile'], 'r') as f: try:
self.cfg = json.loads(f.read()) with open(self.args['cfgfile'], 'rb') as f:
### END LOGGING ### self.cfg = etree.fromstring(f.read())
### ARGS CLEANUP ### except etree.XMLSyntaxError:
self.logger.debug('VARS (before args cleanup): {0}'.format(vars())) self.logger.error('{0} is invalid XML'.format(self.args['cfgfile']))
self.args['repo'] = [i.strip() for i in self.args['repo'].split(',')] raise ValueError(('{0} does not seem to be valid XML. '
if 'all' in self.args['repo']: 'See sample.config.xml for an example configuration.').format(self.args['cfgfile']))
self.args['repo'] = list(self.cfg['repos'].keys()) self.borgbin = self.cfg.attrib.get('borgpath', '/usr/bin/borg')
for r in self.args['repo'][:]:
if r == 'all':
self.args['repo'].remove(r)
elif r not in self.cfg['repos'].keys():
self.logger.warning(
'Repository {0} is not configured; skipping.'.format(
r))
self.args['repo'].remove(r)
self.logger.debug('VARS (after args cleanup): {0}'.format(vars()))
self.logger.debug('END INITIALIZATION')
### CHECK ENVIRONMENT ### ### CHECK ENVIRONMENT ###
# If we're running from cron, we want to print errors to stdout. # If we're running from cron, we want to print errors to stdout.
if os.isatty(sys.stdin.fileno()): if os.isatty(sys.stdin.fileno()):
self.cron = False self.cron = False
else: else:
self.cron = True self.cron = True
### END INIT ### self.logger.debug('END INITIALIZATION')
self.buildRepos()


def cmdExec(self, cmd, stdoutfh = None): def buildRepos(self):
self.logger.debug('Running command: {0}'.format(' '.join(cmd))) def getRepo(server, reponames = None):
if self.args['dryrun']: if not reponames:
return () # no-op reponames = []
if stdoutfh: repos = []
_cmd = subprocess.run(cmd, for repo in server.findall('{0}repo'.format(self.ns)):
stdout = stdoutfh, if reponames and repo.attrib['name'] not in reponames:
stderr = subprocess.PIPE) continue
r = {}
for a in repo.attrib:
r[a] = repo.attrib[a]
for e in ('path', 'exclude'):
r[e] = [i.text for i in repo.findall(self.ns + e)]
for prep in repo.findall('{0}prep'.format(self.ns)):
if 'prep' not in r:
r['prep'] = []
if prep.attrib.get('inline', 'true').lower()[0] in ('0', 'f'):
with open(os.path.abspath(os.path.expanduser(prep.text)), 'r') as f:
r['prep'].append(f.read())
else: else:
_cmd = subprocess.run(cmd, r['prep'].append(prep.text)
stdout = subprocess.PIPE, plugins = repo.find('{0}plugins'.format(self.ns))
stderr = subprocess.PIPE) if plugins is not None:
_out = _cmd.stdout.decode('utf-8').strip() r['plugins'] = {}
_err = _cmd.stderr.decode('utf-8').strip() for plugin in plugins.findall('{0}plugin'.format(self.ns)):
_returncode = _cmd.returncode pname = plugin.attrib['name']
if _returncode != 0: r['plugins'][pname] = {'path': plugin.attrib.get('path'),
self.logger.error('STDERR: ({1})\n{0}'.format(_err, ' '.join(cmd))) 'params': {}}
if _err != '' and self.cron: for param in plugin.findall('{0}param'.format(self.ns)):
self.logger.warning( paramname = param.attrib['key']
'Command {0} failed: {1}'.format(' '.join(cmd), _err)) if param.attrib.get('json', 'false').lower()[0] in ('1', 't'):
r['plugins'][pname]['params'][paramname] = json.loads(param.text)
else:
r['plugins'][pname]['params'][paramname] = param.text
repos.append(r)
return(repos)
self.logger.debug('VARS (before args cleanup): {0}'.format(vars(self)))
self.args['repo'] = [i.strip() for i in self.args['repo'].split(',')]
self.args['server'] = [i.strip() for i in self.args['server'].split(',')]
if 'all' in self.args['repo']:
self.args['repo'] = None
if 'all' in self.args['server']:
self.args['server'] = []
for server in self.cfg.findall('{0}server'.format(self.ns)):
# The server elements are uniquely constrained to the "target" attrib.
# *NO TWO <server> ELEMENTS WITH THE SAME target= SHOULD EXIST.*
self.args['server'].append(server.attrib['target'])
for server in self.cfg.findall('{0}server'.format(self.ns)):
sname = server.attrib['target']
if sname not in self.args['server']:
continue
self.repos[sname] = {}
for x in server.attrib:
if x != 'target':
self.repos[sname][x] = server.attrib[x]
self.repos[sname]['repos'] = getRepo(server, reponames = self.args['repo'])
self.logger.debug('VARS (after args cleanup): {0}'.format(vars(self)))
return() return()


def createRepo(self): def createRepo(self):
for server in self.repos:
_env = os.environ.copy() _env = os.environ.copy()
_env['BORG_RSH'] = self.cfg['config']['ctx'] # https://github.com/borgbackup/borg/issues/2273
for r in self.args['repo']: # https://borgbackup.readthedocs.io/en/stable/internals/frontends.html
self.logger.info('[{0}]: BEGIN INITIALIZATION'.format(r)) _env['LANG'] = 'en_US.UTF-8'
_cmd = ['borg', _env['LC_CTYPE'] = 'en_US.UTF-8'
if self.repos[server]['remote'].lower()[0] in ('1', 't'):
_env['BORG_RSH'] = self.repos[server]['rsh']
_user = self.repos[server].get('user', pwd.getpwuid(os.geteuid()).pw_name)
for repo in self.repos[server]['repos']:
self.logger.info('[{0}]: BEGIN INITIALIZATION'.format(repo['name']))
_loc_env = _env.copy()
if 'password' not in repo:
print('Password not supplied for {0}:{1}.'.format(server, repo['name']))
_loc_env['BORG_PASSPHRASE'] = getpass.getpass('Password (will NOT echo back): ')
else:
_loc_env['BORG_PASSPHRASE'] = repo['password']
_cmd = [self.borgbin,
'--log-json',
'--{0}'.format(self.args['loglevel']),
'init', 'init',
'-v', '-e', 'repokey']
'{0}@{1}:{2}'.format(self.cfg['config']['user'], if self.repos[server]['remote'].lower()[0] in ('1', 't'):
self.cfg['config']['host'], repo_tgt = '{0}@{1}'.format(_user, server)
r)] else:
_env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password'] repo_tgt = os.path.abspath(os.path.expanduser(server))
# We don't use self.cmdExec() here either because _cmd.append('{0}:{1}'.format(repo_tgt,
# again, custom env, etc. repo['name']))
self.logger.debug('VARS: {0}'.format(vars())) self.logger.debug('VARS: {0}'.format(vars(self)))
if not self.args['dryrun']: if not self.args['dryrun']:
_out = subprocess.run(_cmd, _out = subprocess.run(_cmd,
env = _env, env = _loc_env,
stdout = subprocess.PIPE, stdout = subprocess.PIPE,
stderr = subprocess.PIPE) stderr = subprocess.PIPE)
_stdout = _out.stdout.decode('utf-8').strip() _stdout = _out.stdout.decode('utf-8').strip()
_stderr = _out.stderr.decode('utf-8').strip() _stderr = _out.stderr.decode('utf-8').strip()
_returncode = _out.returncode _returncode = _out.returncode
self.logger.debug('[{0}]: (RESULT) {1}'.format(r, _stdout)) self.logger.debug('[{0}]: (RESULT) {1}'.format(repo['name'], _stdout))
# sigh. borg uses stderr for verbose output. # sigh. borg uses stderr for verbose output.
self.logger.debug('[{0}]: STDERR: ({2})\n{1}'.format(r, self.logger.debug('[{0}]: STDERR: ({2})\n{1}'.format(repo['name'],
_stderr, _stderr,
' '.join( ' '.join(_cmd)))
_cmd)))
if _returncode != 0: if _returncode != 0:
self.logger.error( self.logger.error(
'[{0}]: FAILED: {1}'.format(r, ' '.join(_cmd))) '[{0}]: FAILED: {1}'.format(repo['name'], ' '.join(_cmd)))
if _err != '' and self.cron and _returncode != 0: if _stderr != '' and self.cron and _returncode != 0:
self.logger.warning( self.logger.warning('Command {0} failed: {1}'.format(' '.join(_cmd),
'Command {0} failed: {1}'.format(' '.join(cmd), _stderr))
_err)) self.logger.info('[{0}]: END INITIALIZATION'.format(repo['name']))
del (_env['BORG_PASSPHRASE'])
self.logger.info('[{0}]: END INITIALIZATION'.format(r))
return() return()


def create(self): def create(self):
# TODO: support "--strip-components N"? # TODO: support "--strip-components N"?
_env = os.environ.copy()
_env['BORG_RSH'] = self.cfg['config']['ctx']
self.logger.info('START: backup') self.logger.info('START: backup')
for r in self.args['repo']: for server in self.repos:
self.logger.info('[{0}]: BEGIN BACKUP'.format(r)) _env = os.environ.copy()
if 'prep' in self.cfg['repos'][r].keys(): if self.repos[server]['remote'].lower()[0] in ('1', 't'):
for prep in self.cfg['repos'][r]['prep']: _env['BORG_RSH'] = self.repos[server].get('rsh', None)
self.logger.info( _env['LANG'] = 'en_US.UTF-8'
'[{0}]: Running prepfunc {1}'.format(r, prep)) _env['LC_CTYPE'] = 'en_US.UTF-8'
eval('self.{0}'.format( _user = self.repos[server].get('user', pwd.getpwuid(os.geteuid()).pw_name)
prep)) # I KNOW, IT'S TERRIBLE. so sue me. for repo in self.repos[server]['repos']:
self.logger.info( _loc_env = _env.copy()
'[{0}]: Finished prepfunc {1}'.format(r, prep)) if 'password' not in repo:
_cmd = ['borg', print('Password not supplied for {0}:{1}.'.format(server, repo['name']))
_loc_env['BORG_PASSPHRASE'] = getpass.getpass('Password (will NOT echo back): ')
else:
_loc_env['BORG_PASSPHRASE'] = repo['password']
self.logger.info('[{0}]: BEGIN BACKUP: {1}'.format(server, repo['name']))
if 'prep' in repo:
tmpdir = os.path.abspath(os.path.expanduser('~/.cache/.optools_backup'))
os.makedirs(tmpdir, exist_ok = True)
os.chmod(tmpdir, mode = 0o0700)
for idx, prep in enumerate(repo['prep']):
exec_tmp = tempfile.mkstemp(prefix = '_optools.backup.',
suffix = '._tmpexc',
text = True,
dir = tmpdir)[1]
os.chmod(exec_tmp, mode = 0o0700)
with open(exec_tmp, 'w') as f:
f.write(prep)
prep_out = subprocess.run([exec_tmp],
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
if prep_out.returncode != 0:
err = ('Prep job {0} ({1}) for server {2} (repo {3}) '
'returned non-zero').format(idx, exec_tmp, server, repo)
logging.warning(err)
logging.debug('STDOUT: {0}'.format(prep_out.stdout.decode('utf-8')))
logging.debug('STDERR: {0}'.format(prep_out.stderr.decode('utf-8')))
else:
os.remove(exec_tmp)
if 'plugins' in repo:
import importlib
_orig_path = sys.path
for plugin in repo['plugins']:
if repo['plugins'][plugin]['path']:
sys.path.insert(1, repo['plugins'][plugin]['path'] + sys.path)
optools_tmpmod = importlib.import_module(plugin, package = None)
if not repo['plugins'][plugin]['params']:
optools_tmpmod.Backup()
else:
optools_tmpmod.Backup(**repo['plugins'][plugin]['params'])
del(sys.modules[plugin])
del(optools_tmpmod)
sys.path = _orig_path
# This is where we actually do the thing.
_cmd = [self.borgbin,
'--log-json',
'--{0}'.format(self.args['loglevel']),
'create', 'create',
'-v', '--stats', '--stats']
'--compression', 'lzma,9'] if 'compression' in repo:
if 'excludes' in self.cfg['repos'][r].keys(): _cmd.extend(['--compression', repo['compression']])
for e in self.cfg['repos'][r]['excludes']: if 'exclude' in repo:
for e in repo['exclude']:
_cmd.extend(['--exclude', e]) _cmd.extend(['--exclude', e])
_cmd.append('{0}@{1}:{2}::{3}'.format(self.cfg['config']['user'], if self.repos[server]['remote'].lower()[0] in ('1', 't'):
self.cfg['config']['host'], repo_tgt = '{0}@{1}'.format(_user, server)
r, else:
repo_tgt = os.path.abspath(os.path.expanduser(server))
_cmd.append('{0}:{1}::{2}'.format(repo_tgt,
repo['name'],
self.args['archive'])) self.args['archive']))
for p in self.cfg['repos'][r]['paths']: for p in repo['path']:
_cmd.append(p) _cmd.append(p)
_env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password']
self.logger.debug('VARS: {0}'.format(vars())) self.logger.debug('VARS: {0}'.format(vars()))
# We don't use self.cmdExec() here because we want to explicitly # We don't use self.cmdExec() here because we want to explicitly
# pass the env and format the log line differently. # pass the env and format the log line differently.
self.logger.debug( self.logger.debug('[{0}]: Running command: {1}'.format(repo['name'],
'[{0}]: Running command: {1}'.format(r, ' '.join(_cmd))) ' '.join(_cmd)))
if not self.args['dryrun']: if not self.args['dryrun']:
_out = subprocess.run(_cmd, _out = subprocess.run(_cmd,
env = _env, env = _loc_env,
stdout = subprocess.PIPE, stdout = subprocess.PIPE,
stderr = subprocess.PIPE) stderr = subprocess.PIPE)
_stdout = _out.stdout.decode('utf-8').strip() _stdout = _out.stdout.decode('utf-8').strip()
_stderr = _out.stderr.decode('utf-8').strip() _stderr = _out.stderr.decode('utf-8').strip()
_returncode = _out.returncode _returncode = _out.returncode
self.logger.debug('[{0}]: (RESULT) {1}'.format(r, _stdout)) self.logger.debug('[{0}]: (RESULT) {1}'.format(repo['name'], _stdout))
self.logger.error('[{0}]: STDERR: ({2})\n{1}'.format(r, self.logger.debug('[{0}]: STDERR: ({2})\n{1}'.format(repo['name'],
_stderr, _stderr,
' '.join( ' '.join(
_cmd))) _cmd)))
if _returncode != 0: if _returncode != 0:
self.logger.error( self.logger.error(
'[{0}]: FAILED: {1}'.format(r, ' '.join(_cmd))) '[{0}]: FAILED: {1}'.format(repo['name'], ' '.join(_cmd)))
if _stderr != '' and self.cron and _returncode != 0: if _stderr != '' and self.cron and _returncode != 0:
self.logger.warning( self.logger.warning('Command {0} failed: {1}'.format(' '.join(_cmd),
'Command {0} failed: {1}'.format(' '.join(_cmd),
_stderr)) _stderr))
del (_env['BORG_PASSPHRASE']) del (_loc_env['BORG_PASSPHRASE'])
self.logger.info('[{0}]: END BACKUP'.format(r)) self.logger.info('[{0}]: END BACKUP'.format(repo['name']))
self.logger.info('END: backup') self.logger.info('END: backup')
return() return()


@ -257,164 +333,221 @@ class Backup(object):
# TODO: support "--strip-components N"? # TODO: support "--strip-components N"?
# TODO: support add'l args? # TODO: support add'l args?
# https://borgbackup.readthedocs.io/en/stable/usage/extract.html # https://borgbackup.readthedocs.io/en/stable/usage/extract.html
_env = os.environ.copy() orig_dir = os.getcwd()
_env['BORG_RSH'] = self.cfg['config']['ctx']
self.logger.info('START: restore') self.logger.info('START: restore')
for r in self.args['repo']: self.args['target_dir'] = os.path.abspath(os.path.expanduser(self.args['target_dir']))
self.logger.info('[{0}]: BEGIN RESTORE'.format(r)) os.makedirs(self.args['target_dir'], exist_ok = True)
_cmd = ['borg', os.chmod(self.args['target_dir'], mode = 0o0700)
'extract', for server in self.repos:
'-v'] _env = os.environ.copy()
# if 'excludes' in self.cfg['repos'][r].keys(): if self.repos[server]['remote'].lower()[0] in ('1', 't'):
# for e in self.cfg['repos'][r]['excludes']: _env['BORG_RSH'] = self.repos[server].get('rsh', None)
# _cmd.extend(['--exclude', e]) _env['LANG'] = 'en_US.UTF-8'
_cmd.append('{0}@{1}:{2}::{3}'.format(self.cfg['config']['user'], _env['LC_CTYPE'] = 'en_US.UTF-8'
self.cfg['config']['host'], _user = self.repos[server].get('user', pwd.getpwuid(os.geteuid()).pw_name)
r, server_dir = os.path.join(self.args['target_dir'], server)
for repo in self.repos[server]['repos']:
_loc_env = _env.copy()
if 'password' not in repo:
print('Password not supplied for {0}:{1}.'.format(server, repo['name']))
_loc_env['BORG_PASSPHRASE'] = getpass.getpass('Password (will NOT echo back): ')
else:
_loc_env['BORG_PASSPHRASE'] = repo['password']
if len(self.repos[server]) > 1:
dest_dir = os.path.join(server_dir, repo['name'])
else:
dest_dir = server_dir
os.makedirs(dest_dir, exist_ok = True)
os.chmod(dest_dir, mode = 0o0700)
os.chdir(dest_dir)
self.logger.info('[{0}]: BEGIN RESTORE'.format(repo['name']))
_cmd = [self.borgbin,
'--log-json',
'--{0}'.format(self.args['loglevel']),
'extract']
if self.repos[server]['remote'].lower()[0] in ('1', 't'):
repo_tgt = '{0}@{1}'.format(_user, server)
else:
repo_tgt = os.path.abspath(os.path.expanduser(server))
_cmd.append('{0}:{1}::{2}'.format(repo_tgt,
repo['name'],
self.args['archive'])) self.args['archive']))
_cmd.append(os.path.abspath(self.args['target_dir'])) if self.args['archive_path']:
# TODO: support specific path inside archive? _cmd.append(self.args['archive_path'])
# if so, append path(s) here. self.logger.debug('VARS: {0}'.format(vars(self)))
_env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password'] self.logger.debug('[{0}]: Running command: {1}'.format(repo['name'],
self.logger.debug('VARS: {0}'.format(vars())) ' '.join(_cmd)))
# We don't use self.cmdExec() here because we want to explicitly
# pass the env and format the log line differently.
self.logger.debug(
'[{0}]: Running command: {1}'.format(r, ' '.join(_cmd)))
if not self.args['dryrun']: if not self.args['dryrun']:
_out = subprocess.run(_cmd, _out = subprocess.run(_cmd,
env = _env, env = _loc_env,
stdout = subprocess.PIPE, stdout = subprocess.PIPE,
stderr = subprocess.PIPE) stderr = subprocess.PIPE)
_stdout = _out.stdout.decode('utf-8').strip() _stdout = _out.stdout.decode('utf-8').strip()
_stderr = _out.stderr.decode('utf-8').strip() _stderr = _out.stderr.decode('utf-8').strip()
_returncode = _out.returncode _returncode = _out.returncode
self.logger.debug('[{0}]: (RESULT) {1}'.format(r, _stdout)) self.logger.debug('[{0}]: (RESULT) {1}'.format(repo['name'], _stdout))
self.logger.error('[{0}]: STDERR: ({2})\n{1}'.format(r, self.logger.debug('[{0}]: STDERR: ({2})\n{1}'.format(repo['name'],
_stderr, _stderr,
' '.join( ' '.join(_cmd)))
_cmd)))
if _returncode != 0: if _returncode != 0:
self.logger.error( self.logger.error('[{0}]: FAILED: {1}'.format(repo['name'],
'[{0}]: FAILED: {1}'.format(r, ' '.join(_cmd))) ' '.join(_cmd)))
if _stderr != '' and self.cron and _returncode != 0: if _stderr != '' and self.cron and _returncode != 0:
self.logger.warning( self.logger.warning('Command {0} failed: {1}'.format(' '.join(_cmd),
'Command {0} failed: {1}'.format(' '.join(_cmd),
_stderr)) _stderr))
del (_env['BORG_PASSPHRASE']) self.logger.info('[{0}]: END RESTORE'.format(repo['name']))
self.logger.info('[{0}]: END RESTORE'.format(r)) os.chdir(orig_dir)
self.logger.info('END: restore') self.logger.info('END: restore')
return() return()


def listRepos(self): def listRepos(self):
def objPrinter(d, indent = 0):
for k, v in d.items():
if k == 'name':
continue
if k.lower() in ('password', 'path', 'exclude', 'prep', 'plugins', 'params', 'compression'):
keyname = k.title()
else:
keyname = k
if isinstance(v, list):
for i in v:
print('\033[1m{0}{1}:\033[0m {2}'.format(('\t' * indent),
keyname,
i))
elif isinstance(v, dict):
print('\033[1m{0}{1}:\033[0m'.format(('\t' * indent),
keyname))
objPrinter(v, indent = (indent + 1))
else:
print('\033[1m{0}{1}:\033[0m {2}'.format(('\t' * indent),
keyname,
v))
return()
print('\n\033[1mCurrently configured repositories are:\033[0m\n') print('\n\033[1mCurrently configured repositories are:\033[0m\n')
print('\t{0}\n'.format(', '.join(self.cfg['repos'].keys()))) for server in self.repos:
if self.args['verbose']: print('\033[1mTarget:\033[0m {0}'.format(server))
print('\033[1mDETAILS:\033[0m\n') print('\033[1mRepositories:\033[0m')
for r in self.args['repo']: for r in self.repos[server]['repos']:
print('\t\033[1m{0}:\033[0m\n\t\t\033[1mPath(s):\033[0m\t'.format( if not self.args['verbose']:
r.upper()), end = '') print('\t\t{0}'.format(r['name']))
for p in self.cfg['repos'][r]['paths']: else:
print(p, end = ' ') print('\t\t\033[1mName:\033[0m {0}'.format(r['name']))
if 'prep' in self.cfg['repos'][r].keys(): print('\033[1m\t\tDetails:\033[0m')
print('\n\t\t\033[1mPrep:\033[0m\t\t', end = '') objPrinter(r, indent = 3)
for p in self.cfg['repos'][r]['prep']: print()
print(p, end = ' ')
if 'excludes' in self.cfg['repos'][r].keys():
print('\n\t\t\033[1mExclude(s):\033[0m\t', end = '')
for p in self.cfg['repos'][r]['excludes']:
print(p, end = ' ')
print('\n')
return() return()


def printer(self): def printer(self):
# TODO: better alignment. https://stackoverflow.com/a/5676884 # TODO: better alignment. https://stackoverflow.com/a/5676884
_results = self.lister() _results = self.lister()
if not self.args['archive']: # It's a listing of archives timefmt = '%Y-%m-%dT%H:%M:%S.%f'
print('\033[1mREPO:\tSNAPSHOT:\t\tTIMESTAMP:\033[0m\n') if not self.args['archive']:
for r in _results.keys(): # It's a listing of archives
print(r, end = '') for server in _results:
for line in _results[r]: print('\033[1mTarget:\033[0m {0}'.format(server))
_snapshot = line.split() print('\033[1mRepositories:\033[0m')
print('\t{0}\t\t{1}'.format(_snapshot[0], # Normally this is a list everywhere else. For results, however, it's a dict.
' '.join(_snapshot[1:]))) for repo in _results[server]:
print('\t\033[1m{0}:\033[0m'.format(repo))
print('\t\t\033[1mSnapshot\t\tTimestamp\033[0m')
for archive in _results[server][repo]:
print('\t\t{0}\t\t{1}'.format(archive['name'],
datetime.datetime.strptime(archive['time'], timefmt)))
print() print()
else: # It's a listing inside an archive
if self.args['verbose']:
_fields = ['REPO:', 'PERMS:', 'OWNERSHIP:', 'SIZE:', 'TIMESTAMP:', 'PATH:']
for r in _results.keys():
print('\033[1m{0}\t{1}\033[0m'.format(_fields[0], r))
# https://docs.python.org/3/library/string.html#formatspec
print('{0[1]:<15}\t{0[2]:<15}\t{0[3]:<15}\t{0[4]:<24}\t{0[5]:<15}'.format(_fields))
for line in _results[r]:
_fline = line.split()
_perms = _fline[0]
_ownership = '{0}:{1}'.format(_fline[1], _fline[2])
_size = _fline[3]
_time = ' '.join(_fline[4:7])
_path = ' '.join(_fline[7:])
print('{0:<15}\t{1:<15}\t{2:<15}\t{3:<24}\t{4:<15}'.format(_perms,
_ownership,
_size,
_time,
_path))
else: else:
print('\033[1mREPO:\tPATH:\033[0m\n') # It's a listing inside an archive
for r in _results.keys(): if self.args['verbose']:
print(r, end = '') _archive_fields = ['Mode', 'Owner', 'Size', 'Timestamp', 'Path']
for line in _results[r]: for server in _results:
_fline = line.split() print('\033[1mTarget:\033[0m {0}'.format(server))
print('\t{0}'.format(' '.join(_fline[7:]))) print('\033[1mRepositories:\033[0m')
for repo in _results[server]:
print('\t\033[1m{0}:\033[0m'.format(repo))
print(('\t\t\033[1m'
'{0[0]:<10}\t'
'{0[1]:<10}\t'
'{0[2]:<10}\t'
'{0[3]:<19}\t'
'{0[4]}'
'\033[0m').format(_archive_fields))
for file in _results[server][repo]:
file['mtime'] = datetime.datetime.strptime(file['mtime'], timefmt)
print(('\t\t'
'{mode:<10}\t'
'{user:<10}\t'
'{size:<10}\t'
'{mtime}\t'
'{path}').format(**file))
else:
for server in _results:
print('\033[1mTarget:\033[0m {0}'.format(server))
print('\033[1mRepositories:\033[0m')
for repo in _results[server]:
print('\t\033[1m{0}:\033[0m'.format(repo))
for file in _results[server][repo]:
print(file['path'])
return() return()


def lister(self): def lister(self):
output = {} output = {}
_env = os.environ.copy()
self.logger.debug('START: lister') self.logger.debug('START: lister')
_env['BORG_RSH'] = self.cfg['config']['ctx'] for server in self.repos:
for r in self.args['repo']: output[server] = {}
if self.args['archive']: _env = os.environ.copy()
_cmd = ['borg', if self.repos[server]['remote'].lower()[0] in ('1', 't'):
'list', _env['BORG_RSH'] = self.repos[server].get('rsh', None)
'{0}@{1}:{2}::{3}'.format(self.cfg['config']['user'], _env['LANG'] = 'en_US.UTF-8'
self.cfg['config']['host'], _env['LC_CTYPE'] = 'en_US.UTF-8'
r, _user = self.repos[server].get('user', pwd.getpwuid(os.geteuid()).pw_name)
self.args['archive'])] for repo in self.repos[server]['repos']:
_loc_env = _env.copy()
if 'password' not in repo:
print('Password not supplied for {0}:{1}.'.format(server, repo['name']))
_loc_env['BORG_PASSPHRASE'] = getpass.getpass('Password (will NOT echo back): ')
else: else:
_cmd = ['borg', _loc_env['BORG_PASSPHRASE'] = repo['password']
if self.repos[server]['remote'].lower()[0] in ('1', 't'):
repo_tgt = '{0}@{1}'.format(_user, server)
else:
repo_tgt = os.path.abspath(os.path.expanduser(server))
_cmd = [self.borgbin,
'--log-json',
'--{0}'.format(self.args['loglevel']),
'list', 'list',
'{0}@{1}:{2}'.format(self.cfg['config']['user'], ('--json-lines' if self.args['archive'] else '--json')]
self.cfg['config']['host'], _cmd.append('{0}:{1}{2}'.format(repo_tgt,
r)] repo['name'],
_env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password'] ('::{0}'.format(self.args['archive']) if self.args['archive']
else '')))
if not self.args['dryrun']: if not self.args['dryrun']:

_out = subprocess.run(_cmd, _out = subprocess.run(_cmd,
env = _env, env = _loc_env,
stdout = subprocess.PIPE, stdout = subprocess.PIPE,
stderr = subprocess.PIPE) stderr = subprocess.PIPE)
_stdout = [i.strip() for i in _out.stdout.decode('utf-8').splitlines()] _stdout = [i.strip() for i in _out.stdout.decode('utf-8').splitlines()]
_stderr = _out.stderr.decode('utf-8').strip() _stderr = _out.stderr.decode('utf-8').strip()
_returncode = _out.returncode _returncode = _out.returncode
output[r] = _stdout if self.args['archive']:
self.logger.debug('[{0}]: (RESULT) {1}'.format(r, output[server][repo['name']] = [json.loads(i) for i in _stdout.splitlines()]
else:
output[repo['name']] = json.loads(_stdout)['archives']
self.logger.debug('[{0}]: (RESULT) {1}'.format(repo['name'],
'\n'.join(_stdout))) '\n'.join(_stdout)))
if _returncode != 0: self.logger.debug('[{0}]: STDERR: ({2}) ({1})'.format(repo['name'],
self.logger.error('[{0}]: STDERR: ({2}) ({1})'.format(r,
_stderr, _stderr,
' '.join(_cmd))) ' '.join(_cmd)))
if _stderr != '' and self.cron and _returncode != 0: if _stderr != '' and self.cron and _returncode != 0:
self.logger.warning( self.logger.warning('Command {0} failed: {1}'.format(' '.join(_cmd),
'Command {0} failed: {1}'.format(' '.join(cmd), _err)) _stderr))
del(_env['BORG_PASSPHRASE'])
if not self.args['archive']: if not self.args['archive']:
if self.args['numlimit'] > 0: if self.args['numlimit'] > 0:
if self.args['old']: if self.args['old']:
output[r] = output[r][:self.args['numlimit']] output[server][repo['name']] = output[server][repo['name']][:self.args['numlimit']]
else: else:
output[r] = list(reversed(output[r]))[:self.args['numlimit']] output[server][repo['name']] = list(reversed(
output[server][repo['name']]))[:self.args['numlimit']]
if self.args['invert']: if self.args['invert']:
output[r] = reversed(output[r]) output[server][repo['name']] = reversed(output[server][repo['name']])
self.logger.debug('END: lister') self.logger.debug('END: lister')
return(output) return(output)


@ -442,14 +575,6 @@ def parseArgs():
### DEFAULTS ### ### DEFAULTS ###
_date = datetime.datetime.now().strftime("%Y_%m_%d.%H_%M") _date = datetime.datetime.now().strftime("%Y_%m_%d.%H_%M")
_logfile = '/var/log/borg/{0}'.format(_date) _logfile = '/var/log/borg/{0}'.format(_date)
_mysqldir = os.path.abspath(
os.path.join(os.path.expanduser('~'),
'.bak',
'mysql'))
_stagedir = os.path.abspath(
os.path.join(os.path.expanduser('~'),
'.bak',
'misc'))
_cfgfile = os.path.abspath( _cfgfile = os.path.abspath(
os.path.join(os.path.expanduser('~'), os.path.join(os.path.expanduser('~'),
'.config', '.config',
@ -499,6 +624,12 @@ def parseArgs():
help = ('The repository to perform the operation for. ' help = ('The repository to perform the operation for. '
'The default is \033[1mall\033[0m, a special value that specifies all known ' 'The default is \033[1mall\033[0m, a special value that specifies all known '
'repositories. Can also accept a comma-separated list.')) 'repositories. Can also accept a comma-separated list.'))
commonargs.add_argument('-S', '--server',
dest = 'server',
default = 'all',
help = ('The server to perform the operation for. '
'The default is \033[1mall\033[0m, a special value that specifies all known '
'servers. Can also accept a comma-separated list.'))
fileargs = argparse.ArgumentParser(add_help = False) fileargs = argparse.ArgumentParser(add_help = False)
fileargs.add_argument('-a', '--archive', fileargs.add_argument('-a', '--archive',
default = _date, default = _date,
@ -537,16 +668,6 @@ def parseArgs():
help = ('Convert the legacy JSON format to the new XML format and quit')) help = ('Convert the legacy JSON format to the new XML format and quit'))
### OPERATION-SPECIFIC OPTIONS ### ### OPERATION-SPECIFIC OPTIONS ###
# CREATE ("backup") # # CREATE ("backup") #
backupargs.add_argument('-s', '--stagedir',
default = _stagedir,
dest = 'stagedir',
help = ('The directory used for staging temporary files, if necessary. '
'Default: \033[1m{0}\033[0m').format(_stagedir))
backupargs.add_argument('-m', '--mysqldir',
default = _mysqldir,
dest = 'mysqldir',
help = ('The path to where MySQL dumps should go. '
'Default: \033[1m{0}\033[0m').format(_mysqldir))
# DISPLAY/OUTPUT ("list") # # DISPLAY/OUTPUT ("list") #
listargs.add_argument('-a', '--archive', listargs.add_argument('-a', '--archive',
dest = 'archive', dest = 'archive',
@ -579,22 +700,29 @@ def parseArgs():
help = ('Print extended information about how to ' help = ('Print extended information about how to '
'manage the output of listing and exit.')) 'manage the output of listing and exit.'))
## EXTRACT ("restore") ## EXTRACT ("restore")
rstrargs.add_argument('-p', '--path',
dest = 'archive_path',
help = ('If specified, only restore this specific path (and any subpaths).'))
rstrargs.add_argument('-t', '--target', rstrargs.add_argument('-t', '--target',
required = True, required = True,
dest = 'target_dir', dest = 'target_dir',
help = ('The path to the directory where the restore should be dumped to. It is ' help = ('The path to the directory where the restore should be dumped to. It is '
'recommended to NOT restore to the same directory that the archive is taken from.')) 'recommended to not restore to the same directory that the archive is taken from. '
'A subdirectory will be created for each server.'
'If multiple repos (or "all") are provided, subdirectories will be created per '
'repo under their respective server(s).'))
return (args) return (args)


def convertConf(cfgfile): def convertConf(cfgfile):
oldcfgfile = re.sub('\.xml$', '.json', cfgfile)
try: try:
with open(cfgfile, 'r') as f: with open(oldcfgfile, 'r') as f:
oldcfg = json.load(f) oldcfg = json.load(f)
except json.decoder.JSONDecodeError: except json.decoder.JSONDecodeError:
# It's not JSON. It's either already XML or invalid config. # It's not JSON. It's either already XML or invalid config.
return(cfgfile) return(cfgfile)
# Switched from JSON to XML, so we need to do some basic conversion. # Switched from JSON to XML, so we need to do some basic conversion.
newfname = re.sub('(\.json)?$', '.xml', os.path.basename(cfgfile)) newfname = re.sub('\.json$', '.xml', os.path.basename(cfgfile))
newcfg = os.path.join(os.path.dirname(cfgfile), newcfg = os.path.join(os.path.dirname(cfgfile),
newfname) newfname)
if os.path.exists(newcfg): if os.path.exists(newcfg):
@ -609,61 +737,45 @@ def convertConf(cfgfile):
# The old format only supported one server. # The old format only supported one server.
server = etree.Element('server') server = etree.Element('server')
server.attrib['target'] = oldcfg['config']['host'] server.attrib['target'] = oldcfg['config']['host']
server.attrib['remote'] = 'true'
server.attrib['rsh'] = oldcfg['config']['ctx'] server.attrib['rsh'] = oldcfg['config']['ctx']
server.attrib['user'] = oldcfg['config']['user'] server.attrib['user'] = oldcfg['config'].get('user', pwd.getpwnam(os.geteuid()).pw_name)
for r in oldcfg['repos']: for r in oldcfg['repos']:
repo = etree.Element('repo') repo = etree.Element('repo')
repo.attrib['name'] = r repo.attrib['name'] = r
repo.attrib['password'] = oldcfg['repos'][r]['password'] repo.attrib['password'] = oldcfg['repos'][r]['password']
for p in oldcfg['repos'][r]['paths']: for p in oldcfg['repos'][r]['paths']:
path = etree.Element('path') path = etree.Element('path')
path.text = p
repo.append(path)
for e in oldcfg['repos'][r].get('excludes', []):
path = etree.Element('exclude')
path.text = e
repo.append(path)
server.append(repo) server.append(repo)
cfg.append(server)
# Build the full XML spec. # Build the full XML spec.
namespaces = {'borg': 'http://git.square-r00t.net/OpTools/tree/storage/backups/borg/', namespaces = {None: dflt_ns,
'xsi': 'http://www.w3.org/2001/XMLSchema-instance'} 'xsi': 'http://www.w3.org/2001/XMLSchema-instance'}
xsi = {('{http://www.w3.org/2001/' xsi = {('{http://www.w3.org/2001/'
'XMLSchema-instance}schemaLocation'): ('http://git.square-r00t.net/OpTools/plain/' 'XMLSchema-instance}schemaLocation'): ('http://git.square-r00t.net/OpTools/plain/'
'storage/backups/borg/config.xsd')} 'storage/backups/borg/config.xsd')}
if has_lxml:
genname = 'LXML (http://lxml.de/)' genname = 'LXML (http://lxml.de/)'
root = etree.Element('borg', nsmap = namespaces, attrib = xsi) root = etree.Element('borg', nsmap = namespaces, attrib = xsi)
else: root.append(etree.Comment(('Generated by {0} on {1} from {2} via {3}').format(sys.argv[0],
genname = 'Python stdlib "xml" module'
for ns in namespaces.keys():
etree.register_namespace(ns, namespaces[ns])
root = etree.Element('borg')
fromstr = cfgfile
root.append(etree.Comment(
('Generated by {0} on {1} from {2} via {3}').format(sys.argv[0],
datetime.datetime.now(), datetime.datetime.now(),
fromstr, oldcfgfile,
genname))) genname)))
root.append(etree.Comment('THIS FILE CONTAINS SENSITIVE INFORMATION. SHARE/SCRUB WISELY.')) root.append(etree.Comment('THIS FILE CONTAINS SENSITIVE INFORMATION. SHARE/SCRUB WISELY.'))
for x in cfg: for x in cfg:
root.append(x) root.append(x)
# Write out the file to disk. # Write out the file to disk.
if has_lxml:
xml = etree.ElementTree(root) xml = etree.ElementTree(root)
with open(newcfg, 'wb') as f: with open(newcfg, 'wb') as f:
xml.write(f, xml.write(f,
xml_declaration = True, xml_declaration = True,
encoding = 'utf-8', encoding = 'utf-8',
pretty_print = True) pretty_print = True)
else:
import xml.dom.minidom
xmlstr = etree.tostring(root, encoding = 'utf-8')
# holy cats, the xml module sucks.
nsstr = ''
for ns in namespaces.keys():
nsstr += ' xmlns:{0}="{1}"'.format(ns, namespaces[ns])
for x in xsi.keys():
xsiname = x.split('}')[1]
nsstr += ' xsi:{0}="{1}"'.format(xsiname, xsi[x])
outstr = xml.dom.minidom.parseString(xmlstr).toprettyxml(indent = ' ').splitlines()
outstr[0] = '<?xml version=\'1.0\' encoding=\'utf-8\'?>'
outstr[1] = '<borg{0}>'.format(nsstr)
with open(newcfg, 'w') as f:
f.write('\n'.join(outstr))
# Return the new config's path. # Return the new config's path.
return(newcfg) return(newcfg)


@ -682,13 +794,18 @@ def main():
convertConf(args['cfgfile']) convertConf(args['cfgfile'])
return() return()
else: else:
if not os.path.isfile(args['cfgfile']):
oldfile = re.sub('\.xml$', '.json', args['cfgfile'])
if os.path.isfile(oldfile):
try: try:
with open(args['cfgfile'], 'r') as f: with open(oldfile, 'r') as f:
json.load(f) json.load(f)
args['cfgfile'] = convertConf(args['cfgfile']) args['cfgfile'] = convertConf(args['cfgfile'])
except json.decoder.JSONDecodeError: except json.decoder.JSONDecodeError:
# It's not JSON. It's either already XML or invalid config. # It's not JSON. It's either already XML or invalid config.
pass pass
if not os.path.isfile(args['cfgfile']):
raise OSError('{0} does not exist'.format(args['cfgfile']))
# The "Do stuff" part # The "Do stuff" part
bak = Backup(args) bak = Backup(args)
if args['oper'] == 'list': if args['oper'] == 'list':

775
storage/backups/borg/backup.py.old
View File

@ -1,775 +0,0 @@
#!/usr/bin/env python3

# TODO: https://borgbackup.readthedocs.io/en/latest/internals/frontends.html
# will they EVER release a public API? for now we'll just use subprocess since
# we import it for various prep stuff anyways.
# TODO: change loglevel of borg itself in subprocess to match the argparse?
# --debug, --info (same as -v/--verbose), --warning, --error, --critical
# TODO: modify config to add repo to cfg for init? or add new operation, "add"

import argparse
import configparser
import datetime
import json
import getpass
import logging
import logging.handlers
import os
import re
import subprocess
import sys
# TODO: virtual env?
try:
from lxml import etree
has_lxml = True
except ImportError:
import xml.etree.ElementTree as etree # https://docs.python.org/3/library/xml.etree.elementtree.html
has_lxml = False

try:
import pymysql # not stdlib; "python-pymysql" in Arch's AUR
has_mysql = True
except ImportError:
has_mysql = False
try:
# https://www.freedesktop.org/software/systemd/python-systemd/journal.html#journalhandler-class
from systemd import journal
has_systemd = True
except ImportError:
has_systemd = False

### LOG LEVEL MAPPINGS ###
loglvls = {
'critical': logging.CRITICAL,
'error': logging.ERROR,
'warning': logging.WARNING,
'info': logging.INFO,
'debug': logging.DEBUG}


### THE GUTS ###
class Backup(object):
def __init__(self, args):
self.args = args
### DIRECTORIES ###
if self.args['oper'] == 'backup':
for d in (self.args['mysqldir'], self.args['stagedir']):
os.makedirs(d, exist_ok = True, mode = 0o700)
if self.args['oper'] == 'restore':
self.args['target_dir'] = os.path.abspath(os.path.expanduser(
self.args['target_dir']))
os.makedirs(os.path.dirname(self.args['oper']),
exist_ok = True,
mode = 0o700)
### LOGGING ###
# Thanks to:
# https://web.archive.org/web/20170726052946/http://www.lexev.org/en/2013/python-logging-every-day/
# https://stackoverflow.com/a/42604392
# https://plumberjack.blogspot.com/2010/10/supporting-alternative-formatting.html
# and user K900_ on r/python for entertaining my very silly question.
self.logger = logging.getLogger(__name__)
self.logger.setLevel(loglvls[self.args['loglevel']])
_logfmt = logging.Formatter(
fmt = ('{levelname}:{name}: {message} ({asctime}; '
'{filename}:{lineno})'),
style = '{',
datefmt = '%Y-%m-%d %H:%M:%S')
_journalfmt = logging.Formatter(
fmt = '{levelname}:{name}: {message} ({filename}:{lineno})',
style = '{',
datefmt = '%Y-%m-%d %H:%M:%S')
handlers = []
if self.args['disklog']:
os.makedirs(os.path.dirname(self.args['logfile']),
exist_ok = True,
mode = 0o700)
# TODO: make the constraints for rotation in config?
handlers.append(
logging.handlers.RotatingFileHandler(self.args['logfile'],
encoding = 'utf8',
maxBytes = 100000,
backupCount = 1))
if self.args['verbose']:
handlers.append(logging.StreamHandler())
if has_systemd:
h = journal.JournalHandler()
h.setFormatter(_journalfmt)
h.setLevel(loglvls[self.args['loglevel']])
self.logger.addHandler(h)
for h in handlers:
h.setFormatter(_logfmt)
h.setLevel(loglvls[self.args['loglevel']])
self.logger.addHandler(h)
self.logger.debug('BEGIN INITIALIZATION')
### CONFIG ###
if not os.path.isfile(self.args['cfgfile']):
self.logger.error(
'{0} does not exist'.format(self.args['cfgfile']))
exit(1)
with open(self.args['cfgfile'], 'r') as f:
self.cfg = json.loads(f.read())
### END LOGGING ###
### ARGS CLEANUP ###
self.logger.debug('VARS (before args cleanup): {0}'.format(vars()))
self.args['repo'] = [i.strip() for i in self.args['repo'].split(',')]
if 'all' in self.args['repo']:
self.args['repo'] = list(self.cfg['repos'].keys())
for r in self.args['repo'][:]:
if r == 'all':
self.args['repo'].remove(r)
elif r not in self.cfg['repos'].keys():
self.logger.warning(
'Repository {0} is not configured; skipping.'.format(
r))
self.args['repo'].remove(r)
self.logger.debug('VARS (after args cleanup): {0}'.format(vars()))
self.logger.debug('END INITIALIZATION')
### CHECK ENVIRONMENT ###
# If we're running from cron, we want to print errors to stdout.
if os.isatty(sys.stdin.fileno()):
self.cron = False
else:
self.cron = True
### END INIT ###

def cmdExec(self, cmd, stdoutfh = None):
self.logger.debug('Running command: {0}'.format(' '.join(cmd)))
if self.args['dryrun']:
return () # no-op
if stdoutfh:
_cmd = subprocess.run(cmd, stdout = stdoutfh,
stderr = subprocess.PIPE)
else:
_cmd = subprocess.run(cmd,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
_out = _cmd.stdout.decode('utf-8').strip()
_err = _cmd.stderr.decode('utf-8').strip()
_returncode = _cmd.returncode
if _returncode != 0:
self.logger.error('STDERR: ({1})\n{0}'.format(_err, ' '.join(cmd)))
if _err != '' and self.cron:
self.logger.warning(
'Command {0} failed: {1}'.format(' '.join(cmd), _err))
return()

def createRepo(self):
_env = os.environ.copy()
_env['BORG_RSH'] = self.cfg['config']['ctx']
for r in self.args['repo']:
self.logger.info('[{0}]: BEGIN INITIALIZATION'.format(r))
_cmd = ['borg',
'init',
'-v',
'{0}@{1}:{2}'.format(self.cfg['config']['user'],
self.cfg['config']['host'],
r)]
_env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password']
# We don't use self.cmdExec() here either because
# again, custom env, etc.
self.logger.debug('VARS: {0}'.format(vars()))
if not self.args['dryrun']:
_out = subprocess.run(_cmd,
env = _env,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
_stdout = _out.stdout.decode('utf-8').strip()
_stderr = _out.stderr.decode('utf-8').strip()
_returncode = _out.returncode
self.logger.debug('[{0}]: (RESULT) {1}'.format(r, _stdout))
# sigh. borg uses stderr for verbose output.
self.logger.debug('[{0}]: STDERR: ({2})\n{1}'.format(r,
_stderr,
' '.join(
_cmd)))
if _returncode != 0:
self.logger.error(
'[{0}]: FAILED: {1}'.format(r, ' '.join(_cmd)))
if _err != '' and self.cron and _returncode != 0:
self.logger.warning(
'Command {0} failed: {1}'.format(' '.join(cmd),
_err))
del (_env['BORG_PASSPHRASE'])
self.logger.info('[{0}]: END INITIALIZATION'.format(r))
return()

def create(self):
# TODO: support "--strip-components N"?
_env = os.environ.copy()
_env['BORG_RSH'] = self.cfg['config']['ctx']
self.logger.info('START: backup')
for r in self.args['repo']:
self.logger.info('[{0}]: BEGIN BACKUP'.format(r))
if 'prep' in self.cfg['repos'][r].keys():
for prep in self.cfg['repos'][r]['prep']:
self.logger.info(
'[{0}]: Running prepfunc {1}'.format(r, prep))
eval('self.{0}'.format(
prep)) # I KNOW, IT'S TERRIBLE. so sue me.
self.logger.info(
'[{0}]: Finished prepfunc {1}'.format(r, prep))
_cmd = ['borg',
'create',
'-v', '--stats',
'--compression', 'lzma,9']
if 'excludes' in self.cfg['repos'][r].keys():
for e in self.cfg['repos'][r]['excludes']:
_cmd.extend(['--exclude', e])
_cmd.append('{0}@{1}:{2}::{3}'.format(self.cfg['config']['user'],
self.cfg['config']['host'],
r,
self.args['archive']))
for p in self.cfg['repos'][r]['paths']:
_cmd.append(p)
_env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password']
self.logger.debug('VARS: {0}'.format(vars()))
# We don't use self.cmdExec() here because we want to explicitly
# pass the env and format the log line differently.
self.logger.debug(
'[{0}]: Running command: {1}'.format(r, ' '.join(_cmd)))
if not self.args['dryrun']:
_out = subprocess.run(_cmd,
env = _env,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
_stdout = _out.stdout.decode('utf-8').strip()
_stderr = _out.stderr.decode('utf-8').strip()
_returncode = _out.returncode
self.logger.debug('[{0}]: (RESULT) {1}'.format(r, _stdout))
self.logger.error('[{0}]: STDERR: ({2})\n{1}'.format(r,
_stderr,
' '.join(
_cmd)))
if _returncode != 0:
self.logger.error(
'[{0}]: FAILED: {1}'.format(r, ' '.join(_cmd)))
if _stderr != '' and self.cron and _returncode != 0:
self.logger.warning(
'Command {0} failed: {1}'.format(' '.join(_cmd),
_stderr))
del (_env['BORG_PASSPHRASE'])
self.logger.info('[{0}]: END BACKUP'.format(r))
self.logger.info('END: backup')
return()

def restore(self):
# TODO: support "--strip-components N"?
# TODO: support add'l args?
# https://borgbackup.readthedocs.io/en/stable/usage/extract.html
_env = os.environ.copy()
_env['BORG_RSH'] = self.cfg['config']['ctx']
self.logger.info('START: restore')
for r in self.args['repo']:
self.logger.info('[{0}]: BEGIN RESTORE'.format(r))
_cmd = ['borg',
'extract',
'-v']
# if 'excludes' in self.cfg['repos'][r].keys():
# for e in self.cfg['repos'][r]['excludes']:
# _cmd.extend(['--exclude', e])
_cmd.append('{0}@{1}:{2}::{3}'.format(self.cfg['config']['user'],
self.cfg['config']['host'],
r,
self.args['archive']))
_cmd.append(os.path.abspath(self.args['target_dir']))
# TODO: support specific path inside archive?
# if so, append path(s) here.
_env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password']
self.logger.debug('VARS: {0}'.format(vars()))
# We don't use self.cmdExec() here because we want to explicitly
# pass the env and format the log line differently.
self.logger.debug(
'[{0}]: Running command: {1}'.format(r, ' '.join(_cmd)))
if not self.args['dryrun']:
_out = subprocess.run(_cmd,
env = _env,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
_stdout = _out.stdout.decode('utf-8').strip()
_stderr = _out.stderr.decode('utf-8').strip()
_returncode = _out.returncode
self.logger.debug('[{0}]: (RESULT) {1}'.format(r, _stdout))
self.logger.error('[{0}]: STDERR: ({2})\n{1}'.format(r,
_stderr,
' '.join(
_cmd)))
if _returncode != 0:
self.logger.error(
'[{0}]: FAILED: {1}'.format(r, ' '.join(_cmd)))
if _stderr != '' and self.cron and _returncode != 0:
self.logger.warning(
'Command {0} failed: {1}'.format(' '.join(_cmd),
_stderr))
del (_env['BORG_PASSPHRASE'])
self.logger.info('[{0}]: END RESTORE'.format(r))
self.logger.info('END: restore')
return()

def miscBak(self, pkgr):
self.logger.info('BEGIN: miscBak()')
_cmd = None
for p in os.environ['PATH'].split(':'):
d = os.path.expanduser(p)
if os.path.isfile(os.path.join(d, pkgr)):
_pkgr = pkgr
self.logger.debug('Package tool found at {0}'.format(_pkgr))
else:
_pkgr = 'pacman'
self.logger.debug('Using {0} as package tool'.format(_pkgr))
with open(os.path.join(self.args['stagedir'], 'pkg.lst'), 'w') as f:
_cmd = [_pkgr,
'-Qet',
'--color',
'never']
self.cmdExec(_cmd, stdoutfh = f)
self.logger.info('END: miscBak()')
return()

def mysqlBak(self):
self.logger.info('BEGIN: mysqlBak()')
if not has_mysql:
self.logger.error(
'You need to install the PyMySQL module to back up MySQL databases. Skipping.')
return ()
# These are mysqldump options shared by ALL databases
_mysqlopts = ['--routines',
'--add-drop-database',
'--add-drop-table',
'--allow-keywords',
'--complete-insert',
'--create-options',
'--extended-insert']
_DBs = []
_mycnf = os.path.expanduser(os.path.join('~', '.my.cnf'))
if not os.path.isfile(_mycnf):
exit(
'{0}: ERROR: Cannot get credentials for MySQL (cannot find ~/.my.cnf)!')
_mycfg = configparser.ConfigParser()
_mycfg._interpolation = configparser.ExtendedInterpolation()
_mycfg.read(_mycnf)
_sqlcfg = {s: dict(_mycfg.items(s)) for s in _mycfg.sections()}
if 'host' not in _sqlcfg.keys():
_socketpath = '/var/run/mysqld/mysqld.sock' # correct for Arch, YMMV.
_mysql = pymysql.connect(unix_socket = _socketpath,
user = _sqlcfg['client']['user'],
passwd = _sqlcfg['client']['password'])
else:
_mysql = pymysql.connect(host = _sqlcfg['client']['host'],
user = _sqlcfg['client']['user'],
port = _sqlcfg['client']['port'],
passwd = _sqlcfg['client']['password'])
_cur = _mysql.cursor()
_cur.execute('SHOW DATABASES')
for row in _cur.fetchall():
_DBs.append(row[0])
self.logger.debug('Databases: {0}'.format(', '.join(_DBs)))
for db in _DBs:
_cmd = ['mysqldump',
'--result-file={0}.sql'.format(
os.path.join(self.args['mysqldir'], db))]
# These are database-specific options
if db in ('information_schema', 'performance_schema'):
_cmd.append('--skip-lock-tables')
elif db == 'mysql':
_cmd.append('--flush-privileges')
_cmd.extend(_mysqlopts)
_cmd.append(db)
self.cmdExec(_cmd)
self.logger.info('END: mysqlBak()')
return()

def listRepos(self):
print('\n\033[1mCurrently configured repositories are:\033[0m\n')
print('\t{0}\n'.format(', '.join(self.cfg['repos'].keys())))
if self.args['verbose']:
print('\033[1mDETAILS:\033[0m\n')
for r in self.args['repo']:
print('\t\033[1m{0}:\033[0m\n\t\t\033[1mPath(s):\033[0m\t'.format(
r.upper()), end = '')
for p in self.cfg['repos'][r]['paths']:
print(p, end = ' ')
if 'prep' in self.cfg['repos'][r].keys():
print('\n\t\t\033[1mPrep:\033[0m\t\t', end = '')
for p in self.cfg['repos'][r]['prep']:
print(p, end = ' ')
if 'excludes' in self.cfg['repos'][r].keys():
print('\n\t\t\033[1mExclude(s):\033[0m\t', end = '')
for p in self.cfg['repos'][r]['excludes']:
print(p, end = ' ')
print('\n')
return()

def printer(self):
# TODO: better alignment. https://stackoverflow.com/a/5676884
_results = self.lister()
if not self.args['archive']: # It's a listing of archives
print('\033[1mREPO:\tSNAPSHOT:\t\tTIMESTAMP:\033[0m\n')
for r in _results.keys():
print(r, end = '')
for line in _results[r]:
_snapshot = line.split()
print('\t{0}\t\t{1}'.format(_snapshot[0],
' '.join(_snapshot[1:])))
print()
else: # It's a listing inside an archive
if self.args['verbose']:
_fields = ['REPO:', 'PERMS:', 'OWNERSHIP:', 'SIZE:', 'TIMESTAMP:', 'PATH:']
for r in _results.keys():
print('\033[1m{0}\t{1}\033[0m'.format(_fields[0], r))
# https://docs.python.org/3/library/string.html#formatspec
print('{0[1]:<15}\t{0[2]:<15}\t{0[3]:<15}\t{0[4]:<24}\t{0[5]:<15}'.format(_fields))
for line in _results[r]:
_fline = line.split()
_perms = _fline[0]
_ownership = '{0}:{1}'.format(_fline[1], _fline[2])
_size = _fline[3]
_time = ' '.join(_fline[4:7])
_path = ' '.join(_fline[7:])
print('{0:<15}\t{1:<15}\t{2:<15}\t{3:<24}\t{4:<15}'.format(_perms,
_ownership,
_size,
_time,
_path))
else:
print('\033[1mREPO:\tPATH:\033[0m\n')
for r in _results.keys():
print(r, end = '')
for line in _results[r]:
_fline = line.split()
print('\t{0}'.format(' '.join(_fline[7:])))
return()

def lister(self):
output = {}
_env = os.environ.copy()
self.logger.debug('START: lister')
_env['BORG_RSH'] = self.cfg['config']['ctx']
for r in self.args['repo']:
if self.args['archive']:
_cmd = ['borg',
'list',
'{0}@{1}:{2}::{3}'.format(self.cfg['config']['user'],
self.cfg['config']['host'],
r,
self.args['archive'])]
else:
_cmd = ['borg',
'list',
'{0}@{1}:{2}'.format(self.cfg['config']['user'],
self.cfg['config']['host'],
r)]
_env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password']
if not self.args['dryrun']:

_out = subprocess.run(_cmd,
env = _env,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
_stdout = [i.strip() for i in _out.stdout.decode('utf-8').splitlines()]
_stderr = _out.stderr.decode('utf-8').strip()
_returncode = _out.returncode
output[r] = _stdout
self.logger.debug('[{0}]: (RESULT) {1}'.format(r,
'\n'.join(_stdout)))
if _returncode != 0:
self.logger.error('[{0}]: STDERR: ({2}) ({1})'.format(r,
_stderr,
' '.join(_cmd)))
if _stderr != '' and self.cron and _returncode != 0:
self.logger.warning(
'Command {0} failed: {1}'.format(' '.join(cmd), _err))
del(_env['BORG_PASSPHRASE'])
if not self.args['archive']:
if self.args['numlimit'] > 0:
if self.args['old']:
output[r] = output[r][:self.args['numlimit']]
else:
output[r] = list(reversed(output[r]))[:self.args['numlimit']]
if self.args['invert']:
output[r] = reversed(output[r])
self.logger.debug('END: lister')
return(output)


def printMoarHelp():
_helpstr = ('\n\tNOTE: Sorting only applies to listing archives, NOT the contents!\n\n'
'In order to efficiently display results, there are several options to handle it. '
'Namely, these are:\n\n\t\t'
'-s/--sort [direction]\n\t\t'
'-l/--limit [number]\n\t\t'
'-x/--invert\n\n'
'For example, if you want to list the 5 most recently *taken* snapshots, you would use:\n\n\t\t'
'-l 5\n\n'
'If you would want those SAME results SORTED in the reverse order (i.e. the 5 most recently '
'taken snapshots sorted from newest to oldest), then it would be: \n\n\t\t'
'-l 5 -x\n\n'
'Lastly, if you wanted to list the 7 OLDEST TAKEN snapshots in reverse order '
'(that is, sorted from newest to oldest), that\'d be:\n\n\t\t'
'-o -l 7 -x\n')
print(_helpstr)
exit(0)


def parseArgs():
### DEFAULTS ###
_date = datetime.datetime.now().strftime("%Y_%m_%d.%H_%M")
_logfile = '/var/log/borg/{0}'.format(_date)
_mysqldir = os.path.abspath(
os.path.join(os.path.expanduser('~'),
'.bak',
'mysql'))
_stagedir = os.path.abspath(
os.path.join(os.path.expanduser('~'),
'.bak',
'misc'))
_cfgfile = os.path.abspath(
os.path.join(os.path.expanduser('~'),
'.config',
'optools',
'backup.xml'))
_defloglvl = 'info'
######
args = argparse.ArgumentParser(description = 'Backups manager',
epilog = ('TIP: this program has context-specific help. '
'e.g. try "%(prog)s list --help"'))
args.add_argument('-c', '--config',
dest = 'cfgfile',
default = _cfgfile,
help = (
'The path to the config file. '
'Default: \033[1m{0}\033[0m'.format(_cfgfile)))
args.add_argument('-Ll', '--loglevel',
dest = 'loglevel',
default = _defloglvl,
choices = list(loglvls.keys()),
help = (
'The level of logging to perform. \033[1mWARNING:\033[0m \033[1mdebug\033[0m will '
'log VERY sensitive information such as passwords! '
'Default: \033[1m{0}\033[0m'.format(_defloglvl)))
args.add_argument('-Ld', '--log-to-disk',
dest = 'disklog',
action = 'store_true',
help = (
'If specified, log to a specific file (-Lf/--logfile) instead of the system logger.'))
args.add_argument('-Lf', '--logfile',
dest = 'logfile',
default = _logfile,
help = (
'The path to the logfile, only used if -Ld/--log-to-disk is specified. '
'Default: \033[1m{0}\033[0m (dynamic)').format(_logfile))
args.add_argument('-v', '--verbose',
dest = 'verbose',
action = 'store_true',
help = ('If specified, log messages will be printed to STDERR in addition to the other '
'configured log system(s), and verbosity for printing functions is increased. '
'\033[1mWARNING:\033[0m This may display VERY sensitive information such as passwords!'))
### ARGS FOR ALL OPERATIONS ###
commonargs = argparse.ArgumentParser(add_help = False)
commonargs.add_argument('-r', '--repo',
dest = 'repo',
default = 'all',
help = ('The repository to perform the operation for. '
'The default is \033[1mall\033[0m, a special value that specifies all known '
'repositories. Can also accept a comma-separated list.'))
fileargs = argparse.ArgumentParser(add_help = False)
fileargs.add_argument('-a', '--archive',
default = _date,
dest = 'archive',
help = ('The name of the archive/snapshot. '
'Default: \033[1m{0}\033[0m (dynamic)').format(_date))
remoteargs = argparse.ArgumentParser(add_help = False)
remoteargs.add_argument('-d', '--dry-run',
dest = 'dryrun',
action = 'store_true',
help = ('Act as if we are performing tasks, but none will actually be executed '
'(useful for testing logging)'))
### OPERATIONS ###
subparsers = args.add_subparsers(help = 'Operation to perform',
dest = 'oper')
backupargs = subparsers.add_parser('backup',
help = 'Perform a backup.',
parents = [commonargs,
remoteargs,
fileargs])
listargs = subparsers.add_parser('list',
help = 'List available backups.',
parents = [commonargs, remoteargs])
listrepoargs = subparsers.add_parser('listrepos',
help = ('List availabile/configured repositories.'),
parents = [commonargs])
initargs = subparsers.add_parser('init',
help = 'Initialise a repository.',
parents = [commonargs, remoteargs])
rstrargs = subparsers.add_parser('restore',
help = ('Restore ("extract") an archive.'),
parents = [commonargs,
remoteargs,
fileargs])
cvrtargs = subparsers.add_parser('convert',
help = ('Convert the legacy JSON format to the new XML format and quit'))
### OPERATION-SPECIFIC OPTIONS ###
# CREATE ("backup") #
backupargs.add_argument('-s', '--stagedir',
default = _stagedir,
dest = 'stagedir',
help = ('The directory used for staging temporary files, if necessary. '
'Default: \033[1m{0}\033[0m').format(_stagedir))
backupargs.add_argument('-m', '--mysqldir',
default = _mysqldir,
dest = 'mysqldir',
help = ('The path to where MySQL dumps should go. '
'Default: \033[1m{0}\033[0m').format(_mysqldir))
# DISPLAY/OUTPUT ("list") #
listargs.add_argument('-a', '--archive',
dest = 'archive',
default = False,
help = 'If specified, will list the *contents* of the given archive name.')
listargs.add_argument('-l', '--limit',
dest = 'numlimit',
type = int,
default = '5',
help = ('If specified, constrain the outout to this number of results each repo. '
'Default is \033[1m5\033[0m, use 0 for unlimited. See \033[1m-H/--list-help\033[0m'))
listargs.add_argument('-s', '--sort',
dest = 'sortby',
choices = ['newest', 'oldest'],
default = 'oldest',
help = ('The order to sort the results by. See \033[1m-H/--list-help\033[0m. '
'Default: \033[1moldest\033[0m'))
listargs.add_argument('-x', '--invert',
dest = 'invert',
action = 'store_true',
help = 'Invert the order of results. See \033[1m-H/--list-help\033[0m.')
listargs.add_argument('-o', '--old',
dest = 'old',
action = 'store_true',
help = ('Instead of grabbing the latest results, grab the earliest results. This differs '
'from \033[1m-s/--sort\033[0m. See \033[1m-H/--list-help\033[0m.'))
listargs.add_argument('-H', '--list-help',
dest = 'moarhelp',
action = 'store_true',
help = ('Print extended information about how to '
'manage the output of listing and exit.'))
## EXTRACT ("restore")
rstrargs.add_argument('-t', '--target',
required = True,
dest = 'target_dir',
help = ('The path to the directory where the restore should be dumped to. It is '
'recommended to NOT restore to the same directory that the archive is taken from.'))
return (args)

def convertConf(cfgfile):
import json
try:
with open(cfgfile, 'r') as f:
oldcfg = json.load(f)
except json.decoder.JSONDecodeError:
# It's not JSON. It's either already XML or invalid config.
return(cfgfile)
# Switched from JSON to XML, so we need to do some basic conversion.
newfname = re.sub('(\.json)?$', '.xml', os.path.basename(cfgfile))
newcfg = os.path.join(os.path.dirname(cfgfile),
newfname)
if os.path.exists(newcfg):
# Do nothing. We don't want to overwrite an existing config
# and we'll assume it's an already-done conversion.
return(newcfg)
print(('It appears that you are still using the legacy JSON format. '
'We will attempt to convert it to the new XML format ({0}) but it may '
'require modifications, especially if you are using any prep functions as those are not '
'converted automatically. See sample.config.xml for an example of this.').format(newcfg))
cfg = etree.Element('borg')
# The old format only supported one server.
server = etree.Element('server')
server.attrib['target'] = oldcfg['config']['host']
server.attrib['rsh'] = oldcfg['config']['ctx']
server.attrib['user'] = oldcfg['config']['user']
for r in oldcfg['repos']:
repo = etree.Element('repo')
repo.attrib['name'] = r
repo.attrib['password'] = oldcfg['repos'][r]['password']
for p in oldcfg['repos'][r]['paths']:
path = etree.Element('path')
server.append(repo)
# Build the full XML spec.
namespaces = {'borg': 'http://git.square-r00t.net/OpTools/tree/storage/backups/borg/',
'xsi': 'http://www.w3.org/2001/XMLSchema-instance'}
xsi = {('{http://www.w3.org/2001/'
'XMLSchema-instance}schemaLocation'): ('http://git.square-r00t.net/OpTools/plain/'
'storage/backups/borg/config.xsd')}
if has_lxml:
genname = 'LXML (http://lxml.de/)'
root = etree.Element('borg', nsmap = namespaces, attrib = xsi)
else:
genname = 'Python stdlib "xml" module'
for ns in namespaces.keys():
etree.register_namespace(ns, namespaces[ns])
root = etree.Element('borg')
fromstr = cfgfile
root.append(etree.Comment(
('Generated by {0} on {1} from {2} via {3}').format(sys.argv[0],
datetime.datetime.now(),
fromstr,
genname)))
root.append(etree.Comment('THIS FILE CONTAINS SENSITIVE INFORMATION. SHARE/SCRUB WISELY.'))
for x in cfg:
root.append(x)
# Write out the file to disk.
if has_lxml:
xml = etree.ElementTree(root)
with open(newcfg, 'wb') as f:
xml.write(f,
xml_declaration = True,
encoding = 'utf-8',
pretty_print = True)
else:
import xml.dom.minidom
xmlstr = etree.tostring(root, encoding = 'utf-8')
# holy cats, the xml module sucks.
nsstr = ''
for ns in namespaces.keys():
nsstr += ' xmlns:{0}="{1}"'.format(ns, namespaces[ns])
for x in xsi.keys():
xsiname = x.split('}')[1]
nsstr += ' xsi:{0}="{1}"'.format(xsiname, xsi[x])
outstr = xml.dom.minidom.parseString(xmlstr).toprettyxml(indent = ' ').splitlines()
outstr[0] = '<?xml version=\'1.0\' encoding=\'utf-8\'?>'
outstr[1] = '<borg{0}>'.format(nsstr)
with open(newcfg, 'w') as f:
f.write('\n'.join(outstr))
# Return the new config's path.
return(newcfg)


def main():
rawargs = parseArgs()
parsedargs = rawargs.parse_args()
args = vars(parsedargs)
args['cfgfile'] = os.path.abspath(os.path.expanduser(args['cfgfile']))
if not args['oper']:
rawargs.print_help()
exit(0)
if args['oper'] == 'convert':
convertConf(args['cfgfile'])
return()
else:
args['cfgfile'] = convertConf(args['cfgfile'])
if 'moarhelp' in args.keys() and args['moarhelp']:
printMoarHelp()
# The "Do stuff" part
bak = Backup(args)
if args['oper'] == 'list':
bak.printer()
elif args['oper'] == 'listrepos':
bak.listRepos()
elif args['oper'] == 'backup':
bak.create()
elif args['oper'] == 'init':
bak.createRepo()
elif args['oper'] == 'restore':
bak.restore()
return()


if __name__ == '__main__':
main()

3
storage/backups/borg/config.xsd
View File

@ -92,6 +92,7 @@
<!-- Optional. If not specified, the password will <!-- Optional. If not specified, the password will
be interactively (and securely) prompted for. --> be interactively (and securely) prompted for. -->
<xs:attribute name="password" type="xs:string" use="optional"/> <xs:attribute name="password" type="xs:string" use="optional"/>
<xs:attribute name="compression" type="xs:token" use="optional"/>
</xs:complexType> </xs:complexType>
<xs:unique name="uniquePath"> <xs:unique name="uniquePath">
<xs:selector xpath="borg:path"/> <xs:selector xpath="borg:path"/>
@ -103,6 +104,8 @@
<!-- "target" should be either a local filesystem path or the remote hostname. --> <!-- "target" should be either a local filesystem path or the remote hostname. -->
<!-- This should *not* contain a path if it's remote. If it does, you set up Borg wrong. --> <!-- This should *not* contain a path if it's remote. If it does, you set up Borg wrong. -->
<xs:attribute name="target" type="xs:anyURI" use="required"/> <xs:attribute name="target" type="xs:anyURI" use="required"/>
<!-- "remote" is used to determine what type "target" is. -->
<xs:attribute name="remote" type="xs:boolean" use="required"/>
<!-- Only used if "target" is a remote host. --> <!-- Only used if "target" is a remote host. -->
<!-- See "BORG_RSH" at https://borgbackup.readthedocs.io/en/stable/usage/general.html --> <!-- See "BORG_RSH" at https://borgbackup.readthedocs.io/en/stable/usage/general.html -->
<xs:attribute name="rsh" type="xs:string" use="optional"/> <xs:attribute name="rsh" type="xs:string" use="optional"/>

1
storage/backups/borg/plugins/ldap.py
View File

@ -26,6 +26,7 @@ class Backup(object):
self.binddn = binddn self.binddn = binddn
self.outdir = os.path.abspath(os.path.expanduser(outdir)) self.outdir = os.path.abspath(os.path.expanduser(outdir))
os.makedirs(self.outdir, exist_ok = True) os.makedirs(self.outdir, exist_ok = True)
os.chmod(self.outdir, mode = 0o0700)
self.splitldifs = splitldifs self.splitldifs = splitldifs
self.starttls = starttls self.starttls = starttls
if password_file and not password: if password_file and not password:

1
storage/backups/borg/plugins/mysql.py
View File

@ -26,6 +26,7 @@ class Backup(object):
self.outdir = os.path.abspath(os.path.expanduser(outdir)) self.outdir = os.path.abspath(os.path.expanduser(outdir))
self.cfg = os.path.abspath(os.path.expanduser(cfg)) self.cfg = os.path.abspath(os.path.expanduser(cfg))
os.makedirs(self.outdir, exist_ok = True) os.makedirs(self.outdir, exist_ok = True)
os.chmod(self.outdir, mode = 0o0700)
if not os.path.isfile(self.cfg): if not os.path.isfile(self.cfg):
raise OSError(('{0} does not exist!').format(self.cfg)) raise OSError(('{0} does not exist!').format(self.cfg))
if not dumpopts: if not dumpopts:

29
storage/backups/borg/sample.config.xml
View File

@ -1,15 +1,20 @@
<?xml version="1.0" encoding="UTF-8" ?> <?xml version="1.0" encoding="UTF-8" ?>
<borg xmlns="http://git.square-r00t.net/OpTools/tree/storage/backups/borg/" <borg xmlns="http://git.square-r00t.net/OpTools/tree/storage/backups/borg/"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://git.square-r00t.net/OpTools/plain/storage/backups/borg/config.xsd"> xsi:schemaLocation="http://git.square-r00t.net/OpTools/plain/storage/backups/borg/config.xsd"
<!-- "target" = either the local filesystem (absolute) path or the remote host borgpath="/usr/bin/borg">
<!-- You can have multiple server elements, but each one *MUST* have a unique "target" attribute. -->
<!-- "target" = either the local filesystem path (absolute or relative to execution) or the remote host
"remote" = 1/true if "target" is a remote host or 0/false if it's a local filepath
"rsh" = (remote host only) the ssh command to use. The default is given below. "rsh" = (remote host only) the ssh command to use. The default is given below.
"user" = (remote host only) the ssh user to use. --> "user" = (remote host only) the ssh user to use. -->
<server target=" fq.dn.tld" rsh="ssh -p 22" user="root"> <server target="fq.dn.tld" remote="true" rsh="ssh -p 22" user="root">
<!-- You can (and probably will) have multiple repos for each server. -->
<!-- "name" = the repositoriy name. <!-- "name" = the repositoriy name.
"password" = the repository's password for the key. If not specified, you will be prompted "password" = the repository's password for the key. If not specified, you will be prompted
to enter it interactively and securely. --> to enter it interactively and securely.
<repo name="testrepo" password="SuperSecretPassword"> "compression" = see https://borgbackup.readthedocs.io/en/stable/usage/create.html (-C option) -->
<repo name="testrepo" password="SuperSecretPassword" compression="lzma,9">
<!-- Each path entry is a path to back up. <!-- Each path entry is a path to back up.
See https://borgbackup.readthedocs.io/en/stable/usage/create.html for examples of globbing, etc. --> See https://borgbackup.readthedocs.io/en/stable/usage/create.html for examples of globbing, etc. -->
<path>/a</path> <path>/a</path>
@ -20,13 +25,16 @@
If you require them to be in a specific order, you should use a wrapper script and If you require them to be in a specific order, you should use a wrapper script and
use that as a prep item. --> use that as a prep item. -->
<!-- "inline" = if true/1, the provided text will be temporarily written to disk, executed, and deleted. <!-- "inline" = if true/1, the provided text will be temporarily written to disk, executed, and deleted.
if false/0, the provided text is assumed to be a single-shot command/path to a script. --> if false/0, the provided text is assumed to be a single-shot command/path to a script
(arguments are not currently supported, but may be in the future). -->
<!-- If using inline especially, take note of and use XML escape characters: <!-- If using inline especially, take note of and use XML escape characters:
" = &quot; " = &quot;
' = &apos; ' = &apos;
< = &lt; < = &lt;
> = &gt; > = &gt;
& = &amp; --> & = &amp;
and note that whitespace (including leading!) *is* preserved. -->
<!-- It *MUST* return 0 on success. -->
<prep inline="1">#!/bin/bash <prep inline="1">#!/bin/bash
# this is block text # this is block text
</prep> </prep>
@ -47,10 +55,11 @@
If you want a parameter to be provided but with a None value, make it self-enclosed If you want a parameter to be provided but with a None value, make it self-enclosed
(e.g. '<param key="someparam"/>'). (e.g. '<param key="someparam"/>').
If you need to serialize pythonic objects (lists, dicts, booleans), If you need to serialize pythonic objects (lists, dicts, booleans),
then set the "json" attribute to 1/true and provide the data in minified JSON format. --> then set the "json" attribute to 1/true and provide the data in minified
<param key="dbs" json="true">["db1", "db2"]</param> JSON format (also referred to as "compressed JSON") - see "tools/minify_json.py -h". -->
<param key="dbs" json="true">["db1","db2"]</param>
<param key="splitdumps" json="true">true</param> <param key="splitdumps" json="true">true</param>
<param key="dumpopts" json="true">["--routines", "--add-drop-database", "--add-drop-table", "--allow-keywords", "--complete-insert", "--create-options", "--extended-insert"]</param> <param key="dumpopts" json="true">["--routines","--add-drop-database","--add-drop-table","--allow-keywords","--complete-insert","--create-options","--extended-insert"]</param>
</plugin> </plugin>
<plugin name="ldap" path="./plugins"> <plugin name="ldap" path="./plugins">
<param key="server">ldap://my.server.tld</param> <param key="server">ldap://my.server.tld</param>

52
storage/backups/borg/tools/minify_json.py Executable file
View File

@ -0,0 +1,52 @@
#!/usr/bin/env python3

import argparse
import json
import os
import sys

def minify(json_in):
j = json.loads(json_in)
j = json.dumps(j, indent = None, separators = (',', ':'))
return(j)

def parseArgs():
args = argparse.ArgumentParser(description = ('Minify ("compress") JSON input'))
args.add_argument('-o', '--output',
default = '-',
help = ('Write the minified JSON out to a file. The default is "-", which instead prints it to '
'STDOUT. If instead you would like to write out to STDERR, use "+" (otherwise provide a '
'path)'))
args.add_argument('json_in',
default = '-',
nargs = '?',
help = ('The JSON input. If "-" (the default), read STDIN; otherwise provide a path to the '
'JSON file'))
return(args)

def main():
args = parseArgs().parse_args()
if args.json_in.strip() == '-':
stdin = sys.stdin.read()
if not stdin:
raise argparse.ArgumentError('You specified to read from STDIN, but STDIN is blank')
else:
args.json_in = stdin
else:
with open(os.path.abspath(os.path.expanduser(args.json_in)), 'r') as f:
args.json_in = f.read()
minified = minify(args.json_in)
if args.output.strip() not in ('-', '+'):
args.output = os.path.abspath(os.path.expanduser(args.output))
if not args.output.endswith('.json'):
args.output += '.json'
with open(args.output, 'w') as f:
f.write(minified + '\n')
elif args.output.strip() == '+':
sys.stderr.write(minified + '\n')
else:
sys.stdout.write(minified + '\n')
return()

if __name__ == '__main__':
main()