r00t2/optools
1
0
Fork 0
Code Issues Pull Requests Projects Releases Activity

adding restore functionality

Browse Source
This commit is contained in:
brent s 2018-08-07 17:42:54 -04:00
parent b566970d57
commit 120b576a38
1 changed files with 298 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

425
storage/backups/borg/backup.py
View File

@ -3,6 +3,8 @@
# TODO: https://borgbackup.readthedocs.io/en/latest/internals/frontends.html # TODO: https://borgbackup.readthedocs.io/en/latest/internals/frontends.html
# will they EVER release a public API? for now we'll just use subprocess since # will they EVER release a public API? for now we'll just use subprocess since
# we import it for various prep stuff anyways. # we import it for various prep stuff anyways.
# TODO: change loglevel of borg itself in subprocess to match the argparse?
# --debug, --info (same as -v/--verbose), --warning, --error, --critical


import argparse import argparse
import configparser import configparser
@ -13,6 +15,7 @@ import logging.handlers
import os import os
import subprocess import subprocess
import sys import sys

try: try:
import pymysql # not stdlib; "python-pymysql" in Arch's AUR import pymysql # not stdlib; "python-pymysql" in Arch's AUR
has_mysql = True has_mysql = True
@ -26,11 +29,13 @@ except ImportError:
has_systemd = False has_systemd = False


### LOG LEVEL MAPPINGS ### ### LOG LEVEL MAPPINGS ###
loglvls = {'critical': logging.CRITICAL, loglvls = {
'error': logging.ERROR, 'critical': logging.CRITICAL,
'warning': logging.WARNING, 'error': logging.ERROR,
'info': logging.INFO, 'warning': logging.WARNING,
'debug': logging.DEBUG} 'info': logging.INFO,
'debug': logging.DEBUG}



### THE GUTS ### ### THE GUTS ###
class Backup(object): class Backup(object):
@ -48,21 +53,26 @@ class Backup(object):
# and user K900_ on r/python for entertaining my very silly question. # and user K900_ on r/python for entertaining my very silly question.
self.logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
self.logger.setLevel(loglvls[self.args['loglevel']]) self.logger.setLevel(loglvls[self.args['loglevel']])
_logfmt = logging.Formatter(fmt = '{levelname}:{name}: {message} ({asctime}; {filename}:{lineno})', _logfmt = logging.Formatter(
style = '{', fmt = ('{levelname}:{name}: {message} ({asctime}; '
datefmt = '%Y-%m-%d %H:%M:%S') '{filename}:{lineno})'),
_journalfmt = logging.Formatter(fmt = '{levelname}:{name}: {message} ({filename}:{lineno})', style = '{',
style = '{', datefmt = '%Y-%m-%d %H:%M:%S')
datefmt = '%Y-%m-%d %H:%M:%S') _journalfmt = logging.Formatter(
fmt = '{levelname}:{name}: {message} ({filename}:{lineno})',
style = '{',
datefmt = '%Y-%m-%d %H:%M:%S')
handlers = [] handlers = []
if self.args['disklog']: if self.args['disklog']:
os.makedirs(os.path.dirname(self.args['logfile']), os.makedirs(os.path.dirname(self.args['logfile']),
exist_ok = True, exist_ok = True,
mode = 0o700) mode = 0o700)
handlers.append(logging.handlers.RotatingFileHandler(self.args['logfile'], # TODO: make the constraints for rotation in config?
encoding = 'utf8', handlers.append(
maxBytes = 100000, logging.handlers.RotatingFileHandler(self.args['logfile'],
backupCount = 1)) encoding = 'utf8',
maxBytes = 100000,
backupCount = 1))
if self.args['verbose']: if self.args['verbose']:
handlers.append(logging.StreamHandler()) handlers.append(logging.StreamHandler())
if has_systemd: if has_systemd:
@ -77,7 +87,8 @@ class Backup(object):
self.logger.debug('BEGIN INITIALIZATION') self.logger.debug('BEGIN INITIALIZATION')
### CONFIG ### ### CONFIG ###
if not os.path.isfile(self.args['cfgfile']): if not os.path.isfile(self.args['cfgfile']):
self.logger.error('{0} does not exist'.format(self.args['cfgfile'])) self.logger.error(
'{0} does not exist'.format(self.args['cfgfile']))
exit(1) exit(1)
with open(self.args['cfgfile'], 'r') as f: with open(self.args['cfgfile'], 'r') as f:
self.cfg = json.loads(f.read()) self.cfg = json.loads(f.read())
@ -91,7 +102,9 @@ class Backup(object):
if r == 'all': if r == 'all':
self.args['repo'].remove(r) self.args['repo'].remove(r)
elif r not in self.cfg['repos'].keys(): elif r not in self.cfg['repos'].keys():
self.logger.warning('Repository {0} is not configured; skipping.'.format(r)) self.logger.warning(
'Repository {0} is not configured; skipping.'.format(
r))
self.args['repo'].remove(r) self.args['repo'].remove(r)
self.logger.debug('VARS (after args cleanup): {0}'.format(vars())) self.logger.debug('VARS (after args cleanup): {0}'.format(vars()))
self.logger.debug('END INITIALIZATION') self.logger.debug('END INITIALIZATION')
@ -102,13 +115,14 @@ class Backup(object):
else: else:
self.cron = True self.cron = True
### END INIT ### ### END INIT ###

def cmdExec(self, cmd, stdoutfh = None): def cmdExec(self, cmd, stdoutfh = None):
self.logger.debug('Running command: {0}'.format(' '.join(cmd))) self.logger.debug('Running command: {0}'.format(' '.join(cmd)))
if self.args['dryrun']: if self.args['dryrun']:
return() # no-op return () # no-op
if stdoutfh: if stdoutfh:
_cmd = subprocess.run(cmd, stdout = stdoutfh, stderr = subprocess.PIPE) _cmd = subprocess.run(cmd, stdout = stdoutfh,
stderr = subprocess.PIPE)
else: else:
_cmd = subprocess.run(cmd, _cmd = subprocess.run(cmd,
stdout = subprocess.PIPE, stdout = subprocess.PIPE,
@ -119,9 +133,10 @@ class Backup(object):
if _returncode != 0: if _returncode != 0:
self.logger.error('STDERR: ({1})\n{0}'.format(_err, ' '.join(cmd))) self.logger.error('STDERR: ({1})\n{0}'.format(_err, ' '.join(cmd)))
if _err != '' and self.cron: if _err != '' and self.cron:
self.logger.warning('Command {0} failed: {1}'.format(' '.join(cmd), _err)) self.logger.warning(
'Command {0} failed: {1}'.format(' '.join(cmd), _err))
return() return()

def createRepo(self): def createRepo(self):
_env = os.environ.copy() _env = os.environ.copy()
_env['BORG_RSH'] = self.cfg['config']['ctx'] _env['BORG_RSH'] = self.cfg['config']['ctx']
@ -149,16 +164,21 @@ class Backup(object):
# sigh. borg uses stderr for verbose output. # sigh. borg uses stderr for verbose output.
self.logger.debug('[{0}]: STDERR: ({2})\n{1}'.format(r, self.logger.debug('[{0}]: STDERR: ({2})\n{1}'.format(r,
_stderr, _stderr,
' '.join(_cmd))) ' '.join(
_cmd)))
if _returncode != 0: if _returncode != 0:
self.logger.error('[{0}]: FAILED: {1}'.format(r, ' '.join(_cmd))) self.logger.error(
'[{0}]: FAILED: {1}'.format(r, ' '.join(_cmd)))
if _err != '' and self.cron and _returncode != 0: if _err != '' and self.cron and _returncode != 0:
self.logger.warning('Command {0} failed: {1}'.format(' '.join(cmd), _err)) self.logger.warning(
del(_env['BORG_PASSPHRASE']) 'Command {0} failed: {1}'.format(' '.join(cmd),
_err))
del (_env['BORG_PASSPHRASE'])
self.logger.info('[{0}]: END INITIALIZATION'.format(r)) self.logger.info('[{0}]: END INITIALIZATION'.format(r))
return() return()


def create(self): def create(self):
# TODO: support "--strip-components N"?
_env = os.environ.copy() _env = os.environ.copy()
_env['BORG_RSH'] = self.cfg['config']['ctx'] _env['BORG_RSH'] = self.cfg['config']['ctx']
self.logger.info('START: backup') self.logger.info('START: backup')
@ -166,9 +186,12 @@ class Backup(object):
self.logger.info('[{0}]: BEGIN BACKUP'.format(r)) self.logger.info('[{0}]: BEGIN BACKUP'.format(r))
if 'prep' in self.cfg['repos'][r].keys(): if 'prep' in self.cfg['repos'][r].keys():
for prep in self.cfg['repos'][r]['prep']: for prep in self.cfg['repos'][r]['prep']:
self.logger.info('[{0}]: Running prepfunc {1}'.format(r, prep)) self.logger.info(
eval('self.{0}'.format(prep)) # I KNOW, IT'S TERRIBLE. so sue me. '[{0}]: Running prepfunc {1}'.format(r, prep))
self.logger.info('[{0}]: Finished prepfunc {1}'.format(r, prep)) eval('self.{0}'.format(
prep)) # I KNOW, IT'S TERRIBLE. so sue me.
self.logger.info(
'[{0}]: Finished prepfunc {1}'.format(r, prep))
_cmd = ['borg', _cmd = ['borg',
'create', 'create',
'-v', '--stats', '-v', '--stats',
@ -184,9 +207,10 @@ class Backup(object):
_cmd.append(p) _cmd.append(p)
_env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password'] _env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password']
self.logger.debug('VARS: {0}'.format(vars())) self.logger.debug('VARS: {0}'.format(vars()))
# We don't use self.cmdExec() here because we want to explicitly pass the env # We don't use self.cmdExec() here because we want to explicitly
# and format the log line differently. # pass the env and format the log line differently.
self.logger.debug('[{0}]: Running command: {1}'.format(r, ' '.join(_cmd))) self.logger.debug(
'[{0}]: Running command: {1}'.format(r, ' '.join(_cmd)))
if not self.args['dryrun']: if not self.args['dryrun']:
_out = subprocess.run(_cmd, _out = subprocess.run(_cmd,
env = _env, env = _env,
@ -198,16 +222,73 @@ class Backup(object):
self.logger.debug('[{0}]: (RESULT) {1}'.format(r, _stdout)) self.logger.debug('[{0}]: (RESULT) {1}'.format(r, _stdout))
self.logger.error('[{0}]: STDERR: ({2})\n{1}'.format(r, self.logger.error('[{0}]: STDERR: ({2})\n{1}'.format(r,
_stderr, _stderr,
' '.join(_cmd))) ' '.join(
_cmd)))
if _returncode != 0: if _returncode != 0:
self.logger.error('[{0}]: FAILED: {1}'.format(r, ' '.join(_cmd))) self.logger.error(
'[{0}]: FAILED: {1}'.format(r, ' '.join(_cmd)))
if _stderr != '' and self.cron and _returncode != 0: if _stderr != '' and self.cron and _returncode != 0:
self.logger.warning('Command {0} failed: {1}'.format(' '.join(_cmd), _stderr)) self.logger.warning(
del(_env['BORG_PASSPHRASE']) 'Command {0} failed: {1}'.format(' '.join(_cmd),
_stderr))
del (_env['BORG_PASSPHRASE'])
self.logger.info('[{0}]: END BACKUP'.format(r)) self.logger.info('[{0}]: END BACKUP'.format(r))
self.logger.info('END: backup') self.logger.info('END: backup')
return() return()


def restore(self):
# TODO: support "--strip-components N"?
# TODO: support add'l args?
# https://borgbackup.readthedocs.io/en/stable/usage/extract.html
_env = os.environ.copy()
_env['BORG_RSH'] = self.cfg['config']['ctx']
self.logger.info('START: restore')
for r in self.args['repo']:
self.logger.info('[{0}]: BEGIN RESTORE'.format(r))
_cmd = ['borg',
'extract',
'-v', '--stats',
'--compression', 'lzma,9']
# if 'excludes' in self.cfg['repos'][r].keys():
# for e in self.cfg['repos'][r]['excludes']:
# _cmd.extend(['--exclude', e])
_cmd.append('{0}@{1}:{2}::{3}'.format(self.cfg['config']['user'],
self.cfg['config']['host'],
r,
self.args['archive']))
# TODO: support specific path of extract?
# if so, append path(s) here.
_env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password']
self.logger.debug('VARS: {0}'.format(vars()))
# We don't use self.cmdExec() here because we want to explicitly
# pass the env and format the log line differently.
self.logger.debug(
'[{0}]: Running command: {1}'.format(r, ' '.join(_cmd)))
if not self.args['dryrun']:
_out = subprocess.run(_cmd,
env = _env,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE)
_stdout = _out.stdout.decode('utf-8').strip()
_stderr = _out.stderr.decode('utf-8').strip()
_returncode = _out.returncode
self.logger.debug('[{0}]: (RESULT) {1}'.format(r, _stdout))
self.logger.error('[{0}]: STDERR: ({2})\n{1}'.format(r,
_stderr,
' '.join(
_cmd)))
if _returncode != 0:
self.logger.error(
'[{0}]: FAILED: {1}'.format(r, ' '.join(_cmd)))
if _stderr != '' and self.cron and _returncode != 0:
self.logger.warning(
'Command {0} failed: {1}'.format(' '.join(_cmd),
_stderr))
del (_env['BORG_PASSPHRASE'])
self.logger.info('[{0}]: END RESTORE'.format(r))
self.logger.info('END: restore')
return()

def miscBak(self, pkgr): def miscBak(self, pkgr):
self.logger.info('BEGIN: miscBak()') self.logger.info('BEGIN: miscBak()')
_cmd = None _cmd = None
@ -231,8 +312,9 @@ class Backup(object):
def mysqlBak(self): def mysqlBak(self):
self.logger.info('BEGIN: mysqlBak()') self.logger.info('BEGIN: mysqlBak()')
if not has_mysql: if not has_mysql:
self.logger.error('You need to install the PyMySQL module to back up MySQL databases. Skipping.') self.logger.error(
return() 'You need to install the PyMySQL module to back up MySQL databases. Skipping.')
return ()
# These are mysqldump options shared by ALL databases # These are mysqldump options shared by ALL databases
_mysqlopts = ['--routines', _mysqlopts = ['--routines',
'--add-drop-database', '--add-drop-database',
@ -244,11 +326,12 @@ class Backup(object):
_DBs = [] _DBs = []
_mycnf = os.path.expanduser(os.path.join('~', '.my.cnf')) _mycnf = os.path.expanduser(os.path.join('~', '.my.cnf'))
if not os.path.isfile(_mycnf): if not os.path.isfile(_mycnf):
exit('{0}: ERROR: Cannot get credentials for MySQL (cannot find ~/.my.cnf)!') exit(
'{0}: ERROR: Cannot get credentials for MySQL (cannot find ~/.my.cnf)!')
_mycfg = configparser.ConfigParser() _mycfg = configparser.ConfigParser()
_mycfg._interpolation = configparser.ExtendedInterpolation() _mycfg._interpolation = configparser.ExtendedInterpolation()
_mycfg.read(_mycnf) _mycfg.read(_mycnf)
_sqlcfg = {s:dict(_mycfg.items(s)) for s in _mycfg.sections()} _sqlcfg = {s: dict(_mycfg.items(s)) for s in _mycfg.sections()}
if 'host' not in _sqlcfg.keys(): if 'host' not in _sqlcfg.keys():
_socketpath = '/var/run/mysqld/mysqld.sock' # correct for Arch, YMMV. _socketpath = '/var/run/mysqld/mysqld.sock' # correct for Arch, YMMV.
_mysql = pymysql.connect(unix_socket = _socketpath, _mysql = pymysql.connect(unix_socket = _socketpath,
@ -266,7 +349,8 @@ class Backup(object):
self.logger.debug('Databases: {0}'.format(', '.join(_DBs))) self.logger.debug('Databases: {0}'.format(', '.join(_DBs)))
for db in _DBs: for db in _DBs:
_cmd = ['mysqldump', _cmd = ['mysqldump',
'--result-file={0}.sql'.format(os.path.join(self.args['mysqldir'], db))] '--result-file={0}.sql'.format(
os.path.join(self.args['mysqldir'], db))]
# These are database-specific options # These are database-specific options
if db in ('information_schema', 'performance_schema'): if db in ('information_schema', 'performance_schema'):
_cmd.append('--skip-lock-tables') _cmd.append('--skip-lock-tables')
@ -284,7 +368,9 @@ class Backup(object):
if self.args['verbose']: if self.args['verbose']:
print('\033[1mDETAILS:\033[0m\n') print('\033[1mDETAILS:\033[0m\n')
for r in self.args['repo']: for r in self.args['repo']:
print('\t\033[1m{0}:\033[0m\n\t\t\033[1mPath(s):\033[0m\t'.format(r.upper()), end = '') print(
'\t\033[1m{0}:\033[0m\n\t\t\033[1mPath(s):\033[0m\t'.format(
r.upper()), end = '')
for p in self.cfg['repos'][r]['paths']: for p in self.cfg['repos'][r]['paths']:
print(p, end = ' ') print(p, end = ' ')
if 'prep' in self.cfg['repos'][r].keys(): if 'prep' in self.cfg['repos'][r].keys():
@ -307,15 +393,19 @@ class Backup(object):
print(r, end = '') print(r, end = '')
for line in _results[r]: for line in _results[r]:
_snapshot = line.split() _snapshot = line.split()
print('\t{0}\t\t{1}'.format(_snapshot[0], ' '.join(_snapshot[1:]))) print('\t{0}\t\t{1}'.format(_snapshot[0],
' '.join(_snapshot[1:])))
print() print()
else: # It's a listing inside an archive else: # It's a listing inside an archive
if self.args['verbose']: if self.args['verbose']:
_fields = ['REPO:', 'PERMS:', 'OWNERSHIP:', 'SIZE:', 'TIMESTAMP:', 'PATH:'] _fields = ['REPO:', 'PERMS:', 'OWNERSHIP:', 'SIZE:',
'TIMESTAMP:', 'PATH:']
for r in _results.keys(): for r in _results.keys():
print('\033[1m{0}\t{1}\033[0m'.format(_fields[0], r)) print('\033[1m{0}\t{1}\033[0m'.format(_fields[0], r))
# https://docs.python.org/3/library/string.html#formatspec # https://docs.python.org/3/library/string.html#formatspec
print('{0[1]:<15}\t{0[2]:<15}\t{0[3]:<15}\t{0[4]:<24}\t{0[5]:<15}'.format(_fields)) print(
'{0[1]:<15}\t{0[2]:<15}\t{0[3]:<15}\t{0[4]:<24}\t{0[5]:<15}'.format(
_fields))
for line in _results[r]: for line in _results[r]:
_fline = line.split() _fline = line.split()
_perms = _fline[0] _perms = _fline[0]
@ -323,11 +413,13 @@ class Backup(object):
_size = _fline[3] _size = _fline[3]
_time = ' '.join(_fline[4:7]) _time = ' '.join(_fline[4:7])
_path = ' '.join(_fline[7:]) _path = ' '.join(_fline[7:])
print('{0:<15}\t{1:<15}\t{2:<15}\t{3:<24}\t{4:<15}'.format(_perms, print(
_ownership, '{0:<15}\t{1:<15}\t{2:<15}\t{3:<24}\t{4:<15}'.format(
_size, _perms,
_time, _ownership,
_path)) _size,
_time,
_path))
else: else:
print('\033[1mREPO:\tPATH:\033[0m\n') print('\033[1mREPO:\tPATH:\033[0m\n')
for r in _results.keys(): for r in _results.keys():
@ -360,173 +452,249 @@ class Backup(object):
if not self.args['dryrun']: if not self.args['dryrun']:


_out = subprocess.run(_cmd, _out = subprocess.run(_cmd,
env = _env, env = _env,
stdout = subprocess.PIPE, stdout = subprocess.PIPE,
stderr = subprocess.PIPE) stderr = subprocess.PIPE)
_stdout = [i.strip() for i in _out.stdout.decode('utf-8').splitlines()] _stdout = [i.strip() for i in
_out.stdout.decode('utf-8').splitlines()]
_stderr = _out.stderr.decode('utf-8').strip() _stderr = _out.stderr.decode('utf-8').strip()
_returncode = _out.returncode _returncode = _out.returncode
output[r] = _stdout output[r] = _stdout
self.logger.debug('[{0}]: (RESULT) {1}'.format(r, self.logger.debug('[{0}]: (RESULT) {1}'.format(r,
'\n'.join(_stdout))) '\n'.join(
_stdout)))
if _returncode != 0: if _returncode != 0:
self.logger.error('[{0}]: STDERR: ({2}) ({1})'.format(r, self.logger.error('[{0}]: STDERR: ({2}) ({1})'.format(r,
_stderr, _stderr,
' '.join(_cmd))) ' '.join(
_cmd)))
if _stderr != '' and self.cron and _returncode != 0: if _stderr != '' and self.cron and _returncode != 0:
self.logger.warning('Command {0} failed: {1}'.format(' '.join(cmd), _err)) self.logger.warning(
del(_env['BORG_PASSPHRASE']) 'Command {0} failed: {1}'.format(' '.join(cmd),
_err))
del (_env['BORG_PASSPHRASE'])
if not self.args['archive']: if not self.args['archive']:
if self.args['numlimit'] > 0: if self.args['numlimit'] > 0:
if self.args['old']: if self.args['old']:
output[r] = output[r][:self.args['numlimit']] output[r] = output[r][:self.args['numlimit']]
else: else:
output[r] = list(reversed(output[r]))[:self.args['numlimit']] output[r] = list(reversed(output[r]))[
:self.args['numlimit']]
if self.args['invert']: if self.args['invert']:
output[r] = reversed(output[r]) output[r] = reversed(output[r])
self.logger.debug('END: lister') self.logger.debug('END: lister')
return(output) return(output)



def printMoarHelp(): def printMoarHelp():
_helpstr = ('\n\tNOTE: Sorting only applies to listing archives, NOT the contents!\n\n' + _helpstr = (
'In order to efficiently display results, there are several options to handle it. ' + '\n\tNOTE: Sorting only applies to listing archives, NOT the '
'Namely, these are:\n\n\t\t-s/--sort [direction]\n\t\t-l/--limit [number]\n\t\t-x/--invert\n\n' + 'contents!\n\n'
'For example, if you want to list the 5 most recently *taken* snapshots, you would use:\n\n\t\t-l 5\n\n' + 'In order to efficiently display results, there are several options '
'If you would want those SAME results SORTED in the reverse order (i.e. the 5 most recently ' + 'to handle it. Namely, these are:\n\n\t\t'
'taken snapshots sorted from newest to oldest), then it would be: \n\n\t\t-l 5 -x\n\n' + '-s/--sort [direction]\n\t\t'
'Lastly, if you wanted to list the 7 OLDEST TAKEN snapshots in reverse order (that is, ' + '-l/--limit [number]\n\t\t'
'sorted from newest to oldest), that\'d be: \n\n\t\t-o -l 7 -x\n') '-x/--invert\n\n'
'For example, if you want to list the 5 most recently *taken* '
'snapshots, you would use:\n\n\t\t'
'-l 5\n\n'
'If you would want those SAME results SORTED in the reverse order '
'(i.e. the 5 most recently taken snapshots sorted from newest to '
'oldest), then it would be: \n\n\t\t'
'-l 5 -x\n\n'
'Lastly, if you wanted to list the 7 OLDEST TAKEN snapshots in '
'reverse order (that is, sorted from newest to oldest), that\'d be: '
'\n\n\t\t'
'-o -l 7 -x\n')
print(_helpstr) print(_helpstr)
exit(0) exit(0)



def parseArgs(): def parseArgs():
### DEFAULTS ### ### DEFAULTS ###
_date = datetime.datetime.now().strftime("%Y_%m_%d.%H_%M") _date = datetime.datetime.now().strftime("%Y_%m_%d.%H_%M")
_logfile = '/var/log/borg/{0}'.format(_date) _logfile = '/var/log/borg/{0}'.format(_date)
_mysqldir = os.path.abspath(os.path.join(os.path.expanduser('~'), '.bak', 'mysql')) _mysqldir = os.path.abspath(
_stagedir = os.path.abspath(os.path.join(os.path.expanduser('~'), '.bak', 'misc')) os.path.join(os.path.expanduser('~'),
_cfgfile = os.path.abspath(os.path.join(os.path.expanduser('~'), '.config', 'optools', 'backup.json')) '.bak',
'mysql'))
_stagedir = os.path.abspath(
os.path.join(os.path.expanduser('~'),
'.bak',
'misc'))
_cfgfile = os.path.abspath(
os.path.join(os.path.expanduser('~'),
'.config',
'optools',
'backup.json'))
_defloglvl = 'info' _defloglvl = 'info'
###### ######
args = argparse.ArgumentParser(description = 'Backups manager', args = argparse.ArgumentParser(description = 'Backups manager',
epilog = 'TIP: this program has context-specific help. e.g. try "%(prog)s list --help"') epilog = ('TIP: this program has '
'context-specific help. e.g. '
'try "%(prog)s list --help"'))
args.add_argument('-c', '--config', args.add_argument('-c', '--config',
dest = 'cfgfile', dest = 'cfgfile',
default = _cfgfile, default = _cfgfile,
help = ('The path to the config file. Default: \033[1m{0}\033[0m'.format(_cfgfile))) help = (
'The path to the config file. '
'Default: \033[1m{0}\033[0m'.format(_cfgfile)))
args.add_argument('-Ll', '--loglevel', args.add_argument('-Ll', '--loglevel',
dest = 'loglevel', dest = 'loglevel',
default = _defloglvl, default = _defloglvl,
choices = list(loglvls.keys()), choices = list(loglvls.keys()),
help = ('The level of logging to perform. \033[1mWARNING:\033[0m \033[1mdebug\033[0m will log ' + help = (
'VERY sensitive information such as passwords! Default: \033[1m{0}\033[0m'.format(_defloglvl))) 'The level of logging to perform. '
'\033[1mWARNING:\033[0m \033[1mdebug\033[0m will '
'log VERY sensitive information such as passwords! '
'Default: \033[1m{0}\033[0m'.format(_defloglvl)))
args.add_argument('-Ld', '--log-to-disk', args.add_argument('-Ld', '--log-to-disk',
dest = 'disklog', dest = 'disklog',
action = 'store_true', action = 'store_true',
help = ('If specified, log to a specific file (-Lf/--logfile)' + help = (
' instead of the system logger.')) 'If specified, log to a specific file '
'(-Lf/--logfile) instead of the system logger.'))
args.add_argument('-Lf', '--logfile', args.add_argument('-Lf', '--logfile',
dest = 'logfile', dest = 'logfile',
default = _logfile, default = _logfile,
help = ('The path to the logfile, only used if -Ld/--log-to-disk ' + help = (
'is specified. Default: \033[1m{0}\033[0m (dynamic)').format(_logfile)) 'The path to the logfile, only used if '
'-Ld/--log-to-disk is specified. '
'Default: \033[1m{0}\033[0m (dynamic)').format(
_logfile))
args.add_argument('-v', '--verbose', args.add_argument('-v', '--verbose',
dest = 'verbose', dest = 'verbose',
action = 'store_true', action = 'store_true',
help = ('If specified, log messages will be printed to STDERR ' + help = (
'in addition to the other configured log system(s), and verbosity for printing ' + 'If specified, log messages will be printed to '
'functions is increased. \033[1mWARNING:\033[0m This may display VERY sensitive information ' + 'STDERR in addition to the other configured log '
'such as passwords!')) 'system(s), and verbosity for printing '
'functions is increased. '
'\033[1mWARNING:\033[0m This may display VERY '
'sensitive information such as passwords!'))
### ARGS FOR ALL OPERATIONS ### ### ARGS FOR ALL OPERATIONS ###
commonargs = argparse.ArgumentParser(add_help = False) commonargs = argparse.ArgumentParser(add_help = False)
commonargs.add_argument('-r', '--repo', commonargs.add_argument('-r', '--repo',
dest = 'repo', dest = 'repo',
default = 'all', default = 'all',
help = ('The repository to perform the operation for. ' + help = (
'The default is \033[1mall\033[0m, a special value that specifies all known ' + 'The repository to perform the operation for. '
'repositories. Can also accept a comma-separated list.')) 'The default is \033[1mall\033[0m, a special '
'value that specifies all known repositories. '
'Can also accept a comma-separated list.'))
fileargs = argparse.ArgumentParser(add_help = False)
fileargs.add_argument('-a', '--archive',
default = _date,
dest = 'archive',
help = (
'The name of the archive/snapshot. '
'Default: \033[1m{0}\033[0m (dynamic)').format(
_date))
remoteargs = argparse.ArgumentParser(add_help = False) remoteargs = argparse.ArgumentParser(add_help = False)
remoteargs.add_argument('-d', '--dry-run', remoteargs.add_argument('-d', '--dry-run',
dest = 'dryrun', dest = 'dryrun',
action = 'store_true', action = 'store_true',
help = ('Act as if we are performing tasks, but none will actually be executed ' + help = ('Act as if we are performing tasks, but '
'none will actually be executed '
'(useful for testing logging)')) '(useful for testing logging)'))
### OPERATIONS ### ### OPERATIONS ###
subparsers = args.add_subparsers(help = 'Operation to perform', subparsers = args.add_subparsers(help = 'Operation to perform',
dest = 'oper') dest = 'oper')
backupargs = subparsers.add_parser('backup', backupargs = subparsers.add_parser('backup',
help = 'Perform a backup.', help = 'Perform a backup.',
parents = [commonargs, remoteargs]) parents = [commonargs,
remoteargs,
fileargs])
listargs = subparsers.add_parser('list', listargs = subparsers.add_parser('list',
help = 'List available backups.', help = 'List available backups.',
parents = [commonargs, remoteargs]) parents = [commonargs, remoteargs])
listrepoargs = subparsers.add_parser('listrepos', listrepoargs = subparsers.add_parser('listrepos',
help = 'List availabile/configured repositories.', help = ('List availabile/configured '
'repositories.'),
parents = [commonargs]) parents = [commonargs])
initargs = subparsers.add_parser('init', initargs = subparsers.add_parser('init',
help = 'Initialise a repository.', help = 'Initialise a repository.',
parents = [commonargs, remoteargs]) parents = [commonargs, remoteargs])
rstrargs = subparsers.add_parser('restore',
help = ('Restore ("extract") an '
'archive.'),
parents = [commonargs,
remoteargs,
fileargs])
### OPERATION-SPECIFIC OPTIONS ### ### OPERATION-SPECIFIC OPTIONS ###
# CREATE ("backup") # # CREATE ("backup") #
backupargs.add_argument('-a', backupargs.add_argument('-s', '--stagedir',
'--archive',
default = _date,
dest = 'archive',
help = ('The name of the archive. Default: \033[1m{0}\033[0m (dynamic)').format(_date))
backupargs.add_argument('-s',
'--stagedir',
default = _stagedir, default = _stagedir,
dest = 'stagedir', dest = 'stagedir',
help = ('The directory used for staging temporary files, ' + help = (
'if necessary. Default: \033[1m{0}\033[0m').format(_stagedir)) 'The directory used for staging '
backupargs.add_argument('-m', 'temporary files, if necessary. '
'--mysqldir', 'Default: \033[1m{0}\033[0m').format(
_stagedir))
backupargs.add_argument('-m', '--mysqldir',
default = _mysqldir, default = _mysqldir,
dest = 'mysqldir', dest = 'mysqldir',
help = ('The path to where MySQL dumps should go. ' + help = (
'Default: \033[1m{0}\033[0m').format(_mysqldir)) 'The path to where MySQL dumps should go. '
'Default: \033[1m{0}\033[0m').format(
_mysqldir))
# DISPLAY/OUTPUT ("list") # # DISPLAY/OUTPUT ("list") #
listargs.add_argument('-a', listargs.add_argument('-a', '--archive',
'--archive',
dest = 'archive', dest = 'archive',
default = False, default = False,
help = 'If specified, will list the *contents* of the given archive name.') help = 'If specified, will list the *contents* of '
listargs.add_argument('-l', 'the given archive name.')
'--limit', listargs.add_argument('-l', '--limit',
dest = 'numlimit', dest = 'numlimit',
type = int, type = int,
default = '5', default = '5',
help = ('If specified, constrain the outout to this number of ' + help = (
'results each repo. Default is \033[1m5\033[0m, use 0 for unlimited. ' + 'If specified, constrain the outout to this '
'See \033[1m-H/--list-help\033[0m')) 'number of results each repo. '
listargs.add_argument('-s', 'Default is \033[1m5\033[0m, use 0 for '
'--sort', 'unlimited. See '
'\033[1m-H/--list-help\033[0m'))
listargs.add_argument('-s', '--sort',
dest = 'sortby', dest = 'sortby',
choices = ['newest', 'oldest'], choices = ['newest', 'oldest'],
default = 'oldest', default = 'oldest',
help = ('The order to sort the results by. See \033[1m-H/--list-help\033[0m. ' + help = (
'The order to sort the results by. '
'See \033[1m-H/--list-help\033[0m. '
'Default: \033[1moldest\033[0m')) 'Default: \033[1moldest\033[0m'))
listargs.add_argument('-x', listargs.add_argument('-x', '--invert',
'--invert',
dest = 'invert', dest = 'invert',
action = 'store_true', action = 'store_true',
help = 'Invert the order of results. See \033[1m-H/--list-help\033[0m.') help = 'Invert the order of results. '
listargs.add_argument('-o', 'See \033[1m-H/--list-help\033[0m.')
'--old', listargs.add_argument('-o', '--old',
dest = 'old', dest = 'old',
action = 'store_true', action = 'store_true',
help = ('Instead of grabbing the latest results, grab the earliest results. ' + help = (
'This differs from \033[1m-s/--sort\033[0m. See \033[1m-H/--list-help\033[0m.')) 'Instead of grabbing the latest results, '
listargs.add_argument('-H', 'grab the earliest results. '
'--list-help', 'This differs from \033[1m-s/--sort\033[0m. '
'See \033[1m-H/--list-help\033[0m.'))
listargs.add_argument('-H', '--list-help',
dest = 'moarhelp', dest = 'moarhelp',
action = 'store_true', action = 'store_true',
help = 'Print extended information about how to manage the output of listing and exit.') help = ('Print extended information about how to '
return(args) 'manage the output of listing and exit.'))
## EXTRACT ("restore")
rstrargs.add_argument('-t', '--target',
required = True,
dest = 'target_dir',
help = ('The path to the directory where the '
'restore should be dumped to. It is '
'recommended to NOT restore to the same '
'directory that the archive is taken from.'))
return (args)



def main(): def main():
rawargs = parseArgs() rawargs = parseArgs()
args = vars(rawargs.parse_args()) parsedargs = rawargs.parse_args()
args = vars(parsedargs)
args['cfgfile'] = os.path.abspath(os.path.expanduser(args['cfgfile'])) args['cfgfile'] = os.path.abspath(os.path.expanduser(args['cfgfile']))
if not args['oper']: if not args['oper']:
rawargs.print_help() rawargs.print_help()
@ -543,7 +711,10 @@ def main():
bak.create() bak.create()
elif args['oper'] == 'init': elif args['oper'] == 'init':
bak.createRepo() bak.createRepo()
return() elif args['oper'] == 'restore':
bak.restore()
return ()


if __name__ == '__main__': if __name__ == '__main__':
main() main()