borg backup script added. ready for testing
This commit is contained in:
brent s
parent
33558610a6
commit
b4ee009465
524
storage/backups/borg/backup.py
Executable file
524
storage/backups/borg/backup.py
Executable file
@ -0,0 +1,524 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
# TODO: https://borgbackup.readthedocs.io/en/latest/internals/frontends.html
|
||||||
|
# will they EVER release a public API? for now we'll just use subprocess since
|
||||||
|
# we import it for various prep stuff anyways.
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import configparser
|
||||||
|
import datetime
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import logging.handlers
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
try:
|
||||||
|
import pymysql # not stdlib; "python-pymysql" in Arch's AUR
|
||||||
|
has_mysql = True
|
||||||
|
except ImportError:
|
||||||
|
has_mysql = False
|
||||||
|
try:
|
||||||
|
from systemd import journal
|
||||||
|
has_systemd = True
|
||||||
|
except ImportError:
|
||||||
|
has_systemd = False
|
||||||
|
|
||||||
|
### LOG LEVEL MAPPINGS ###
|
||||||
|
loglvls = {'critical': logging.CRITICAL,
|
||||||
|
'error': logging.ERROR,
|
||||||
|
'warning': logging.WARNING,
|
||||||
|
'info': logging.INFO,
|
||||||
|
'debug': logging.DEBUG}
|
||||||
|
|
||||||
|
### THE GUTS ###
|
||||||
|
class Backup(object):
|
||||||
|
def __init__(self, args):
|
||||||
|
self.args = args
|
||||||
|
### DIRECTORIES ###
|
||||||
|
if self.args['oper'] == 'backup':
|
||||||
|
for d in (self.args['mysqldir'], self.args['stagedir']):
|
||||||
|
os.makedirs(d, exist_ok = True, mode = 0o700)
|
||||||
|
### LOGGING ###
|
||||||
|
# Thanks to:
|
||||||
|
# https://web.archive.org/web/20170726052946/http://www.lexev.org/en/2013/python-logging-every-day/
|
||||||
|
# https://stackoverflow.com/a/42604392
|
||||||
|
# https://plumberjack.blogspot.com/2010/10/supporting-alternative-formatting.html
|
||||||
|
# and user K900_ on r/python for entertaining my very silly question.
|
||||||
|
self.logger = logging.getLogger(__name__)
|
||||||
|
self.logger.setLevel(loglvls[self.args['loglevel']])
|
||||||
|
_logfmt = logging.Formatter(fmt = '{levelname}:{name}: {message} ({asctime}; {filename}:{lineno})',
|
||||||
|
style = '{',
|
||||||
|
datefmt = '%Y-%m-%d %H:%M:%S')
|
||||||
|
_journalfmt = logging.Formatter(fmt = '{levelname}:{name}: {message} ({filename}:{lineno})',
|
||||||
|
style = '{',
|
||||||
|
datefmt = '%Y-%m-%d %H:%M:%S')
|
||||||
|
handlers = []
|
||||||
|
if self.args['disklog']:
|
||||||
|
os.makedirs(os.path.dirname(self.args['logfile']),
|
||||||
|
exist_ok = True,
|
||||||
|
mode = 0o700)
|
||||||
|
handlers.append(logging.handlers.RotatingFileHandler(self.args['logfile'],
|
||||||
|
encoding = 'utf8',
|
||||||
|
maxBytes = 100000,
|
||||||
|
backupCount = 1))
|
||||||
|
if self.args['verbose']:
|
||||||
|
handlers.append(logging.StreamHandler())
|
||||||
|
if has_systemd:
|
||||||
|
h = journal.JournalHandler()
|
||||||
|
h.setFormatter(_journalfmt)
|
||||||
|
h.setLevel(loglvls[self.args['loglevel']])
|
||||||
|
self.logger.addHandler(h)
|
||||||
|
for h in handlers:
|
||||||
|
h.setFormatter(_logfmt)
|
||||||
|
h.setLevel(loglvls[self.args['loglevel']])
|
||||||
|
self.logger.addHandler(h)
|
||||||
|
self.logger.debug('BEGIN INITIALIZATION')
|
||||||
|
### CONFIG ###
|
||||||
|
if not os.path.isfile(self.args['cfgfile']):
|
||||||
|
self.logger.error('{0} does not exist'.format(self.args['cfgfile']))
|
||||||
|
exit(1)
|
||||||
|
with open(self.args['cfgfile'], 'r') as f:
|
||||||
|
self.cfg = json.loads(f.read())
|
||||||
|
### END LOGGING ###
|
||||||
|
### ARGS CLEANUP ###
|
||||||
|
self.logger.debug('VARS (before args cleanup): {0}'.format(vars()))
|
||||||
|
self.args['repo'] = [i.strip() for i in self.args['repo'].split(',')]
|
||||||
|
if 'all' in self.args['repo']:
|
||||||
|
self.args['repo'] = list(self.cfg['repos'].keys())
|
||||||
|
for r in self.args['repo'][:]:
|
||||||
|
if r == 'all':
|
||||||
|
self.args['repo'].remove(r)
|
||||||
|
elif r not in self.cfg['repos'].keys():
|
||||||
|
self.logger.warning('Repository {0} is not configured; skipping.'.format(r))
|
||||||
|
self.args['repo'].remove(r)
|
||||||
|
self.logger.debug('VARS (after args cleanup): {0}'.format(vars()))
|
||||||
|
self.logger.debug('END INITIALIZATION')
|
||||||
|
### END INIT ###
|
||||||
|
|
||||||
|
def cmdExec(self, cmd, stdoutfh = None):
|
||||||
|
self.logger.debug('Running command: {0}'.format(' '.join(cmd)))
|
||||||
|
if self.args['dryryun']:
|
||||||
|
return() # no-op
|
||||||
|
if outfh:
|
||||||
|
_err = subprocess.run(cmd, stdout = stdoutfh, stderr = subprocess.PIPE).stderr.decode('utf-8').strip()
|
||||||
|
else:
|
||||||
|
with open(os.devnull, 'w') as DEVNULL:
|
||||||
|
_err = subprocess.run(cmd, stdout = DEVNULL, stderr = subprocess.PIPE).stderr.decode('utf-8').strip()
|
||||||
|
if _err != '':
|
||||||
|
self.logger.error('STDERR: {0} ({1})'.format(_out.stderr.decode('utf-8'),
|
||||||
|
' '.join(cmd)))
|
||||||
|
return()
|
||||||
|
|
||||||
|
def createRepo(self):
|
||||||
|
_env = os.environ.copy()
|
||||||
|
_env['BORG_RSH'] = self.cfg['config']['ctx']
|
||||||
|
for r in self.args['repo']:
|
||||||
|
self.logger.info('[{0}]: BEGIN INITIALIZATION'.format(r))
|
||||||
|
_cmd = ['borg',
|
||||||
|
'init',
|
||||||
|
'-v',
|
||||||
|
'{0}@{1}:{2}'.format(self.cfg['config']['user'],
|
||||||
|
self.cfg['config']['host'],
|
||||||
|
r)]
|
||||||
|
_env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password']
|
||||||
|
# We don't use self.cmdExec() here either because
|
||||||
|
# again, custom env, etc.
|
||||||
|
self.logger.debug('VARS: {0}'.format(vars()))
|
||||||
|
if not self.args['dryrun']:
|
||||||
|
_out = subprocess.run(_cmd,
|
||||||
|
env = _env,
|
||||||
|
stdout = subprocess.PIPE,
|
||||||
|
stderr = subprocess.PIPE)
|
||||||
|
_stdout = _out.stdout.decode('utf-8').strip()
|
||||||
|
_stderr = _out.stderr.decode('utf-8').strip()
|
||||||
|
self.logger.debug('[{0}]: (RESULT) {1}'.format(r, _stdout))
|
||||||
|
if _stderr != '':
|
||||||
|
self.logger.error('[{0}]: STDERR: {1} ({2})'.format(r,
|
||||||
|
_stderr,
|
||||||
|
' '.join(_cmd)))
|
||||||
|
del(_env['BORG_PASSPHRASE'])
|
||||||
|
self.logger.info('[{0}]: END INITIALIZATION'.format(r))
|
||||||
|
return()
|
||||||
|
|
||||||
|
def create(self):
|
||||||
|
_env = os.environ.copy()
|
||||||
|
_env['BORG_RSH'] = self.cfg['config']['ctx']
|
||||||
|
self.logger.info('START: backup')
|
||||||
|
for r in self.args['repo']:
|
||||||
|
self.logger.info('[{0}]: BEGIN BACKUP'.format(r))
|
||||||
|
if 'prep' in self.cfg['repos'][r].keys():
|
||||||
|
for prep in self.cfg['repos'][r]['prep']:
|
||||||
|
self.logger.info('[{0}]: Running prepfunc {1}'.format(r, prep))
|
||||||
|
eval('self.{0}'.format(prep)) # I KNOW, IT'S TERRIBLE. so sue me.
|
||||||
|
self.logger.info('[{0}]: Finished prepfunc {1}'.format(r, prep))
|
||||||
|
_cmd = ['borg',
|
||||||
|
'create',
|
||||||
|
'-v', '--stats',
|
||||||
|
'--compression', 'lzma,9']
|
||||||
|
if 'excludes' in self.cfg['repos'][r].keys():
|
||||||
|
for e in self.cfg['repos'][r]['excludes']:
|
||||||
|
_cmd.extend(['--exclude', e])
|
||||||
|
_cmd.append('{0}@{1}:{2}::{3}'.format(self.cfg['config']['user'],
|
||||||
|
self.cfg['config']['host'],
|
||||||
|
r,
|
||||||
|
self.args['archive']))
|
||||||
|
for p in cfg['repos'][r]['paths']:
|
||||||
|
_cmd.append(p)
|
||||||
|
_env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password']
|
||||||
|
self.logger.debug('VARS: {0}'.format(vars()))
|
||||||
|
# We don't use self.cmdExec() here because we want to explicitly pass the env
|
||||||
|
# and format the log line differently.
|
||||||
|
self.logger.debug('[{0}]: Running command: {1}'.format(r, ' '.join(cmd)))
|
||||||
|
if not self.args['dryrun']:
|
||||||
|
_out = subprocess.run(_cmd,
|
||||||
|
env = _env,
|
||||||
|
stdout = subprocess.PIPE,
|
||||||
|
stderr = subprocess.PIPE)
|
||||||
|
_stdout = _out.stdout.decode('utf-8').strip()
|
||||||
|
_stderr = _out.stderr.decode('utf-8').strip()
|
||||||
|
self.logger.debug('[{0}]: (RESULT) {1}'.format(r, _stdout))
|
||||||
|
if _stderr != '':
|
||||||
|
self.logger.error('[{0}]: STDERR: {1} ({2})'.format(r,
|
||||||
|
_stderr,
|
||||||
|
' '.join(_cmd)))
|
||||||
|
del(_env['BORG_PASSPHRASE'])
|
||||||
|
self.logger.info('[{0}]: END BACKUP'.format(r))
|
||||||
|
self.logger.info('END: backup')
|
||||||
|
return()
|
||||||
|
|
||||||
|
def miscBak(self, pkgr):
|
||||||
|
self.logger.info('BEGIN: miscBak()')
|
||||||
|
_cmd = None
|
||||||
|
for p in os.environ['PATH'].split(':'):
|
||||||
|
d = os.path.expanduser(p)
|
||||||
|
if os.path.isfile(os.path.join(d, pkgtool)):
|
||||||
|
_pkgr = pkgtool
|
||||||
|
self.logger.debug('Package tool found at {0}'.format(_pkgr))
|
||||||
|
else:
|
||||||
|
_pkgr = 'pacman'
|
||||||
|
self.logger.debug('Using {0} as package tool'.format(_pkgr))
|
||||||
|
with open(os.path.join(self.args['stagedir'], 'pkg.lst'), 'w') as f:
|
||||||
|
_cmd = [_pkgr,
|
||||||
|
'-Qet',
|
||||||
|
'--color',
|
||||||
|
'never']
|
||||||
|
self.cmdExec(_cmd, stdoutfh = f)
|
||||||
|
self.logger.info('END: miscBak()')
|
||||||
|
return()
|
||||||
|
|
||||||
|
def mysqlBak(self):
|
||||||
|
self.logger.info('BEGIN: mysqlBak()')
|
||||||
|
if not has_mysql:
|
||||||
|
self.logger.error('You need to install the PyMySQL module to back up MySQL databases. Skipping.')
|
||||||
|
return()
|
||||||
|
# These are mysqldump options shared by ALL databases
|
||||||
|
_mysqlopts = ['--routines',
|
||||||
|
'--add-drop-database',
|
||||||
|
'--add-drop-table',
|
||||||
|
'--allow-keywords',
|
||||||
|
'--complete-insert',
|
||||||
|
'--create-options',
|
||||||
|
'--extended-insert']
|
||||||
|
_DBs = []
|
||||||
|
_mycnf = os.path.expanduser(os.path.join('~', '.my.cnf'))
|
||||||
|
if not os.path.isfile(_mycnf):
|
||||||
|
exit('{0}: ERROR: Cannot get credentials for MySQL (cannot find ~/.my.cnf)!')
|
||||||
|
_mycfg = configparser.ConfigParser()
|
||||||
|
_mycfg._interpolation = configparser.ExtendedInterpolation()
|
||||||
|
_mycfg.read(_mycnf)
|
||||||
|
_sqlcfg = {s:dict(_mycfg.items(s)) for s in _mycfg.sections()}
|
||||||
|
if 'host' not in _sqlcfg.keys():
|
||||||
|
_socketpath = '/var/run/mysqld/mysqld.sock' # correct for Arch, YMMV.
|
||||||
|
_mysql = pymysql.connect(unix_socket = _socketpath,
|
||||||
|
user = _sqlcfg['client']['user'],
|
||||||
|
passwd = _sqlcfg['client']['password'])
|
||||||
|
else:
|
||||||
|
_mysql = pymysql.connect(host = _sqlcfg['client']['host'],
|
||||||
|
user = _sqlcfg['client']['user'],
|
||||||
|
port = _sqlcfg['client']['port'],
|
||||||
|
passwd = _sqlcfg['client']['password'])
|
||||||
|
_cur = _mysql.cursor()
|
||||||
|
_cur.execute('SHOW DATABASES')
|
||||||
|
for row in _cur.fetchall():
|
||||||
|
_DBs.append(row[0])
|
||||||
|
self.logger.debug('Databases: {0}'.format(', '.join(_DBs)))
|
||||||
|
for db in _DBs:
|
||||||
|
_cmd = ['mysqldump',
|
||||||
|
'--result-file={0}.sql'.format(os.path.join(mysqldir, db))]
|
||||||
|
# These are database-specific options
|
||||||
|
if db in ('information_schema', 'performance_schema'):
|
||||||
|
_cmd.append('--skip-lock-tables')
|
||||||
|
elif db == 'mysql':
|
||||||
|
_cmd.append('--flush-privileges')
|
||||||
|
_cmd.extend(_mysqlopts)
|
||||||
|
_cmd.append(db)
|
||||||
|
self.cmdExec(_cmd)
|
||||||
|
self.logger.info('END: mysqlBak()')
|
||||||
|
return()
|
||||||
|
|
||||||
|
def listRepos(self):
|
||||||
|
print('\n\033[1mCurrently configured repositories are:\033[0m\n')
|
||||||
|
print('\t{0}\n'.format(', '.join(self.cfg['repos'].keys())))
|
||||||
|
if self.args['verbose']:
|
||||||
|
print('\033[1mDETAILS:\033[0m\n')
|
||||||
|
for r in self.args['repo']:
|
||||||
|
print('\t\033[1m{0}:\033[0m\n\t\t\033[1mPath(s):\033[0m\t'.format(r.upper()), end = '')
|
||||||
|
for p in self.cfg['repos'][r]['paths']:
|
||||||
|
print(p, end = ' ')
|
||||||
|
if 'prep' in self.cfg['repos'][r].keys():
|
||||||
|
print('\n\t\t\033[1mPrep:\033[0m\t\t', end = '')
|
||||||
|
for p in self.cfg['repos'][r]['prep']:
|
||||||
|
print(p, end = ' ')
|
||||||
|
if 'excludes' in self.cfg['repos'][r].keys():
|
||||||
|
print('\n\t\t\033[1mExclude(s):\033[0m\t', end = '')
|
||||||
|
for p in self.cfg['repos'][r]['excludes']:
|
||||||
|
print(p, end = ' ')
|
||||||
|
print('\n')
|
||||||
|
return()
|
||||||
|
|
||||||
|
def printer(self):
|
||||||
|
_results = self.lister()
|
||||||
|
if not self.args['archive']: # It's a listing of archives
|
||||||
|
print('\033[1mREPO:\tSNAPSHOT:\t\tTIMESTAMP:\033[0m\n')
|
||||||
|
for r in _results.keys():
|
||||||
|
print(r, end = '')
|
||||||
|
for line in _results[r]:
|
||||||
|
_snapshot = line.split()
|
||||||
|
print('\t\033[1m{0}\033[0m\t\t{1}'.format(_snapshot[0], ' '.join(_snapshot[1:])))
|
||||||
|
print()
|
||||||
|
else: # It's a listing inside an archive
|
||||||
|
if self.args['verbose']:
|
||||||
|
fields = ['REPO:', 'PERMS:', 'OWNERSHIP:', 'SIZE:', 'TIMESTAMP:', 'PATH:']
|
||||||
|
for r in results.keys():
|
||||||
|
print('\033[1m{0}\t{1}\033[0m'.format(fields[0], r))
|
||||||
|
# https://docs.python.org/3/library/string.html#formatspec
|
||||||
|
print('{0[1]:<15}\t{0[2]:<15}\t{0[3]:<15}\t{0[4]:<24}\t{0[5]:<15}'.format(fields))
|
||||||
|
for line in _results[r]:
|
||||||
|
_fline = line.split()
|
||||||
|
_perms = fline[0]
|
||||||
|
_ownership = '{0}:{1}'.format(fline[1], fline[2])
|
||||||
|
_size = fline[3]
|
||||||
|
_time = ' '.join(fline[4:7])
|
||||||
|
_path = ' '.join(fline[7:])
|
||||||
|
print('{0:<15}\t{1:<15}\t{2:<15}\t{3:<24}\t{4:<15}'.format(_perms,
|
||||||
|
_ownership,
|
||||||
|
_size,
|
||||||
|
_time,
|
||||||
|
_path))
|
||||||
|
else:
|
||||||
|
print('\033[1mREPO:\tPATH:\033[0m\n')
|
||||||
|
for r in _results.keys():
|
||||||
|
print(r, end = '')
|
||||||
|
for line in _results[r]:
|
||||||
|
_fline = line.split()
|
||||||
|
print('\t{0}'.format(' '.join(_fline[7:])))
|
||||||
|
return()
|
||||||
|
|
||||||
|
def lister(self):
|
||||||
|
output = {}
|
||||||
|
_env = os.environ.copy()
|
||||||
|
self.logger.info('START: lister')
|
||||||
|
_env['BORG_RSH'] = self.cfg['config']['ctx']
|
||||||
|
for r in self.args['repo']:
|
||||||
|
if self.args['archive']:
|
||||||
|
_cmd = ['borg',
|
||||||
|
'list',
|
||||||
|
'{0}@{1}:{2}::{3}'.format(self.cfg['config']['user'],
|
||||||
|
self.cfg['config']['host'],
|
||||||
|
r,
|
||||||
|
self.args['archive'])]
|
||||||
|
else:
|
||||||
|
_cmd = ['borg',
|
||||||
|
'list',
|
||||||
|
'{0}@{1}:{2}'.format(self.cfg['config']['user'],
|
||||||
|
self.cfg['config']['host'],
|
||||||
|
r)]
|
||||||
|
_env['BORG_PASSPHRASE'] = self.cfg['repos'][r]['password']
|
||||||
|
if not self.args['dryrun']:
|
||||||
|
|
||||||
|
_out = subprocess.run(_cmd,
|
||||||
|
env = _env,
|
||||||
|
stdout = subprocess.PIPE,
|
||||||
|
stderr = subprocess.PIPE)
|
||||||
|
_stdout = [i.strip() for i in _out.stdout.decode('utf-8').splitlines()]
|
||||||
|
_stderr = _out.stderr.decode('utf-8').strip()
|
||||||
|
output[r] = _stdout
|
||||||
|
self.logger.debug('[{0}]: (RESULT) {1}'.format(r,
|
||||||
|
'\n'.join(_stdout)))
|
||||||
|
if _stderr != '':
|
||||||
|
self.logger.error('[{0}]: STDERR: {1} ({2})'.format(r,
|
||||||
|
_stderr,
|
||||||
|
' '.join(_cmd)))
|
||||||
|
del(_env['BORG_PASSPHRASE'])
|
||||||
|
if not self.args['archive']:
|
||||||
|
if self.args['numlimit'] > 0:
|
||||||
|
if self.args['old']:
|
||||||
|
output[r] = output[r][:self.args['numlimit']]
|
||||||
|
else:
|
||||||
|
output[r] = list(reversed(output[r]))[:self.args['numlimit']]
|
||||||
|
if self.args['invert']:
|
||||||
|
output[r] = reversed(output[r])
|
||||||
|
self.logger.info('END: lister')
|
||||||
|
return(output)
|
||||||
|
|
||||||
|
def printMoarHelp():
|
||||||
|
_helpstr = ('\n\tNOTE: Sorting only applies to listing archives, NOT the contents!\n\n' +
|
||||||
|
'In order to efficiently display results, there are several options to handle it. ' +
|
||||||
|
'Namely, these are:\n\n\t\t-s/--sort [direction]\n\t\t-l/--limit [number]\n\t\t-x/--invert\n\n' +
|
||||||
|
'For example, if you want to list the 5 most recently *taken* snapshots, you would use:\n\n\t\t-l 5\n\n' +
|
||||||
|
'If you would want those SAME results SORTED in the reverse order (i.e. the 5 most recently ' +
|
||||||
|
'taken snapshots sorted from newest to oldest), then it would be: \n\n\t\t-l 5 -x\n\n' +
|
||||||
|
'Lastly, if you wanted to list the 7 OLDEST TAKEN snapshots in reverse order (that is, ' +
|
||||||
|
'sorted from newest to oldest), that\'d be: \n\n\t\t-o -l 7 -x\n')
|
||||||
|
print(_helpstr)
|
||||||
|
exit(0)
|
||||||
|
|
||||||
|
def parseArgs():
|
||||||
|
### DEFAULTS ###
|
||||||
|
_date = datetime.datetime.now().strftime("%Y_%m_%d.%H_%M")
|
||||||
|
_logfile = '/var/log/borg/{0}'.format(_date)
|
||||||
|
_mysqldir = os.path.abspath(os.path.join(os.path.expanduser('~'), 'bak', 'mysql'))
|
||||||
|
_stagedir = os.path.abspath(os.path.join(os.path.expanduser('~'), '.bak', 'misc'))
|
||||||
|
_cfgfile = os.path.abspath(os.path.join(os.path.expanduser('~'), '.config', 'optools', 'backup.json'))
|
||||||
|
_defloglvl = 'info'
|
||||||
|
######
|
||||||
|
args = argparse.ArgumentParser(description = 'Backups manager',
|
||||||
|
epilog = 'TIP: this program has context-specific help. e.g. try "%(prog)s list --help"')
|
||||||
|
args.add_argument('-c', '--config',
|
||||||
|
dest = 'cfgfile',
|
||||||
|
default = _cfgfile,
|
||||||
|
help = ('The path to the config file. Default: \033[1m{0}\033[0m'.format(_cfgfile)))
|
||||||
|
args.add_argument('-Ll', '--loglevel',
|
||||||
|
dest = 'loglevel',
|
||||||
|
default = _defloglvl,
|
||||||
|
choices = list(loglvls.keys()),
|
||||||
|
help = ('The level of logging to perform. \033[1mWARNING:\033[0m \033[1mdebug\033[0m will log ' +
|
||||||
|
'VERY sensitive information such as passwords! Default: \033[1m{0}\033[0m'.format(_defloglvl)))
|
||||||
|
args.add_argument('-Ld', '--log-to-disk',
|
||||||
|
dest = 'disklog',
|
||||||
|
action = 'store_true',
|
||||||
|
help = ('If specified, log to a specific file (-Lf/--logfile)' +
|
||||||
|
' instead of the system logger.'))
|
||||||
|
args.add_argument('-Lf', '--logfile',
|
||||||
|
dest = 'logfile',
|
||||||
|
default = _logfile,
|
||||||
|
help = ('The path to the logfile, only used if -Ld/--log-to-disk ' +
|
||||||
|
'is specified. Default: \033[1m{0}\033[0m (dynamic)').format(_logfile))
|
||||||
|
args.add_argument('-v', '--verbose',
|
||||||
|
dest = 'verbose',
|
||||||
|
action = 'store_true',
|
||||||
|
help = ('If specified, log messages will be printed to STDERR ' +
|
||||||
|
'in addition to the other configured log system(s), and verbosity for printing ' +
|
||||||
|
'functions is increased. \033[1mWARNING:\033[0m This may display VERY sensitive information ' +
|
||||||
|
'such as passwords!'))
|
||||||
|
### ARGS FOR ALL OPERATIONS ###
|
||||||
|
commonargs = argparse.ArgumentParser(add_help = False)
|
||||||
|
commonargs.add_argument('-r', '--repo',
|
||||||
|
dest = 'repo',
|
||||||
|
default = 'all',
|
||||||
|
help = ('The repository to perform the operation for. ' +
|
||||||
|
'The default is \033[1mall\033[0m, a special value that specifies all known ' +
|
||||||
|
'repositories. Can also accept a comma-separated list.'))
|
||||||
|
remoteargs = argparse.ArgumentParser(add_help = False)
|
||||||
|
remoteargs.add_argument('-d', '--dry-run',
|
||||||
|
dest = 'dryrun',
|
||||||
|
action = 'store_true',
|
||||||
|
help = ('Act as if we are performing tasks, but none will actually be executed ' +
|
||||||
|
'(useful for testing logging)'))
|
||||||
|
### OPERATIONS ###
|
||||||
|
subparsers = args.add_subparsers(help = 'Operation to perform',
|
||||||
|
dest = 'oper')
|
||||||
|
backupargs = subparsers.add_parser('backup',
|
||||||
|
help = 'Perform a backup.',
|
||||||
|
parents = [commonargs, remoteargs])
|
||||||
|
listargs = subparsers.add_parser('list',
|
||||||
|
help = 'List available backups.',
|
||||||
|
parents = [commonargs, remoteargs])
|
||||||
|
listrepoargs = subparsers.add_parser('listrepos',
|
||||||
|
help = 'List availabile/configured repositories.',
|
||||||
|
parents = [commonargs])
|
||||||
|
initargs = subparsers.add_parser('init',
|
||||||
|
help = 'Initialise a repository.',
|
||||||
|
parents = [commonargs, remoteargs])
|
||||||
|
### OPERATION-SPECIFIC OPTIONS ###
|
||||||
|
# CREATE ("backup") #
|
||||||
|
backupargs.add_argument('-a',
|
||||||
|
'--archive',
|
||||||
|
default = _date,
|
||||||
|
dest = 'archive',
|
||||||
|
help = ('The name of the archive. Default: \033[1m{0}\033[0m (dynamic)').format(_date))
|
||||||
|
backupargs.add_argument('-s',
|
||||||
|
'--stagedir',
|
||||||
|
default = _stagedir,
|
||||||
|
dest = 'stagedir',
|
||||||
|
help = ('The directory used for staging temporary files, ' +
|
||||||
|
'if necessary. Default: \033[1m{0}\033[0m').format(_stagedir))
|
||||||
|
backupargs.add_argument('-m',
|
||||||
|
'--mysqldir',
|
||||||
|
default = _mysqldir,
|
||||||
|
dest = 'mysqldir',
|
||||||
|
help = ('The path to where MySQL dumps should go. ' +
|
||||||
|
'Default: \033[1m{0}\033[0m').format(_mysqldir))
|
||||||
|
# DISPLAY/OUTPUT ("list") #
|
||||||
|
listargs.add_argument('-a',
|
||||||
|
'--archive',
|
||||||
|
dest = 'archive',
|
||||||
|
default = False,
|
||||||
|
help = 'If specified, will list the *contents* of the given archive name.')
|
||||||
|
listargs.add_argument('-l',
|
||||||
|
'--limit',
|
||||||
|
dest = 'numlimit',
|
||||||
|
type = int,
|
||||||
|
default = '5',
|
||||||
|
help = ('If specified, constrain the outout to this number of ' +
|
||||||
|
'results each repo. Default is \033[1m5\033[0m, use 0 for unlimited. ' +
|
||||||
|
'See \033[1m-H/--list-help\033[0m'))
|
||||||
|
listargs.add_argument('-s',
|
||||||
|
'--sort',
|
||||||
|
dest = 'sortby',
|
||||||
|
choices = ['newest', 'oldest'],
|
||||||
|
default = 'oldest',
|
||||||
|
help = ('The order to sort the results by. See \033[1m-H/--list-help\033[0m. ' +
|
||||||
|
'Default: \033[1moldest\033[0m'))
|
||||||
|
listargs.add_argument('-x',
|
||||||
|
'--invert',
|
||||||
|
dest = 'invert',
|
||||||
|
action = 'store_true',
|
||||||
|
help = 'Invert the order of results. See \033[1m-H/--list-help\033[0m.')
|
||||||
|
listargs.add_argument('-o',
|
||||||
|
'--old',
|
||||||
|
dest = 'old',
|
||||||
|
action = 'store_true',
|
||||||
|
help = ('Instead of grabbing the latest results, grab the earliest results. ' +
|
||||||
|
'This differs from \033[1m-s/--sort\033[0m. See \033[1m-H/--list-help\033[0m.'))
|
||||||
|
listargs.add_argument('-H',
|
||||||
|
'--list-help',
|
||||||
|
dest = 'moarhelp',
|
||||||
|
action = 'store_true',
|
||||||
|
help = 'Print extended information about how to manage the output of listing and exit.')
|
||||||
|
return(args)
|
||||||
|
|
||||||
|
def main():
|
||||||
|
rawargs = parseArgs()
|
||||||
|
args = vars(rawargs.parse_args())
|
||||||
|
args['cfgfile'] = os.path.abspath(os.path.expanduser(args['cfgfile']))
|
||||||
|
if not args['oper']:
|
||||||
|
rawargs.print_help()
|
||||||
|
exit(0)
|
||||||
|
if 'moarhelp' in args.keys() and args['moarhelp']:
|
||||||
|
printMoarHelp()
|
||||||
|
# The "Do stuff" part
|
||||||
|
bak = Backup(args)
|
||||||
|
if args['oper'] == 'list':
|
||||||
|
bak.printer()
|
||||||
|
elif args['oper'] == 'listrepos':
|
||||||
|
bak.listRepos()
|
||||||
|
elif args['oper'] == 'backup':
|
||||||
|
bak.createBak()
|
||||||
|
elif args['oper'] == 'init':
|
||||||
|
bak.createRepo()
|
||||||
|
return()
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
@ -1,224 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
|
|
||||||
# TODO: https://borgbackup.readthedocs.io/en/latest/internals/frontends.html
|
|
||||||
# will they EVER release a public API? for now we'll just use subprocess since
|
|
||||||
# we import it for various prep stuff anyways.
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import configparser
|
|
||||||
import datetime
|
|
||||||
import json
|
|
||||||
import logging
|
|
||||||
import logging.handlers
|
|
||||||
import os
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
try:
|
|
||||||
import pymysql # not stdlib; "python-pymysql" in Arch's AUR
|
|
||||||
has_mysql = True
|
|
||||||
except ImportError:
|
|
||||||
has_mysql = False
|
|
||||||
try:
|
|
||||||
from systemd import journal
|
|
||||||
has_systemd = True
|
|
||||||
except ImportError:
|
|
||||||
has_systemd = False
|
|
||||||
|
|
||||||
### LOG LEVEL MAPPINGS ###
|
|
||||||
loglvls = {'critical': logging.CRITICAL,
|
|
||||||
'error': logging.ERROR,
|
|
||||||
'warning': logging.WARNING,
|
|
||||||
'info': logging.INFO,
|
|
||||||
'debug': logging.DEBUG}
|
|
||||||
|
|
||||||
### THE GUTS ###
|
|
||||||
class Backup(object):
|
|
||||||
def __init__(self, args):
|
|
||||||
self.args = args
|
|
||||||
### LOGGING ###
|
|
||||||
# Thanks to:
|
|
||||||
# https://web.archive.org/web/20170726052946/http://www.lexev.org/en/2013/python-logging-every-day/
|
|
||||||
# https://stackoverflow.com/a/42604392
|
|
||||||
# https://plumberjack.blogspot.com/2010/10/supporting-alternative-formatting.html
|
|
||||||
# and user K900_ on r/python for entertaining my very silly question.
|
|
||||||
self.logger = logging.getLogger(__name__)
|
|
||||||
self.logger.setLevel(loglvls[self.args['loglevel']])
|
|
||||||
_logfmt = logging.Formatter(fmt = '{levelname}:{name}: {message} ({asctime}; {filename}:{lineno})',
|
|
||||||
style = '{',
|
|
||||||
datefmt = '%Y-%m-%d %H:%M:%S')
|
|
||||||
_journalfmt = logging.Formatter(fmt = '{levelname}:{name}: {message} ({filename}:{lineno})',
|
|
||||||
style = '{',
|
|
||||||
datefmt = '%Y-%m-%d %H:%M:%S')
|
|
||||||
handlers = []
|
|
||||||
if self.args['disklog']:
|
|
||||||
handlers.append(logging.handlers.RotatingFileHandler(self.args['logfile'],
|
|
||||||
encoding = 'utf8',
|
|
||||||
maxBytes = 100000,
|
|
||||||
backupCount = 1))
|
|
||||||
if self.args['verbose']:
|
|
||||||
handlers.append(logging.StreamHandler())
|
|
||||||
if has_systemd:
|
|
||||||
handlers.append(journal.JournalHandler())
|
|
||||||
for h in handlers:
|
|
||||||
h.setFormatter(_logfmt)
|
|
||||||
h.setLevel(loglvls[self.args['loglevel']])
|
|
||||||
self.logger.addHandler(h)
|
|
||||||
### CONFIG ###
|
|
||||||
if not os.path.isfile(self.args['cfgfile']):
|
|
||||||
self.logger.error('{0} does not exist'.format(self.args['cfgfile']))
|
|
||||||
exit(1)
|
|
||||||
|
|
||||||
def create(self):
|
|
||||||
pass
|
|
||||||
|
|
||||||
def miscBak(self, pkgr):
|
|
||||||
pass
|
|
||||||
|
|
||||||
def mysqlBak(self):
|
|
||||||
pass
|
|
||||||
|
|
||||||
def listRepos(self):
|
|
||||||
pass
|
|
||||||
|
|
||||||
def printer(self):
|
|
||||||
pass
|
|
||||||
|
|
||||||
def lister(self):
|
|
||||||
pass
|
|
||||||
|
|
||||||
def printMoarHelp():
|
|
||||||
helpstr = ('\n\tNOTE: Sorting only applies to listing archives, NOT the contents!\n\n' +
|
|
||||||
'In order to efficiently display results, there are several options to handle it. ' +
|
|
||||||
'Namely, these are:\n\n\t\t-s/--sort [direction]\n\t\t-l/--limit [number]\n\t\t-x/--invert\n\n' +
|
|
||||||
'For example, if you want to list the 5 most recently *taken* snapshots, you would use:\n\n\t\t-l 5\n\n' +
|
|
||||||
'If you would want those SAME results SORTED in the reverse order (i.e. the 5 most recently ' +
|
|
||||||
'taken snapshots sorted from newest to oldest), then it would be: \n\n\t\t-l 5 -x\n\n' +
|
|
||||||
'Lastly, if you wanted to list the 7 OLDEST TAKEN snapshots in reverse order (that is, ' +
|
|
||||||
'sorted from newest to oldest), that\'d be: \n\n\t\t-o -l 7 -x\n')
|
|
||||||
print(helpstr)
|
|
||||||
exit(0)
|
|
||||||
|
|
||||||
def parseArgs():
|
|
||||||
### DEFAULTS ###
|
|
||||||
_date = datetime.datetime.now().strftime("%Y_%m_%d.%H_%M")
|
|
||||||
_logfile = '/var/log/borg/{0}'.format(_date)
|
|
||||||
_mysqldir = os.path.abspath(os.path.join(os.path.expanduser('~'), 'bak', 'mysql'))
|
|
||||||
_stagedir = '/root/bak/misc'
|
|
||||||
_cfgfile = os.path.abspath(os.path.join(os.path.expanduser('~'), '.config', 'optools', 'backup.json'))
|
|
||||||
_defloglvl = 'info'
|
|
||||||
######
|
|
||||||
args = argparse.ArgumentParser(description = 'Backups manager',
|
|
||||||
epilog = 'TIP: this program has context-specific help. e.g. try "%(prog)s list --help"')
|
|
||||||
args.add_argument('-c', '--config',
|
|
||||||
dest = 'cfgfile',
|
|
||||||
default = _cfgfile,
|
|
||||||
help = ('The path to the config file. Default: \033[1m{0}\033[0m'.format(_cfgfile)))
|
|
||||||
args.add_argument('-Ll', '--loglevel',
|
|
||||||
dest = 'loglevel',
|
|
||||||
default = _defloglvl,
|
|
||||||
choices = list(loglvls.keys()),
|
|
||||||
help = 'The level of logging to perform. Default: \033[1m{0}\033[0m'.format(_defloglvl))
|
|
||||||
args.add_argument('-Ld', '--log-to-disk',
|
|
||||||
dest = 'disklog',
|
|
||||||
action = 'store_true',
|
|
||||||
help = ('If specified, log to a specific file (-Lf/--logfile)' +
|
|
||||||
' instead of the system logger.'))
|
|
||||||
args.add_argument('-Lf', '--logfile',
|
|
||||||
dest = 'logfile',
|
|
||||||
default = _logfile,
|
|
||||||
help = ('The path to the logfile, only used if -Ld/--log-to-disk ' +
|
|
||||||
'is specified. Default: \033[1m{0}\033[0m (dynamic)').format(_logfile))
|
|
||||||
args.add_argument('-v', '--verbose',
|
|
||||||
dest = 'verbose',
|
|
||||||
action = 'store_true',
|
|
||||||
help = ('If specified, log messages will be printed to STDOUT/STDERR ' +
|
|
||||||
'in addition to the other configured log systems.'))
|
|
||||||
### ARGS FOR ALL OPERATIONS ###
|
|
||||||
commonargs = argparse.ArgumentParser(add_help = False)
|
|
||||||
commonargs.add_argument('-r', '--repo',
|
|
||||||
dest = 'repo',
|
|
||||||
default = 'all',
|
|
||||||
help = ('The repository to perform the operation for. ' +
|
|
||||||
'The default is \033[1mall\033[0m, a special value that specifies all known ' +
|
|
||||||
'repositories. Can also accept a comma-separated list.'))
|
|
||||||
### OPERATIONS ###
|
|
||||||
subparsers = args.add_subparsers(help = 'Operation to perform',
|
|
||||||
dest = 'oper')
|
|
||||||
backupargs = subparsers.add_parser('backup',
|
|
||||||
help = 'Perform a backup.',
|
|
||||||
parents = [commonargs])
|
|
||||||
listargs = subparsers.add_parser('list',
|
|
||||||
help = 'List available backups.',
|
|
||||||
parents = [commonargs])
|
|
||||||
listrepoargs = subparsers.add_parser('listrepos',
|
|
||||||
help = 'List availabile/configured repositories.',
|
|
||||||
parents = [commonargs])
|
|
||||||
initargs = subparsers.add_parser('init',
|
|
||||||
help = 'Initialise a repository.',
|
|
||||||
parents = [commonargs])
|
|
||||||
### OPERATION-SPECIFIC OPTIONS ###
|
|
||||||
# CREATE ("backup") #
|
|
||||||
backupargs.add_argument('-a',
|
|
||||||
'--archive',
|
|
||||||
default = _date,
|
|
||||||
dest = 'archive',
|
|
||||||
help = ('The name of the archive. Default: \033[1m{0}\033[0m (dynamic)').format(_date))
|
|
||||||
backupargs.add_argument('-s',
|
|
||||||
'--stagedir',
|
|
||||||
default = _stagedir,
|
|
||||||
dest = 'stagedir',
|
|
||||||
help = ('The directory used for staging temporary files, ' +
|
|
||||||
'if necessary. Default: \033[1m{0}\033[0m').format(_stagedir))
|
|
||||||
# DISPLAY/OUTPUT ("list") #
|
|
||||||
listargs.add_argument('-a',
|
|
||||||
'--archive',
|
|
||||||
dest = 'archive',
|
|
||||||
default = False,
|
|
||||||
help = 'If specified, will list the *contents* of a certain archive.')
|
|
||||||
listargs.add_argument('-l',
|
|
||||||
'--limit',
|
|
||||||
dest = 'numlimit',
|
|
||||||
type = int,
|
|
||||||
default = '5',
|
|
||||||
help = ('If specified, constrain the outout to this number of ' +
|
|
||||||
'results each repo. Default is \033[1m5\033[0m, use 0 for unlimited. ' +
|
|
||||||
'See \033[1m-H/--list-help\033[0m'))
|
|
||||||
listargs.add_argument('-s',
|
|
||||||
'--sort',
|
|
||||||
dest = 'sortby',
|
|
||||||
choices = ['newest', 'oldest'],
|
|
||||||
default = 'oldest',
|
|
||||||
help = ('The order to sort the results by. See \033[1m-H/--list-help\033[0m. ' +
|
|
||||||
'Default: \033[1moldest\033[0m'))
|
|
||||||
listargs.add_argument('-x',
|
|
||||||
'--invert',
|
|
||||||
dest = 'invert',
|
|
||||||
action = 'store_true',
|
|
||||||
help = 'Invert the order of results. See \033[1m-H/--list-help\033[0m.')
|
|
||||||
listargs.add_argument('-o',
|
|
||||||
'--old',
|
|
||||||
dest = 'old',
|
|
||||||
action = 'store_true',
|
|
||||||
help = ('Instead of grabbing the latest results, grab the earliest results. ' +
|
|
||||||
'This differs from \033[1m-s/--sort\033[0m. See \033[1m-H/--list-help\033[0m.'))
|
|
||||||
listargs.add_argument('-H',
|
|
||||||
'--list-help',
|
|
||||||
dest = 'moarhelp',
|
|
||||||
action = 'store_true',
|
|
||||||
help = 'Print extended information about how to manage the output of listing and exit.')
|
|
||||||
return(args)
|
|
||||||
|
|
||||||
def main():
|
|
||||||
rawargs = parseArgs()
|
|
||||||
args = vars(rawargs.parse_args())
|
|
||||||
import pprint # DEBUG
|
|
||||||
pprint.pprint(args) # DEBUG
|
|
||||||
if not args['oper']:
|
|
||||||
rawargs.print_help()
|
|
||||||
exit(0)
|
|
||||||
if 'moarhelp' in args.keys() and args['moarhelp']:
|
|
||||||
printMoarHelp()
|
|
||||||
bak = Backup(args)
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
35
storage/backups/borg/sample.config.json
Normal file
35
storage/backups/borg/sample.config.json
Normal file
@ -0,0 +1,35 @@
|
|||||||
|
# A normal config would be placed in the current user's come directory,
|
||||||
|
# typically under ~/.config/optools/backup.json
|
||||||
|
|
||||||
|
{
|
||||||
|
"repos": {
|
||||||
|
"web": {
|
||||||
|
"paths": ["/srv/http"],
|
||||||
|
"password": "PASSWORD1", # OBVIOUSLY, change all these.
|
||||||
|
"excludes": ["/srv/http/sks/dumps"] # you can specify paths to exclude.
|
||||||
|
},
|
||||||
|
"etc": {
|
||||||
|
"paths": ["/etc"],
|
||||||
|
"password": "PASSWORD2"
|
||||||
|
},
|
||||||
|
"home": {
|
||||||
|
"paths": ["/home"],
|
||||||
|
"password": "PASSWORD3"
|
||||||
|
},
|
||||||
|
"misc": {
|
||||||
|
"paths": ["/usr/local/bin", "/var/lib/awstats", "/var/spool/anacron", "/var/spool/cron", "/usr/local/bin", "/root/.bak/misc"],
|
||||||
|
"prep": ["miscBak('apacman')"], # i'm... thinking of ways to make this more flexible, but for now it supports pacman/apacman
|
||||||
|
"password": "PASSWORD4"
|
||||||
|
},
|
||||||
|
"root": {
|
||||||
|
"paths": ["/root"],
|
||||||
|
"password": "PASSWORD5",
|
||||||
|
"excludes": ["/root/.cache", "/root/.bak", "*.sql*"]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"config": {
|
||||||
|
"user": "username", # the username on your backup box
|
||||||
|
"host": "backups.domain.tld", # where the borg server is
|
||||||
|
"ctx": "ssh -p 1234" # if you run a borg-specific SSHD on an alternate port
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user