minor updates
This commit is contained in:
brent s
parent
3d537e42bc
commit
ea414ca5b7
174
gpg/sksdump.py
174
gpg/sksdump.py
@ -3,6 +3,7 @@
|
||||
# Thanks to Matt Rude and https://gist.github.com/mattrude/b0ac735d07b0031bb002 so I can know what the hell I'm doing.
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import configparser
|
||||
import datetime
|
||||
import getpass
|
||||
@ -31,61 +32,75 @@ def getDefaults():
|
||||
'sync': {'throttle': 0},
|
||||
'paths': {'basedir': '/var/lib/sks',
|
||||
'destdir': '/srv/http/sks/dumps',
|
||||
'rsync': 'root@mirror.square-r00t.net:/srv/http/sks/dumps',
|
||||
'rsync': ('root@mirror.square-r00t.net:' +
|
||||
'/srv/http/sks/dumps'),
|
||||
'sksbin': '/usr/bin/sks'},
|
||||
'runtime': {'nodump': None, 'nocompress': None, 'nosync': None}}
|
||||
## Build out the default .ini.
|
||||
dflt_str = ('# IMPORTANT: This script uses certain permissions functions that require some forethought.\n' +
|
||||
'# You can either run as root, which is the "easy" way, OR you can run as the sks user.\n' +
|
||||
'# Has to be one or the other; you\'ll SERIOUSLY mess things up otherwise.\n' +
|
||||
'# If you run as the sks user, MAKE SURE the following is set in your sudoers\n' +
|
||||
'# (where SKSUSER is the username sks runs as):\n#\tCmnd_Alias SKSCMDS = ' +
|
||||
'/usr/bin/systemctl start sks-db,\\\n#\t\t/usr/bin/systemctl stop sks-db,\\\n#\t\t' +
|
||||
'/usr/bin/systemctl start sks-recon,\\\n#\t\t/usr/bin/systemctl stop sks-recon\n#\t' +
|
||||
'SKSUSER ALL = NOPASSWD: SKSCMDS\n\n')
|
||||
dflt_str += ('# This was written for systemd systems only. Tweaking would be needed for non-systemd systems\n' +
|
||||
'# (since every non-systemd uses their own init system callables...)\n\n')
|
||||
# [system]
|
||||
d = dflt['system']
|
||||
dflt_str += ('## SKSDUMP CONFIG FILE ##\n\n# This section controls various system configuration.\n' +
|
||||
'[system]\n# This should be the user SKS runs as.\nuser = {0}\n# This is the group that' +
|
||||
'SKS runs as.\ngroup = {1}\n# If None, don\'t compress dumps.\n# If one of: ' +
|
||||
'xz, gz, bz2, or lrz (for lrzip) then use that compression algo.\ncompress = {2}\n' +
|
||||
'# These services will be started/stopped, in order, before/after dumps. ' +
|
||||
'Comma-separated.\nsvcs = {3}\n# The path to the logfile.\nlogfile = {4}\n# The number ' +
|
||||
'of days of rotated key dumps. If None, don\'t rotate.\ndays = {5}\n# How many keys to include in each ' +
|
||||
'dump file.\ndumpkeys = {6}\n\n\n').format(d['user'],
|
||||
d['group'],
|
||||
d['compress'],
|
||||
','.join(d['svcs']),
|
||||
d['logfile'],
|
||||
d['days'],
|
||||
d['dumpkeys'])
|
||||
# [sync]
|
||||
d = dflt['sync']
|
||||
dflt_str += ('# This section controls sync settings.\n[sync]\n# This setting is what the speed should be throttled to, '+
|
||||
'in KiB/s. Use 0 for no throttling.\nthrottle = {0}\n\n').format(d['throttle'])
|
||||
# [paths]
|
||||
d = dflt['paths']
|
||||
dflt_str += ('# This section controls where stuff goes and where we should find it.\n[paths]\n# ' +
|
||||
'Where your SKS DB is.\nbasedir = {0}\n# This is the base directory where the dumps should go.\n' +
|
||||
'# There will be a sub-directory created for each date.\ndestdir = {1}\n# The ' +
|
||||
'path for rsyncing the dumps. If None, don\'t rsync.\nrsync = {2}\n' +
|
||||
'# The path to the sks binary to use\nsksbin = {3}\n\n').format(d['basedir'],
|
||||
d['destdir'],
|
||||
d['rsync'],
|
||||
d['sksbin'])
|
||||
# [runtime]
|
||||
d = dflt['runtime']
|
||||
dflt_str += ('# This section controls runtime options. These can be overridden at the commandline.\n' +
|
||||
'# They take no values; they\'re merely options.\n[runtime]\n# Don\'t dump any keys.\n' +
|
||||
'# Useful for dedicated in-transit/prep boxes.\n;nodump\n# Don\'t compress the dumps, even if ' +
|
||||
'we have a compression scheme specified in [system:compress].\n;nocompress\n# Don\'t sync to' +
|
||||
'another server/path, even if one is specified in [paths:rsync].\n;nosync\n')
|
||||
dflt_b64 = ("""IyBJTVBPUlRBTlQ6IFRoaXMgc2NyaXB0IHVzZXMgY2VydGFpbiBwZXJtaXNz
|
||||
aW9ucyBmdW5jdGlvbnMgdGhhdCByZXF1aXJlIHNvbWUKIyBmb3JldGhvdWdo
|
||||
dC4KIyBZb3UgY2FuIGVpdGhlciBydW4gYXMgcm9vdCwgd2hpY2ggaXMgdGhl
|
||||
ICJlYXN5IiB3YXksIE9SIHlvdSBjYW4gcnVuIGFzIHRoZQojIHNrcyB1c2Vy
|
||||
IChvci4uLiB3aGF0ZXZlciB1c2VyIHlvdXIgU0tTIGluc3RhbmNlIHJ1bnMg
|
||||
YXMpLgojIEl0IGhhcyB0byBiZSBvbmUgb3IgdGhlIG90aGVyOyB5b3UnbGwg
|
||||
U0VSSU9VU0xZIG1lc3MgdGhpbmdzIHVwIG90aGVyd2lzZS4KIyBJZiB5b3Ug
|
||||
cnVuIGFzIHRoZSBza3MgdXNlciwgTUFLRSBTVVJFIHRoZSBmb2xsb3dpbmcg
|
||||
aXMgc2V0IGluIHlvdXIgc3Vkb2VycwojICh3aGVyZSBTS1NVU0VSIGlzIHRo
|
||||
ZSB1c2VybmFtZSBza3MgcnVucyBhcyk6CiMJQ21uZF9BbGlhcyBTS1NDTURT
|
||||
ID0gL3Vzci9iaW4vc3lzdGVtY3RsIHN0YXJ0IHNrcy1kYixcCiMJICAgICAg
|
||||
ICAgICAgICAgICAgICAgL3Vzci9iaW4vc3lzdGVtY3RsIHN0b3Agc2tzLWRi
|
||||
LFwKIyAgICAgICAgICAgICAgICAgICAgICAgIC91c3IvYmluL3N5c3RlbWN0
|
||||
bCBzdGFydCBza3MtcmVjb24sXAojCQkgICAgICAgICAgICAgICAgIC91c3Iv
|
||||
YmluL3N5c3RlbWN0bCBzdG9wIHNrcy1yZWNvbgojCVNLU1VTRVIgQUxMID0g
|
||||
Tk9QQVNTV0Q6IFNLU0NNRFMKCiMgVGhpcyB3YXMgd3JpdHRlbiBmb3Igc3lz
|
||||
dGVtZCBzeXN0ZW1zIG9ubHkuIFR3ZWFraW5nIHdvdWxkIGJlIG5lZWRlZCBm
|
||||
b3IKIyBub24tc3lzdGVtZCBzeXN0ZW1zIChzaW5jZSBldmVyeSBub24tc3lz
|
||||
dGVtZCB1c2VzIHRoZWlyIG93biBpbml0IHN5c3RlbQojIGNhbGxhYmxlcy4u
|
||||
LikKCiMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMj
|
||||
IyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMKCiMgVGhp
|
||||
cyBzZWN0aW9uIGNvbnRyb2xzIHZhcmlvdXMgc3lzdGVtIGNvbmZpZ3VyYXRp
|
||||
b24uCltzeXN0ZW1dCgojIFRoaXMgc2hvdWxkIGJlIHRoZSB1c2VyIFNLUyBy
|
||||
dW5zIGFzLgp1c2VyID0gc2tzCgojIFRoaXMgaXMgdGhlIGdyb3VwIHRoYXQg
|
||||
U0tTIHJ1bnMgYXMuCmdyb3VwID0gc2tzCgojIElmIGVtcHR5LCBkb24ndCBj
|
||||
b21wcmVzcyBkdW1wcy4KIyBJZiBvbmUgb2Y6IHh6LCBneiwgYnoyLCBvciBs
|
||||
cnogKGZvciBscnppcCkgdGhlbiB1c2UgdGhhdCBjb21wcmVzc2lvbiBhbGdv
|
||||
LgojIE5vdGUgdGhhdCBscnppcCByZXF1aXJlcyBleHRyYSBpbnN0YWxsYXRp
|
||||
b24uCmNvbXByZXNzID0geHoKCiMgVGhlc2Ugc2VydmljZXMgd2lsbCBiZSBz
|
||||
dG9wcGVkL3N0YXJ0ZWQsIGluIG9yZGVyLCBiZWZvcmUvYWZ0ZXIgZHVtcHMu
|
||||
IElmIG1vcmUKIyB0aGFuIG9uZSwgc2VwZXJhdGUgYnkgY29tbWFzLgpzdmNz
|
||||
ID0gc2tzLWRiLHNrcy1yZWNvbgoKIyBUaGUgcGF0aCB0byB0aGUgbG9nZmls
|
||||
ZS4KbG9nZmlsZSA9IC92YXIvbG9nL3Nrc2R1bXAubG9nCgojIFRoZSBudW1i
|
||||
ZXIgb2YgZGF5cyBvZiByb3RhdGVkIGtleSBkdW1wcy4gSWYgZW1wdHksIGRv
|
||||
bid0IHJvdGF0ZS4KZGF5cyA9IDEKCiMgSG93IG1hbnkga2V5cyB0byBpbmNs
|
||||
dWRlIGluIGVhY2ggZHVtcCBmaWxlLgpkdW1wa2V5cyA9IDE1MDAwCgoKIyBU
|
||||
aGlzIHNlY3Rpb24gY29udHJvbHMgc3luYyBzZXR0aW5ncy4KW3N5bmNdCgoj
|
||||
IFRoaXMgc2V0dGluZyBpcyB3aGF0IHRoZSBzcGVlZCBzaG91bGQgYmUgdGhy
|
||||
b3R0bGVkIHRvLCBpbiBLaUIvcy4gSWYgZW1wdHkgb3IKIyAwLCBwZXJmb3Jt
|
||||
IG5vIHRocm90dGxpbmcuCnRocm90dGxlID0gMAoKCiMgVGhpcyBzZWN0aW9u
|
||||
IGNvbnRyb2xzIHdoZXJlIHN0dWZmIGdvZXMgYW5kIHdoZXJlIHdlIHNob3Vs
|
||||
ZCBmaW5kIGl0LgpbcGF0aHNdCgojIFdoZXJlIHlvdXIgU0tTIERCIGlzLgpi
|
||||
YXNlZGlyID0gL3Zhci9saWIvc2tzCgojIFRoaXMgaXMgdGhlIGJhc2UgZGly
|
||||
ZWN0b3J5IHdoZXJlIHRoZSBkdW1wcyBzaG91bGQgZ28uCiMgVGhlcmUgd2ls
|
||||
bCBiZSBhIHN1Yi1kaXJlY3RvcnkgY3JlYXRlZCBmb3IgZWFjaCBkYXRlLgpk
|
||||
ZXN0ZGlyID0gL3Nydi9odHRwL3Nrcy9kdW1wcwoKIyBUaGUgcGF0aCBmb3Ig
|
||||
cnN5bmNpbmcgdGhlIGR1bXBzLiBJZiBlbXB0eSwgZG9uJ3QgcnN5bmMuCnJz
|
||||
eW5jID0gcm9vdEBtaXJyb3Iuc3F1YXJlLXIwMHQubmV0Oi9zcnYvaHR0cC9z
|
||||
a3MvZHVtcHMKCiMgVGhlIHBhdGggdG8gdGhlIHNrcyBiaW5hcnkgdG8gdXNl
|
||||
Lgpza3NiaW4gPSAvdXNyL2Jpbi9za3MKCgojIFRoaXMgc2VjdGlvbiBjb250
|
||||
cm9scyBydW50aW1lIG9wdGlvbnMuIFRoZXNlIGNhbiBiZSBvdmVycmlkZGVu
|
||||
IGF0IHRoZQojIGNvbW1hbmRsaW5lLiBUaGV5IHRha2Ugbm8gdmFsdWVzOyB0
|
||||
aGV5J3JlIG1lcmVseSBvcHRpb25zLgpbcnVudGltZV0KCiMgRG9uJ3QgZHVt
|
||||
cCBhbnkga2V5cy4KIyBVc2VmdWwgZm9yIGRlZGljYXRlZCBpbi10cmFuc2l0
|
||||
L3ByZXAgYm94ZXMuCjtub2R1bXAKCiMgRG9uJ3QgY29tcHJlc3MgdGhlIGR1
|
||||
bXBzLCBldmVuIGlmIHdlIGhhdmUgYSBjb21wcmVzc2lvbiBzY2hlbWUgc3Bl
|
||||
Y2lmaWVkIGluCiMgdGhlIFtzeXN0ZW06Y29tcHJlc3NdIHNlY3Rpb246ZGly
|
||||
ZWN0aXZlLgo7bm9jb21wcmVzcwoKIyBEb24ndCBzeW5jIHRvIGFub3RoZXIg
|
||||
c2VydmVyL3BhdGgsIGV2ZW4gaWYgb25lIGlzIHNwZWNpZmllZCBpbiBbcGF0
|
||||
aHM6cnN5bmNdLgo7bm9zeW5j""")
|
||||
realcfg = configparser.ConfigParser(defaults = dflt, allow_no_value = True)
|
||||
if not os.path.isfile(cfgfile):
|
||||
with open(cfgfile, 'w') as f:
|
||||
f.write(dflt_str)
|
||||
f.write(base64.b64decode(dflt_b64).decode('utf-8'))
|
||||
realcfg.read(cfgfile)
|
||||
return(realcfg)
|
||||
|
||||
@ -135,7 +150,9 @@ def destPrep(args):
|
||||
if getpass.getuser() == 'root':
|
||||
uid = getpwnam(args['user']).pw_uid
|
||||
gid = getgrnam(args['group']).gr_gid
|
||||
for d in (args['destdir'], nowdir): # we COULD set it as part of the os.makedirs, but iirc it doesn't set it for existing dirs
|
||||
# we COULD set it as part of the os.makedirs, but iirc it doesn't set
|
||||
# it for existing dirs.
|
||||
for d in (args['destdir'], nowdir):
|
||||
os.chown(d, uid, gid)
|
||||
if os.path.isdir(curdir):
|
||||
os.remove(curdir)
|
||||
@ -168,7 +185,9 @@ def compressDB(args):
|
||||
if not args['compress']:
|
||||
return()
|
||||
curdir = os.path.join(args['destdir'], NOWstr)
|
||||
for thisdir, dirs, files in os.walk(curdir): # I use os.walk here because we might handle this differently in the future...
|
||||
# I use os.walk here because we might handle this differently in the
|
||||
# future...
|
||||
for thisdir, dirs, files in os.walk(curdir):
|
||||
files.sort()
|
||||
for f in files:
|
||||
fullpath = os.path.join(thisdir, f)
|
||||
@ -177,22 +196,30 @@ def compressDB(args):
|
||||
# However, I can't do this on memory-constrained systems for lrzip.
|
||||
# See: https://github.com/kata198/python-lrzip/issues/1
|
||||
with open(args['logfile'], 'a') as f:
|
||||
f.write('===== {0} Now compressing {1} =====\n'.format(str(datetime.datetime.utcnow()), fullpath))
|
||||
f.write('===== {0} Now compressing {1} =====\n'.format(
|
||||
str(datetime.datetime.utcnow()),
|
||||
fullpath))
|
||||
if args['compress'].lower() == 'gz':
|
||||
import gzip
|
||||
with open(fullpath, 'rb') as fh_in, gzip.open(newfile, 'wb') as fh_out:
|
||||
with open(fullpath, 'rb') as fh_in, gzip.open(newfile,
|
||||
'wb') as fh_out:
|
||||
fh_out.writelines(fh_in)
|
||||
elif args['compress'].lower() == 'xz':
|
||||
import lzma
|
||||
with open(fullpath, 'rb') as fh_in, lzma.open(newfile, 'wb', preset = 9|lzma.PRESET_EXTREME) as fh_out:
|
||||
with open(fullpath, 'rb') as fh_in, \
|
||||
lzma.open(newfile,
|
||||
'wb',
|
||||
preset = 9|lzma.PRESET_EXTREME) as fh_out:
|
||||
fh_out.writelines(fh_in)
|
||||
elif args['compress'].lower() == 'bz2':
|
||||
import bz2
|
||||
with open(fullpath, 'rb') as fh_in, bz2.open(newfile, 'wb') as fh_out:
|
||||
with open(fullpath, 'rb') as fh_in, bz2.open(newfile,
|
||||
'wb') as fh_out:
|
||||
fh_out.writelines(fh_in)
|
||||
elif args['compress'].lower() == 'lrz':
|
||||
import lrzip
|
||||
with open(fullpath, 'rb') as fh_in, open(newfile, 'wb') as fh_out:
|
||||
with open(fullpath, 'rb') as fh_in, open(newfile,
|
||||
'wb') as fh_out:
|
||||
fh_out.write(lrzip.compress(fh_in.read()))
|
||||
os.remove(fullpath)
|
||||
if getpass.getuser() == 'root':
|
||||
@ -212,7 +239,8 @@ def syncDB(args):
|
||||
if args['throttle'] > 0.0:
|
||||
cmd.insert(-1, '--bwlimit={0}'.format(str(args['throttle'])))
|
||||
with open(args['logfile'], 'a') as f:
|
||||
f.write('===== {0} Rsyncing to mirror =====\n'.format(str(datetime.datetime.utcnow())))
|
||||
f.write('===== {0} Rsyncing to mirror =====\n'.format(
|
||||
str(datetime.datetime.utcnow())))
|
||||
with open(args['logfile'], 'a') as f:
|
||||
subprocess.run(cmd, stdout = f, stderr = f)
|
||||
return()
|
||||
@ -223,8 +251,10 @@ def parseArgs():
|
||||
paths = cfg['paths']
|
||||
sync = cfg['sync']
|
||||
runtime = cfg['runtime']
|
||||
args = argparse.ArgumentParser(description = 'sksdump - a tool for dumping the SKS Database',
|
||||
epilog = 'brent s. || 2017 || https://square-r00t.net')
|
||||
args = argparse.ArgumentParser(description = ('sksdump - a tool for ' +
|
||||
'dumping an SKS Database'),
|
||||
epilog = ('brent s. || 2018 || ' +
|
||||
'https://square-r00t.net'))
|
||||
args.add_argument('-u',
|
||||
'--user',
|
||||
default = system['user'],
|
||||
@ -245,7 +275,9 @@ def parseArgs():
|
||||
'--services',
|
||||
default = system['svcs'],
|
||||
dest = 'svcs',
|
||||
help = 'A comma-separated list of services that will be stopped/started for the dump (in the provided order).')
|
||||
help = ('A comma-separated list of services that will ' +
|
||||
'be stopped/started for the dump (in the ' +
|
||||
'provided order).'))
|
||||
args.add_argument('-l',
|
||||
'--log',
|
||||
default = system['logfile'],
|
||||
@ -272,23 +304,28 @@ def parseArgs():
|
||||
'--sks-binary',
|
||||
default = paths['sksbin'],
|
||||
dest = 'sksbin',
|
||||
help = 'The path to the SKS binary/executable to use to perform the dumps.')
|
||||
help = ('The path to the SKS binary/executable to use ' +
|
||||
'to perform the dump.'))
|
||||
args.add_argument('-e',
|
||||
'--destdir',
|
||||
default = paths['destdir'],
|
||||
dest = 'destdir',
|
||||
help = 'The directory where the dumps should be saved (a sub-directory with the date will be created).')
|
||||
help = ('The directory where the dumps should be ' +
|
||||
'saved (a sub-directory with the date will be ' +
|
||||
'created).'))
|
||||
args.add_argument('-r',
|
||||
'--rsync',
|
||||
default = paths['rsync'],
|
||||
dest = 'rsync',
|
||||
help = 'The remote (user@host:/path/) or local (/path/) path to use to sync the dumps to.')
|
||||
help = ('The remote (user@host:/path/) or local '+
|
||||
'(/path/) path to use to sync the dumps to.'))
|
||||
args.add_argument('-t',
|
||||
'--throttle',
|
||||
default = float(sync['throttle']),
|
||||
dest = 'throttle',
|
||||
type = float,
|
||||
help = 'The amount in KiB/s to throttle the rsync to. Use 0 for no throttling.')
|
||||
help = ('The amount in KiB/s to throttle the rsync ' +
|
||||
'to. Use 0 for no throttling.'))
|
||||
args.add_argument('-D',
|
||||
'--no-dump',
|
||||
dest = 'nodump',
|
||||
@ -300,7 +337,8 @@ def parseArgs():
|
||||
dest = 'nocompress',
|
||||
action = 'store_true',
|
||||
default = ('nocompress' in runtime),
|
||||
help = 'Don\'t compress the DB dumps (default is to compress)')
|
||||
help = ('Don\'t compress the DB dumps (default is to ' +
|
||||
'compress)'))
|
||||
args.add_argument('-S',
|
||||
'--no-sync',
|
||||
dest = 'nosync',
|
||||
@ -315,7 +353,8 @@ def main():
|
||||
if getpass.getuser() not in ('root', args['user']):
|
||||
exit('ERROR: You must be root or {0}!'.format(args['user']))
|
||||
with open(args['logfile'], 'a') as f:
|
||||
f.write('===== {0} STARTING =====\n'.format(str(datetime.datetime.utcnow())))
|
||||
f.write('===== {0} STARTING =====\n'.format(
|
||||
str(datetime.datetime.utcnow())))
|
||||
if not args['nodump']:
|
||||
dumpDB(args)
|
||||
if not args['nocompress']:
|
||||
@ -323,7 +362,8 @@ def main():
|
||||
if not args['nosync']:
|
||||
syncDB(args)
|
||||
with open(args['logfile'], 'a') as f:
|
||||
f.write('===== {0} DONE =====\n'.format(str(datetime.datetime.utcnow())))
|
||||
f.write('===== {0} DONE =====\n'.format(
|
||||
str(datetime.datetime.utcnow())))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Loading…
Reference in New Issue
Block a user