args' corresponding functions spec'd out and found (and have workaround for) racetime condition in Vault.

testing/serverconf.py

@@ -15,6 +15,7 @@ default_conf = {'listener': [
                 'storage': {'file': {'path': './data'}},
                 'log_level': 'Debug',  # highest is 'Trace'
                 'pid_file': './test.pid',
+                'disable_mlock': True,
                 'raw_storage_endpoint': True,
                 'log_format': 'json',  # or String
                 'ui': True}

testing/spawn.py

@@ -217,6 +217,9 @@ class VaultSpawner(object):
         return(None)
 
     def cleanup(self):
+        self._connCheck(bind = False)
+        if self.is_running:
+            return(None)
         storage = self.conf.get('storage', {}).get('file', {}).get('path')
         if not storage:
             return(None)
@@ -249,7 +252,7 @@ class VaultSpawner(object):
             mounts['secret'] = 'kv2'
             mounts['secret_legacy'] = 'kv1'
         for idx, (mname, mtype) in enumerate(mounts.items()):
-            opts = None
+            opts = {}
             orig_mtype = mtype
             if mtype.startswith('kv'):
                 opts = {'version': re.sub(r'^kv([0-9]+)$', r'\g<1>', mtype)}
@@ -259,9 +262,12 @@ class VaultSpawner(object):
                                                       path = mname,
                                                       description = 'Testing mount ({0})'.format(mtype),
                                                       options = opts)
-            except hvac.exceptions.InvalidRequest:
+                # We might have some issues writing secrets on fast machines.
+                time.sleep(2)
+            except hvac.exceptions.InvalidRequest as e:
                 # It probably already exists.
-                pass
+                print('Exception creating {0}: {1} ({2})'.format(mname, e, e.__class__))
+                print(opts)
             if orig_mtype not in ('kv1', 'kv2', 'cubbyhole'):
                 continue
             args = {'path': 'test_secret{0}/foo{1}'.format(idx, mname),
@@ -279,11 +285,12 @@ class VaultSpawner(object):
             elif orig_mtype == 'kv2':
                 handler = self.client.secrets.kv.v2.create_or_update_secret
             try:
-                handler(**args)
+                resp = handler(**args)
             except hvac.exceptions.InvalidPath:
                 print('{0} path invalid'.format(args['path']))
             except Exception as e:
-                print('Exception: {0} ({1})'.format(e, e.__class__))
+                print('Exception creating {0} on {1}: {2} ({3})'.format(args['path'], args['mount_point'], e, e.__class__))
+                print(args)
         return(None)
 
     def start(self):
@@ -324,8 +331,9 @@ class VaultSpawner(object):
         if self.cmd:
             self.cmd.kill()
         else:
-            import signal
+            if self.pid:
-            os.kill(self.pid, signal.SIGKILL)
+                import signal
+                os.kill(self.pid, signal.SIGKILL)
         return(None)
 
 
@@ -337,7 +345,7 @@ def parseArgs():
                       help = ('If specified, do not populate with test data (if it doesn\'t exist)'))
     args.add_argument('-d', '--delete',
                       dest = 'delete_storage',
-                      action = 'store_false',
+                      action = 'store_true',
                       help = ('If specified, delete the storage backend first so a fresh instance is created'))
     args.add_argument('-c', '--cleanup',
                       dest = 'cleanup',
@@ -368,8 +376,9 @@ def main():
             s.populate()
     elif args.oper == 'stop':
         s.stop()
-    if args.cleanup:
+        if args.cleanup:
-        s.cleanup()
+            time.sleep(2)
+            s.cleanup()
     return(None)
 
 

vaultpass/__init__.py

@@ -8,11 +8,12 @@ from . import auth
 from . import clipboard
 from . import config
 from . import constants
+from . import gpg_handler
 from . import mounts
 from . import pass_import
 
 
-class PassMan(object):
+class VaultPass(object):
     client = None
     auth = None
     uri = None
@@ -64,6 +65,61 @@ class PassMan(object):
         _logger.debug('Set URI to {0}'.format(self.uri))
         return(None)
 
+    def convert(self,
+                mount,
+                force = False,
+                gpghome = constants.GPG_HOMEDIR,
+                pass_dir = constants.PASS_DIR,
+                *args, **kwargs):
+        pass  # TODO
+
+    def copySecret(self, oldpath, newpath, mount, newmount, force = False, remove_old = False, *args, **kwargs):
+        pass  # TODO
+
+        if remove_old:
+            self.deleteSecret(oldpath, mount, force = force)
+        return(None)
+
+    def createSecret(self, secret_dict, path, mount_name, *args, **kwargs):
+        mtype = self.mount.mounts.get(mount_name)
+        handler = None
+        if not mtype:
+            _logger.error('Could not determine mount type')
+            _logger.debug('Could not determine mount type for mount {0}'.format(mount_name))
+            raise RuntimeError('Could not determine mount type')
+        args = {'path': path,
+                'mount_point': mount_name,
+                'secret': secret_dict}
+        if mtype == 'cubbyhole':
+            handler = self.mount.cubbyhandler.write_secret
+        elif mtype == 'kv1':
+            handler = self.client.secrets.kv.v1.create_or_update_secret
+        elif mtype == 'kv2':
+            handler = self.client.secrets.kv.v2.create_or_update_secret
+        resp = handler(**args)
+        return(resp)
+
+    def deleteSecret(self, path, mount_name, force = False, recursive = False, *args, **kwargs):
+        pass  # TODO
+
+    def editSecret(self, path, mount, editor = constants.EDITOR, *args, **kwargs):
+        pass  # TODO
+
+    def generateSecret(self,
+                       path,
+                       mount,
+                       symbols = True,
+                       clip = False,
+                       seconds = constants.CLIP_TIMEOUT,
+                       chars = constants.SELECTED_PASS_CHARS,
+                       chars_plain = constants.SELECTED_PASS_NOSYMBOL_CHARS,
+                       in_place = False,
+                       qr = False,
+                       force = False,
+                       length = constants.GENERATED_LENGTH,
+                       *args, **kwargs):
+        pass  # TODO
+
     def getClient(self):
         auth_xml = self.cfg.xml.find('.//auth')
         if auth_xml is None:
@@ -95,3 +151,28 @@ class PassMan(object):
             _logger.error('Not initialized')
             raise RuntimeError('Not initialized')
         return(None)
+
+    def getSecret(self, path, mount, clip = None, qr = None, seconds = constants.CLIP_TIMEOUT, *args, **kwargs):
+        pass  # TODO
+
+    def initVault(self, *args, **kwargs):
+        pass  # TODO
+
+    def insertSecret(self,
+                     path,
+                     mount,
+                     allow_shouldersurf = False,
+                     multiline = False,
+                     force = False,
+                     confirm = True,
+                     *args, **kwargs):
+        pass  # TODO
+
+    def listSecretNames(self, path, mount, output = None, indent = 4, *args, **kwargs):
+        pass  # TODO
+
+    def searchSecrets(self, pattern, mount, *args, **kwargs):
+        pass  # TODO
+
+    def searchSecretNames(self, pattern, mount, *args, **kwargs):
+        pass  # TODO

vaultpass/args.py

@@ -18,9 +18,8 @@ def parseArgs():
                       help = ('The path to your configuration file. Default: ~/.config/vaultpass.xml'))
     args.add_argument('-m', '--mount',
                       dest = 'mount',
-                      required = False,
+                      default = 'secret',
-                      help = ('The mount to use in OPERATION. If not specified, assume all mounts we have access '
+                      help = ('The mount to use in OPERATION. If not specified, assume a mount named "secret"'))
-                              'to/all mounts specified in -c/--config'))
     # I wish argparse supported default subcommands. It doesn't as of python 3.8.
     subparser = args.add_subparsers(help = ('Operation to perform'),
                                     metavar = 'OPERATION',
@@ -78,10 +77,16 @@ def parseArgs():
                                        description = ('Import your existing Pass into Vault'),
                                        help = ('Import your existing Pass into Vault'))
     # CP/COPY
+    # vp.copySecret()
     cp.add_argument('-f', '--force',
                     dest = 'force',
                     action = 'store_true',
                     help = ('If specified, replace NEWPATH if it exists'))
+    cp.add_argument('-m', '--mount',
+                    dest = 'newmount',
+                    nargs = 1,
+                    required = False,
+                    help = ('The mount for the destination. Default is to use the main command\'s -m/--mount'))
     cp.add_argument('oldpath',
                     metavar = 'OLDPATH',
                     help = ('The original ("source") path for the secret'))
@@ -89,6 +94,7 @@ def parseArgs():
                     metavar = 'NEWPATH',
                     help = ('The new ("destination") path for the secret'))
     # EDIT
+    # vp.editSecret()
     edit.add_argument('-e', '--editor',
                       metavar = '/PATH/TO/EDITOR',
                       dest = 'editor',
@@ -100,10 +106,12 @@ def parseArgs():
                       help = ('Insert a new secret at PATH_TO_SECRET if it does not exist, otherwise edit it using '
                               'your default editor (see -e/--editor)'))
     # FIND/SEARCH
+    # vp.searchSecretNames()
     find.add_argument('pattern',
                       metavar = 'NAME_PATTERN',
                       help = ('List secrets\' paths whose names match the regex NAME_PATTERN'))
     # GENERATE
+    # vp.generateSecret()
     # TODO: feature parity with passgen (spaces? etc.)
     gen.add_argument('-n', '--no-symbols',
                      dest = 'symbols',
@@ -160,6 +168,7 @@ def parseArgs():
                      metavar = 'dummy',
                      help = ('(Unused; kept for compatibility reasons)'))
     # GREP
+    # vp.searchSecrets()
     # I wish argparse supported arbitrary arguments.
     # It *KIND* of does: https://stackoverflow.com/a/37367814/733214 but then I wouldn't be able to properly grab the
     # regex pattern without more hackery. So here's to wasting my life.
@@ -323,6 +332,7 @@ def parseArgs():
                       help = ('Regex pattern to search passwords'))
     # HELP has no arguments.
     # INIT
+    # vp.initVault()
     initvault.add_argument('-p', '--path',
                            dest = 'path',
                            help = ('(Dummy option; kept for compatibility reasons)'))
@@ -330,6 +340,7 @@ def parseArgs():
                            dest = 'gpg_id',
                            help = ('(Dummy option; kept for compatibility reasons)'))
     # INSERT
+    # vp.insertSecret()
     # TODO: if -e/--echo is specified and sys.stdin, use sys.stdin rather than prompt
     insertval.add_argument('-e', '--echo',
                            dest = 'allow_shouldersurf',
@@ -348,11 +359,30 @@ def parseArgs():
                            action = 'store_false',
                            help = ('If specified, disable password prompt confirmation. '
                                    'Has no effect if -e/--echo is specified'))
+    insertval.add_argument('path',
+                           metavar = 'PATH/TO/SECRET',
+                           help = ('The path to the secret'))
     # LS
+    # vp.listSecretNames()/vp.mount.print() ?
+    ls.add_argument('-o', '--output',
+                    dest = 'output',
+                    choices = constants.SUPPORTED_OUTPUT_FORMATS,
+                    metavar = 'OUTPUT_FORMAT',
+                    help = ('The format to output the hierarchy in. '
+                            'If specified, must be one of: {0} '
+                            '(the default is a condensed python '
+                            'dict repr)').format(', '.join(constants.SUPPORTED_OUTPUT_FORMATS)))
+    ls.add_argument('-i', '--indent',
+                    type = int,
+                    default = 4,
+                    dest = 'indent',
+                    help = ('If -o/--output is "pretty", "yaml", or "json", specify the indent level. '
+                            'Default is 4'))
     ls.add_argument('path',
                     metavar = 'PATH/TO/TREE/BASE',
                     help = ('List names of secrets recursively, starting at PATH/TO/TREE/BASE'))
     # MV
+    # vp.copySecret(remove_old = True)
     mv.add_argument('-f', '--force',
                     dest = 'force',
                     action = 'store_true',
@@ -364,6 +394,7 @@ def parseArgs():
                     metavar = 'NEWPATH',
                     help = ('The new ("destination") path for the secret'))
     # RM
+    # vp.deleteSecret()
     # Is this argument even sensible since it isn't a filesystem?
     rm.add_argument('-r', '--recursive',
                     dest = 'recurse',
@@ -377,6 +408,8 @@ def parseArgs():
                     metavar = 'PATH/TO/SECRET',
                     help = ('The path to the secret or subdirectory'))
     # SHOW
+    # vp.getSecret() ? plus QR etc. printing
+    # TODO: does the default overwrite the None if not specified?
     show.add_argument('-c', '--clip',
                       nargs = '?',
                       type = int,
@@ -387,6 +420,7 @@ def parseArgs():
                               'clipboard instead of printing it. '
                               'Use 0 for LINE_NUMBER for the entire secret').format(constants.SHOW_CLIP_LINENUM))
     show.add_argument('-q', '--qrcode',
+                      dest = 'qr',
                       nargs = '?',
                       type = int,
                       metavar = 'LINE_NUMBER',
@@ -406,26 +440,18 @@ def parseArgs():
                       help = ('The path to the secret'))
     # VERSION has no args.
     # IMPORT
-    def_pass_dir = os.path.abspath(os.path.expanduser(os.environ.get('PASSWORD_STORE_DIR', '~/.password-store')))
+    # vp.convert()
-    def_gpg_dir = os.path.abspath(os.path.expanduser(constants.SELECTED_GPG_HOMEDIR))
     importvault.add_argument('-d', '--directory',
-                             default = def_pass_dir,
+                             default = constants.PASS_DIR,
                              metavar = '/PATH/TO/PASSWORD_STORE/DIR',
                              dest = 'pass_dir',
-                             help = ('The path to your Pass data directory. Default: {0}').format(def_pass_dir))
+                             help = ('The path to your Pass data directory. Default: {0}').format(constants.PASS_DIR))
-    importvault.add_argument('-k', '--gpg-key-id',
-                             metavar = 'KEY_ID',
-                             dest = 'key_id',
-                             default = constants.PASS_KEY,
-                             help = ('The GPG key ID to use. Default: {0}. '
-                                     '(If None, the default is to first check /PATH/TO/PASSWORD_STORE/DIR/.gpg-id if '
-                                     'it exists, otherwise use the '
-                                     'first private key we find)').format(constants.PASS_KEY))
     importvault.add_argument('-H', '--gpg-homedir',
-                             default = def_gpg_dir,
+                             default = constants.GPG_HOMEDIR,
                              dest = 'gpghome',
                              metavar = '/PATH/TO/GNUPG/HOMEDIR',
-                             help = ('The GnuPG "homedir". Default: {0}').format(def_gpg_dir))
+                             help = ('The GnuPG "homedir". It MUST contain the private key that Pass uses. '
+                                     'Default: {0}').format(constants.GPG_HOMEDIR))
     importvault.add_argument('-f', '--force',
                              dest = 'force',
                              action = 'store_true',

vaultpass/constants.py

@@ -4,6 +4,8 @@ import string
 # These are static.
 NAME = 'VaultPass'
 VERSION = '0.0.1'
+SUPPORTED_ENGINES = ('kv1', 'kv2', 'cubbyhole')
+SUPPORTED_OUTPUT_FORMATS = ('pretty', 'yaml', 'json')
 ALPHA_LOWER_PASS_CHARS = string.ascii_lowercase
 ALPHA_UPPER_PASS_CHARS = string.ascii_uppercase
 ALPHA_PASS_CHARS = ALPHA_LOWER_PASS_CHARS + ALPHA_UPPER_PASS_CHARS
@@ -19,9 +21,9 @@ SELECTED_PASS_NOSYMBOL_CHARS = ALPHANUM_PASS_CHARS
 CLIPBOARD = 'clipboard'
 GENERATED_LENGTH = 25  # I personally would prefer 32, but Pass compatibility...
 EDITOR = 'vi'  # vi is on ...every? single distro and UNIX/UNIX-like, to my knowledge.
-PASS_KEY = None
 GPG_HOMEDIR = '~/.gnupg'
 SELECTED_GPG_HOMEDIR = GPG_HOMEDIR
+PASS_DIR = '~/.password-store'
 
 if not os.environ.get('NO_VAULTPASS_ENVS'):
     # These are dynamically generated from the environment.
@@ -32,5 +34,9 @@ if not os.environ.get('NO_VAULTPASS_ENVS'):
     CLIPBOARD = os.environ.get('PASSWORD_STORE_X_SELECTION', CLIPBOARD)
     GENERATED_LENGTH = int(os.environ.get('PASSWORD_STORE_GENERATED_LENGTH', GENERATED_LENGTH))
     EDITOR = os.environ.get('EDITOR', EDITOR)
-    PASS_KEY = os.environ.get('PASSWORD_STORE_KEY', PASS_KEY)
     SELECTED_GPG_HOMEDIR = os.environ.get('GNUPGHOME', GPG_HOMEDIR)
+    PASS_DIR = os.environ.get('PASSWORD_STORE_DIR', PASS_DIR)
+
+# These are made more sane.
+PASS_DIR = os.path.abspath(os.path.expanduser(PASS_DIR))
+SELECTED_GPG_HOMEDIR = os.path.abspath(os.path.expanduser(SELECTED_GPG_HOMEDIR))

vaultpass/gpg_handler.py

@@ -18,8 +18,17 @@ class GPG(object):
 
     def decrypt(self, fpath):
         fpath = os.path.abspath(os.path.expanduser(fpath))
+        _logger.debug('Opening {0} for decryption'.format(fpath))
         with open(fpath, 'rb') as fh:
-            iobuf = io.BytesIO(fh.read())
+            data = fh.read()
+        decrypted = self.decryptData(data)
+        return(decrypted)
+
+    def decryptData(self, data):
+        if isinstance(data, str):
+            data = data.encode('utf-8')
+        _logger.debug('Decrypting {0} bytes'.format(len(data)))
+        iobuf = io.BytesIO(data)
         iobuf.seek(0, 0)
         rslt = self.gpg.decrypt(iobuf)
         decrypted = rslt[0]

vaultpass/mounts.py

@@ -1,15 +1,23 @@
+import copy
 import logging
 import re
 import shutil
+import time
 import warnings
 ##
 import dpath.util  # https://pypi.org/project/dpath/
 import hvac.exceptions
+##
+from . import constants
 
 
 _logger = logging.getLogger()
 _mount_re = re.compile(r'^(?P<mount>.*)/$')
 _subpath_re = re.compile(r'^/?(?P<path>.*)/$')
+_kv_re = re.compile(r'^kv(?:-v)?(?P<version>[0-9]+)$')
+
+
+# TODO: for all write operations, modify handler call to first check if path exists and patch if it does?
 
 
 class CubbyHandler(object):
@@ -30,6 +38,18 @@ class CubbyHandler(object):
         resp = self.client._adapter.get(url = uri)
         return(resp.json())
 
+    def write_secret(self, path, secret, mount_point = 'cubbyhole'):
+        path = path.lstrip('/')
+        args = {'path': '/'.join((mount_point, path))}
+        for k, v in secret.items():
+            if k in args.keys():
+                _logger.error('Cannot use reserved secret name')
+                _logger.debug('Cannot use secret name {0} as it is reserved'.format(k))
+                raise ValueError('Cannot use reserved secret name')
+            args[k] = v
+        resp = self.client.write(**args)
+        return(resp)
+
 
 class MountHandler(object):
     internal_mounts = ('identity', 'sys')
@@ -42,6 +62,38 @@ class MountHandler(object):
         self.paths = {}
         self.getSysMounts()
 
+    def createMount(self, mount_name, mount_type = 'kv2'):
+        orig_mtype = mount_type
+        if mount_type not in constants.SUPPORTED_ENGINES:
+            _logger.error('Invalid mount type')
+            _logger.debug(('The mount type {0} is invalid. '
+                           'It must be one of: {1}').format(mount_type, ', '.join(constants.SUPPORTED_ENGINES)))
+            raise ValueError('Invalid mount type')
+        options = {}
+        r = _kv_re.search(mount_type)
+        if r:
+            mount_type = 'kv'
+            options['version'] = r.groupdict()['version']
+        created = False
+        try:
+            self.client.sys.enable_secrets_engine(mount_type,
+                                                  path = mount_name,
+                                                  description = 'Created automatically by VaultPass',
+                                                  options = options)
+            created = True
+        except hvac.exceptions.InvalidPath as e:
+            _logger.error('Invalid path')
+            _logger.debug('The mount path {0} (type {1}) is invalid: {2}'.format(mount_name, orig_mtype, e))
+            raise ValueError('Invalid path')
+        except hvac.exceptions.InvalidRequest as e:
+            _logger.error('Invalid request; does mount already exist?')
+            _logger.debug(('The creation of mount path {0} (type {1}) generated an invalid request: '
+                           '{2}. Does it already exist?').format(mount_name, orig_mtype, e))
+        # Due to how KV2 is created, we can hit a timing/race condition.
+        if orig_mtype == 'kv2' and created:
+            time.sleep(2)
+        return(created)
+
     def getMountType(self, mount):
         if not self.mounts:
             self.getSysMounts()
@@ -53,7 +105,19 @@ class MountHandler(object):
         return(mtype)
 
     def getSecret(self, path, mount, version = None):
-        pass
+        if not self.mounts:
+            self.getSysMounts()
+        mtype = self.getMountType(mount)
+        args = {}
+        handler = None
+        if mtype == 'cubbyhole':
+            handler = self.cubbyhandler.read_secret
+        elif mtype == 'kv1':
+            handler = self.client.secrets.kv.v1.read_secret
+        if mtype == 'kv2':
+            args['version'] = version
+        data = self.client.secrets
+        # TODO
 
     def getSecretNames(self, path, mount, version = None):
         reader = None
@@ -214,6 +278,3 @@ class MountHandler(object):
         elif not output:
             return(str(self.paths))
         return(None)
-
-    def search(self):
-        pass