r00t2/bdisk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Activity

checking in a ton of population

Browse Source
This commit is contained in:
brent s. 2018-09-23 07:55:58 -04:00
parent c48c752f84
commit 0dd54a604d
12 changed files with 426 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
TODO
View File

@ -1,11 +1,11 @@
- write classes/functions
- XML-based config
-x XML syntax
--- regex btags - case-insensitive? this can be represented in-pattern:
https://stackoverflow.com/a/9655186/733214
--- xregex btags - case-insensitive? this can be represented in-pattern:
xhttps://stackoverflow.com/a/9655186/733214
-x configuration generator
--- print end result xml config to stderr for easier redirection? or print prompts to stderr and xml to stdout?
-- XSD for validation
--- xprint end result xml config to stderr for easier redirection? or print prompts to stderr and xml to stdout?
-- xXSD for validation
-- Flask app for generating config?
-- TKinter (or pygame?) GUI?
--- https://docs.python.org/3/faq/gui.html
@ -16,12 +16,9 @@

- locking
- for docs, 3.x (as of 3.10) was 2.4M.
- Need ability to write/parse mtree specs (or a similar equivalent) for applying ownerships/permissions to overlay files
- xNeed ability to write/parse mtree specs (or a similar equivalent) for applying ownerships/permissions to overlay files
-- parsing is done. writing may? come later.

- need to package:
python-hashid (https://psypanda.github.io/hashID/,
https://github.com/psypanda/hashID,
https://pypi.org/project/hashID/)

- package for PyPI:
# https://packaging.python.org/tutorials/distributing-packages/
@ -37,7 +34,6 @@ BUGS.SQUARE-R00T.NET bugs/tasks:
#14: Use os.path.join() for more consistency/pythonicness
#24: Run as regular user? (pychroot? fakeroot?)
#34: Build-time support for only building single phase of build
#36: Allow parsing pkg lists with inline comments
#39: Fix UEFI
#40: ISO overlay (to add e.g. memtest86+ to final ISO)
#43: Support resuming partial tarball downloads (Accept-Ranges: bytes)

1
bdisk/BIOS.py
View File

@ -1,3 +1,4 @@
import jinja2
import os
import shutil


30
bdisk/GPG.py
View File

@ -3,6 +3,17 @@ import os
import psutil
import gpg.errors


# This helps translate the input name from the conf to a string compatible with the gpg module.
_algmaps = {#'cv': 'cv{keysize}', # DISABLED, can't sign (only encrypt). Currently only 25519
'ed': 'ed{keysize}', # Currently only 25519
#'elg': 'elg{}', # DISABLED, can't sign (only encrypt). 1024, 2048, 4096
'nist': 'nistp{keysize}', # 256, 384, 521
'brainpool.1': 'brainpoolP{keysize}r1', # 256, 384, 512
'sec.k1': 'secp{keysize}k1', # Currently only 256
'rsa': 'rsa{keysize}', # Variable (1024 <> 4096), but we only support 1024, 2048, 4096
'dsa': 'dsa{keysize}'} # Variable (768 <> 3072), but we only support 768, 2048, 3072

# http://files.au.adversary.org/crypto/GPGMEpythonHOWTOen.html
# https://www.gnupg.org/documentation/manuals/gpgme.pdf
# Support ECC? https://www.gnupg.org/faq/whats-new-in-2.1.html#ecc
@ -60,7 +71,7 @@ class GPGHandler(object):
self._prep_home()
else:
self._check_home()
self.ctx = self.get_context(home_dir = self.home)
self.ctx = self.GetContext(home_dir = self.home)

def _check_home(self, home = None):
if not home:
@ -94,11 +105,12 @@ class GPGHandler(object):
'write to')
return()

def get_context(self, **kwargs):
def GetContext(self, **kwargs):
ctx = gpg.Context(**kwargs)
return(ctx)

def kill_stale_agent(self):
def KillStaleAgent(self):
# Is this even necessary since I switched to the native gpg module instead of the gpgme one?
_process_list = []
# TODO: optimize; can I search by proc name?
for p in psutil.process_iter():
@ -113,7 +125,13 @@ class GPGHandler(object):
# for p in plst:
# psutil.Process(p).terminate()

def get_sigs(self, data_in):
def CreateKey(self, params): # TODO: explicit params
# We can't use self.ctx.create_key; it's a little limiting.
# It's a fairly thin wrapper to .op_createkey() (the C GPGME API gpgme_op_createkey) anyways.

pass

def GetSigs(self, data_in):
key_ids = []
# Currently as of May 13, 2018 there's no way using the GPGME API to do
# the equivalent of the CLI's --list-packets.
@ -131,3 +149,7 @@ class GPGHandler(object):
l = [i.strip() for i in line.split(':')]
key_ids.append(l[0])
return(key_ids)

def CheckSigs(self, keys, sig_data):
try:
self.ctx.verify(sig_data)

7
bdisk/basedistro/archlinux.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python3.6
#!/usr/bin/env python3

# Supported initsys values:
# systemd
@ -41,6 +41,7 @@ pkg_mgr_prep = """#!/bin/bash
pacman -Syy
pacman-key --init
pacman-key --populate archlinux
pacman -S --noconfirm --needed base
pacman -S --noconfirm --needed base-devel multilib-devel git linux-headers \
mercurial subversion vala xorg-server-devel
cd /tmp
@ -62,8 +63,8 @@ rm apacman*
# should try to install it.
#### AUR SUPPORT ####
packager = {'pre_check': False,
'sys_update': ['/usr/bin/aurman', '-S', '-u'],
'sync_cmd': ['/usr/bin/aurman', '-S', '-y', '-y'],
'sys_update': ['/usr/bin/apacman', '-S', '-u'],
'sync_cmd': ['/usr/bin/apacman', '-S', '-y', '-y'],
'check_cmds': {'versioned': ['/usr/bin/pacman',
'-Q', '-s',
'{PACKAGE}'],

9
bdisk/bdisk.xsd
View File

@ -669,8 +669,17 @@
<xs:attribute name="algo" use="optional">
<xs:simpleType>
<xs:restriction base="xs:string">
<!-- rsa, dsa, and elgamal are "normal". Newer GnuPG supports ECC (yay!), so we have support for those in the XSD (you can get a list with gpg -with-colons -list-config curve | cut -f3 -d":" | tr ';' '\n'). -->
<!-- We test in-code if the host supports it. -->
<xs:enumeration value="rsa"/>
<xs:enumeration value="dsa"/>
<!-- The following only support encryption. The entire reason we'd be generating a key is to sign files, so we disable them. -->
<!-- <xs:enumeration value="elg"/> -->
<!-- <xs:enumeration value="cv"/> -->
<xs:enumeration value="ed"/>
<xs:enumeration value="nist"/>
<xs:enumeration value="brainpool.1"/>
<xs:enumeration value="sec.k1"/>
</xs:restriction>
</xs:simpleType>
</xs:attribute>

2
bdisk/confgen.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python3.6
#!/usr/bin/env python3

# Ironically enough, I think building a GUI for this would be *cleaner*.
# Go figure.

57
bdisk/confparse.py
View File

@ -40,6 +40,8 @@ class Conf(object):

You can provide any combination of these
(e.g. "profile={'id': 2, 'name' = 'some_profile'}").
Non-greedy matching (meaning ALL attributes specified
must match).
"""
if validate_cfg == 'pre':
# Validate before attempting any other operations
@ -57,6 +59,7 @@ class Conf(object):
if validate_cfg:
# Validation post-substitution
self.validate(parsed = False)
# TODO: populate checksum{} with hash_algo if explicit

def get_pki_obj(self, pki, pki_type):
elem = {}
@ -272,6 +275,9 @@ class Conf(object):
self.cfg['profile'][a] = transform.xml2py(
self.profile.attrib[a],
attrib = True)
# Small bug in transform.xml2py that we unfortunately can't fix, so we manually fix.
if 'id' in self.cfg['profile'] and isinstance(self.cfg['profile']['id'], bool):
self.cfg['profile']['id'] = int(self.cfg['profile']['id'])
return()

def parse_sources(self):
@ -323,11 +329,12 @@ class Conf(object):
xml = etree.fromstring(self.xml_suppl.return_full())
self.xsd.assertValid(xml)
if parsed:
# TODO: perform further validations that we can't do in XSD.
# We wait until after it's parsed to evaluate because otherwise we
# can't use utils.valid().
# We only bother with stuff that would hinder building, though -
# e.g. we don't check that profile's UUID is a valid UUID4.
# The XSD can catch a lot of stuff, but it's not so hot with things like URI validation,
# email validation, etc.
# URLs
for url in (self.cfg['uri'], self.cfg['dev']['website']):
if not valid.url(url):
@ -335,25 +342,41 @@ class Conf(object):
# Emails
for k in self.cfg['gpg']['keys']:
if not valid.email(k['email']):
raise ValueError(
'GPG key {0}: {1} is not a valid email '
'address'.format(k['name'], k['email']))
raise ValueError('GPG key {0}: {1} is not a valid email address'.format(k['name'], k['email']))
if not valid.email(self.cfg['dev']['email']):
raise ValueError('{0} is not a valid email address'.format(
self.cfg['dev']['email']))
raise ValueError('{0} is not a valid email address'.format(self.cfg['dev']['email']))
if self.cfg['pki']:
if 'subject' in self.cfg['pki']['ca']:
if not valid.email(
self.cfg['pki']['ca']['subject']['emailAddress']):
raise ValueError('{0} is not a valid email '
'address'.format(
if not valid.email(self.cfg['pki']['ca']['subject']['emailAddress']):
raise ValueError('{0} is not a valid email address'.format(
self.cfg['pki']['ca']['subject']['emailAddress']))

if not self.cfg['pki'][x]['subject']:
for cert in self.cfg['pki']['clients']:
if not cert['subject']:
continue
if not valid.email(
self.cfg['pki'][x]['subject']['emailAddress']):
raise ValueError('{0} is not a valid email '
'address'.format(
self.cfg['pki'][x]['subject']['email']))
if not valid.email(cert['subject']['emailAddress']):
raise ValueError('{0} is not a valid email address'.format(cert['subject']['email']))
# Salts/hashes
if self.cfg['root']['salt']:
if not valid.salt_hash(self.cfg['root']['salt']):
raise ValueError('{0} is not a valid salt'.format(self.cfg['root']['salt']))
if self.cfg['root']['hashed']:
if not valid.salt_hash_full(self.cfg['root']['salt_hash'], self.cfg['root']['hash_algo']):
raise ValueError('{0} is not a valid hash of type {1}'.format(self.cfg['root']['salt_hash'],
self.cfg['root']['hash_algo']))
for u in self.cfg['users']:
if u['salt']:
if not valid.salt_hash(u['salt']):
raise ValueError('{0} is not a valid salt'.format(u['salt']))
if u['hashed']:
if not valid.salt_hash_full(u['salt_hash'], u['hash_algo']):
raise ValueError('{0} is not a valid hash of type {1}'.format(u['salt_hash'], u['hash_algo']))
# GPG Key IDs
if self.cfg['gpg']['keyid']:
if not valid.gpgkeyID(self.cfg['gpg']['keyid']):
raise ValueError('{0} is not a valid GPG Key ID/fingerprint'.format(self.cfg['gpg']['keyid']))
for s in self.cfg['sources']:
if 'sig' in s:
for k in s['sig']['keys']:
if not valid.gpgkeyID(k):
raise ValueError('{0} is not a valid GPG Key ID/fingerprint'.format(k))
return()

68
bdisk/env_prep.py
View File

@ -1,3 +1,67 @@
import copy
import importlib
import hashlib
import importlib # needed for the guest-os-specific stuff...
import os
from . import utils
from urllib.parse import urljoin


def hashsum_downloader(url, filename = None):
# TODO: support "latest" and "regex" flags? or remove from specs (since the tarball can be specified by these)?
# move that to the utils.DOwnload() class?
d = utils.Download(url, progress = False)
hashes = {os.path.basename(k):v for (v, k) in [line.split() for line in d.fetch().decode('utf-8').splitlines()]}
if filename:
if filename in hashes:
return(hashes[filename])
else:
raise KeyError('Filename {0} not in the list of hashes'.format(filename))
return(hashes)


class Prepper(object):
def __init__(self, dirs, sources, gpg = None):
# dirs is a ConfParse.cfg['build']['paths'] dict of dirs
self.CreateDirs(dirs)
# TODO: set up GPG env here so we can use it to import sig key and verify sources
for idx, s in enumerate(sources):
self._download(idx)

def CreateDirs(self, dirs):
for d in dirs:
os.makedirs(d, exist_ok = True)
return()


def _download(self, source_idx):
download = True
_source = self.cfg['sources'][source_idx]
_dest_dir = os.path.join(self.cfg['build']['paths']['cache'], source_idx)
_tarball = os.path.join(_dest_dir, _source['tarball']['fname'])
_remote_dir = urljoin(_source['mirror'], _source['rootpath'])
_remote_tarball = urljoin(_remote_dir + '/', _source['tarball']['fname'])
def _hash_verify(): # TODO: move to utils.valid()?
# Get a checksum.
if 'checksum' in _source:
if not _source['checksum']['explicit']:
_source['checksum']['value'] = hashsum_downloader(urljoin(_remote_dir + '/',
_source['checksum']['fname']))
if not _source['checksum']['hash_algo']:
_source['checksum']['hash_algo'] = utils.detect.any_hash(_source['checksum']['value'],
normalize = True)[0]
_hash = hashlib.new(_source['checksum']['hash_algo'])
with open(_tarball, 'rb') as f:
# It's potentially a large file, so we chunk it 64kb at a time.
_hashbuf = f.read(64000)
while len(_hashbuf) > 0:
_hash.update(_hashbuf)
_hashbuf = f.read(64000)
if _hash.hexdigest().lower() != _source['checksum']['value'].lower():
return(False)
return(True)
def _sig_verify(gpg_instance): # TODO: move to utils.valid()? or just use as part of the bdisk.GPG module?
pass
if os.path.isfile(_tarball):
download = _hash_verify()
download = _sig_verify()
if download:
d = utils.Download(_remote_tarball)

8
bdisk/main.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python3.6
#!/usr/bin/env python3

import argparse
import confparse
@ -14,8 +14,10 @@ def parseArgs():
epilog = ('https://git.square-r00t.net'))
return(args)

def run():
pass
def run(cfg):
cfg = confparse.Conf(cfg, validate_cfg = True)
cfg.parse_all()


def run_interactive():
args = vars(parseArgs().parse_args())

182
bdisk/mtree.py
View File

@ -3,15 +3,188 @@
import argparse
import copy
import datetime
import grp
import hashlib
import os
import pathlib
import platform
import pwd
import re
import stat
from collections import OrderedDict
try:
import pycksum
has_cksum = True
except ImportError:
has_cksum = False

# Parse BSD mtree spec files.
# On arch, BSD mtree is ported in the AUR as nmtree.
# TODO: add a generator class as well?
# TODO: add a generator class as well? (in process)
# TODO: add a checking function as well?

# The format used for headers
_header_strptime_fmt = '%a %b %d %H:%M:%S %Y'

# Supported hash types (for generation). These are globally available always.
_hashtypes = ['md5', 'sha1', 'sha256', 'sha384', 'sha512']
# If RIPEMD-160 is supported, we add it (after MD5).
if 'ripemd160' in hashlib.algorithms_available:
_hashtypes.insert(1, 'rmd160')

# Iterative to determine which type an item is.
_stype_map = {'block': stat.S_ISBLK,
'char': stat.S_ISCHR,
'dir': stat.S_ISDIR,
'fifo': stat.S_ISFIFO,
'file': stat.S_ISREG,
'link': stat.S_ISLNK,
'socket': stat.S_ISSOCK}

# Regex pattern for cleaning up an octal perm mode into a string representation.
_octre = re.compile('^0o')

class MTreeGen(object):
def __init__(self, path):
self.path = pathlib.PosixPath(os.path.abspath(os.path.expanduser(path)))
# These are used to keep a cached copy of the info.
self._sysinfo = {'uids': {}, 'gids': {}}
self._build_header()
# We use this to keep track of where we are exactly in the tree so we can generate a full absolute path at
# any moment relative to the tree.
self._path_pointer = copy.deepcopy(self.path)


def paths_iterator(self):
for root, dirs, files in os.walk(self.path):
for f in files:
_fname = self.path.joinpath(f)
_stats = self._get_stats(_fname)
if not _stats:
print(('WARNING: {0} either disappeared while we were trying to parse it or '
'it is a broken symlink.').format(_fname))
continue
# TODO: get /set line here?
item = ' {0} \\\n'.format(f)
_type = 'file' # TODO: stat this more accurately
_cksum = self._gen_cksum(_fname)
item += ' {0} {1} {2}\\\n'.format(_stats['size'],
_stats['time'],
('{0} '.format(_cksum) if _cksum else ''))
# TODO: here's where the hashes would get added
# TODO: here's where we parse dirs. maybe do that before files?
# remember: mtree specs use ..'s to traverse upwards when done with a dir
for d in dirs:
_dname = self.path.joinpath(d)
_stats = self._get_stats(_dname)
if not _stats:
print(('WARNING: {0} either disappeared while we were trying to parse it or '
'it is a broken symlink.').format(_dname))
continue
# TODO: get /set line here?
return()


def _gen_cksum(self, fpath):
if not has_cksum:
return(None)
if not os.path.isfile(fpath):
return(None)
# TODO: waiting on https://github.com/sobotklp/pycksum/issues/2 for byte iteration (because large files maybe?)
c = pycksum.Cksum()
with open(fpath, 'rb') as f:
c.add(f)
return(c.get_cksum())


def _get_stats(self, path):
stats = {}
try:
_st = os.stat(path, follow_symlinks = False)
except FileNotFoundError:
# Broken symlink? Shouldn't occur since follow_symlinks is False anyways, BUT...
return(None)
# Ownership
stats['uid'] = _st.st_uid
stats['gid'] = _st.st_gid
if _st.st_uid in self._sysinfo['uids']:
stats['uname'] = self._sysinfo['uids'][_st.st_uid]
else:
_pw = pwd.getpwuid(_st.st_uid).pw_name
stats['uname'] = _pw
self._sysinfo['uids'][_st.stuid] = _pw
if _st.st_gid in self._sysinfo['gids']:
stats['gname'] = self._sysinfo['gids'][_st.st_gid]
else:
_grp = grp.getgrgid(_st.st_gid).gr_name
stats['gname'] = _grp
self._sysinfo['gids'][_st.stgid] = _grp
# Type and Mode
for t in _stype_map:
if _stype_map[t](_st.st_mode):
stats['type'] = t
# TODO: need a reliable way of parsing this.
# for instance, for /dev/autofs, _st.st_dev = 6 (os.makedev(6) confirms major is 0, minor is 6)
# but netBSD mtree (ported) says it's "0xaeb" (2795? or, as str, "®b" apparently).
# I'm guessing the kernel determines this, but where is it pulling it from/how?
# We can probably do 'format,major,minor' (or, for above, 'linux,0,6').
# if t in ('block', 'char'):
# stats['device'] = None
# Handle symlinks.
if t == 'link':
_target = path
while os.path.islink(_target):
_target = os.path.realpath(_target)
stats['link'] = _target
break
stats['mode'] = '{0:0>4}'.format(_octre.sub('', str(oct(stat.S_IMODE(_st.st_mode)))))
stats['size'] = _st.st_size
stats['time'] = str(float(_st.st_mtime))
stats['nlink'] = _st.st_nlink
# TODO: "flags" keyword? is that meaningful on linux?
stats['flags'] = 'none'
return(stats)



def _gen_hashes(self, fpath):
hashes = OrderedDict({})
if not os.path.isfile(fpath):
return(hashes)
_hashnums = len(_hashtypes)
for idx, h in enumerate(_hashtypes):
# Stupid naming inconsistencies.
_hashname = (h if h is not 'rmd160' else 'ripemd160')
_hasher = hashlib.new(_hashname)
with open(fpath, 'rb') as f:
# Hash 64kb at a time in case it's a huge file. TODO: is this the most ideal chunk size?
_hashbuf = f.read(64000)
while len(_hashbuf) > 0:
_hasher.update(_hashbuf)
_hashbuf = f.read(64000)
hashes[h] = _hasher.hexdigest()
return(hashes)
# if idx + 1 < _hashnums:
# hashes += ' {0}={1} \\\n'.format(h, _hasher.hexdigest())
# else:
# hashes += ' {0}={1}\n'.format(h, _hasher.hexdigest())
# return(hashes)


def _build_header(self):
self.spec = ''
_header = OrderedDict({})
_header['user'] = pwd.getpwuid(os.geteuid()).pw_name
_header['machine'] = platform.node()
_header['tree'] = str(self.path)
_header['date'] = datetime.datetime.utcnow().strftime(_header_strptime_fmt)
for h in _header:
self.spec += '#\t{0:>7}: {1}\n'.format(h, _header[h])
self.spec += '\n'
return()



class MTreeParse(object):
def __init__(self, spec):
if not isinstance(spec, (str, bytes)):
@ -21,7 +194,6 @@ class MTreeParse(object):
spec = spec.decode('utf-8')
except UnicodeDecodeError:
raise ValueError('spec must be a utf-8 encoded set of bytes if using byte mode')
self._strptime_fmt = '%a %b %d %H:%M:%S %Y'
self.orig_spec = copy.deepcopy(spec) # For referencing in case someone wanted to write it out.
# We NOW need to handle the escaped linebreaking it does.
self._specdata = re.sub('\\\\\s+', '', spec).splitlines()
@ -82,7 +254,7 @@ class MTreeParse(object):
# They are restored by an "/unset". Since they're global and stateful, they're handled as a class attribute.
self.settings = copy.deepcopy(self._tplitem)
self._parse_items()
del(self.settings, self._tplitem, self._strptime_fmt)
del(self.settings, self._tplitem)


def _get_header(self):
@ -96,7 +268,7 @@ class MTreeParse(object):
header = l[0]
val = (l[1] if l[1] is not '(null)' else None)
if header == 'date':
val = datetime.datetime.strptime(val, self._strptime_fmt)
val = datetime.datetime.strptime(val, _header_strptime_fmt)
elif header == 'tree':
val = pathlib.PosixPath(val)
self.header[header] = val
@ -158,6 +330,8 @@ class MTreeParse(object):
return(out)
def _unset_parse(unsetline):
out = {}
if unsetline[1] == 'all':
return(copy.deepcopy(self._tplitem))
for i in unsetline:
out[i] = self._tplitem[i]
return(out)

21
bdisk/prompt_strings.py
View File

@ -11,18 +11,31 @@ class PromptStrings(object):
'attribs': {
'algo': {
'text': 'the subkey\'s encryption type/algorithm',
'choices': ['rsa', 'dsa'],
'default': 'rsa'
# The following can ONLY be used for encryption, not signing: elg, cv
#'choices': ['rsa', 'dsa', 'elg', 'ed', 'cv', 'nistp', 'brainpool.1', 'secp.k1'],
'choices': ['rsa', 'dsa', 'ed', 'nist', 'brainpool.1', 'sec.k1'],
#'default': 'rsa'
'default': 'ed'
},
'keysize': {
'text': 'the subkey\'s key size (in bits)',
'choices': {
'rsa': ['1024', '2048', '4096'],
'dsa': ['768', '2048', '3072']
'dsa': ['768', '2048', '3072'],
#'elg': ['1024', '2048', '4096'], # Invalid for signing, etc.
'ed': ['25519'],
#'cv': ['25519'],
'nistp': ['256', '384', '521'],
'brainpool.1': ['256', '384', '512'],
'sec.k1': ['256']
},
'default': {
'rsa': '4096',
'dsa': '3072'
'dsa': '3072',
'ed': '25519',
'nistp': '521',
'brainpool.1': '512',
'sec.k1': '256'
}
}
},

81
bdisk/utils.py
View File

@ -1,3 +1,5 @@
# Yes, this is messy. They doesn't belong anywhere else, leave me alone.

import _io
import copy
import crypt
@ -14,6 +16,7 @@ import string
import uuid
import validators
import zlib
import requests
import lxml.etree
import lxml.objectify
from bs4 import BeautifulSoup
@ -30,7 +33,7 @@ passlib_schemes = ['des_crypt', 'md5_crypt', 'sha256_crypt', 'sha512_crypt']
# Build various hash digest name lists
digest_schemes = list(hashlib.algorithms_available)
# Provided by zlib
# TODO
# TODO?
digest_schemes.append('adler32')
digest_schemes.append('crc32')

@ -39,6 +42,54 @@ crypt_map = {'sha512': crypt.METHOD_SHA512,
'md5': crypt.METHOD_MD5,
'des': crypt.METHOD_CRYPT}


class Download(object):
def __init__(self, url, progress = True, offset = None, chunksize = 1024):
self.cnt_len = None
self.head = requests.head(url, allow_redirects = True).headers
self.req_headers = {}
self.range = False
self.url = url
self.offset = offset
self.chunksize = chunksize
self.progress = progress
if 'accept-ranges' in self.head:
if self.head['accept-ranmges'].lower() != 'none':
self.range = True
if 'content-length' in self.head:
try:
self.cnt_len = int(self.head['content-length'])
except TypeError:
pass
if self.cnt_len and self.offset and self.range:
if not self.offset <= self.cnt_len:
raise ValueError(('The offset requested ({0}) is greater than '
'the content-length value').format(self.offset, self.cnt_len))
self.req_headers['range'] = 'bytes={0}-'.format(self.offset)

def fetch(self):
if not self.progress:
self.req = requests.get(self.url, allow_redirects = True, headers = self.req_headers)
self.bytes_obj = self.req.content
else:
self.req = requests.get(self.url, allow_redirects = True, stream = True, headers = self.req_headers)
self.bytes_obj = bytes()
_bytelen = 0
# TODO: better handling for logging instead of print()s?
for chunk in self.req.iter_content(chunk_size = self.chunksize):
self.bytes_obj += chunk
if self.cnt_len:
print('\033[F')
print('{0:.2f}'.format((_bytelen / float(self.head['content-length'])) * 100),
end = '%',
flush = True)
_bytelen += self.chunksize
else:
print('.', end = '')
print()
return(self.bytes_obj)


class XPathFmt(string.Formatter):
def get_field(self, field_name, args, kwargs):
vals = self.get_value(field_name, args, kwargs), field_name
@ -50,18 +101,19 @@ class detect(object):
def __init__(self):
pass

def any_hash(self, hash_str):
def any_hash(self, hash_str, normalize = False):
h = hashid.HashID()
hashes = []
for i in h.identifyHash(hash_str):
if i.extended:
continue
x = i.name
if x.lower() in ('crc-32', 'ripemd-160', 'sha-1', 'sha-224',
'sha-256', 'sha-384', 'sha-512'):
if x.lower() in ('crc-32', 'ripemd-160', 'sha-1', 'sha-224', 'sha-256', 'sha-384', 'sha-512'):
# Gorram you, c0re.
x = re.sub('-', '', x.lower())
_hashes = [h.lower() for h in digest_schemes]
_hashes = [h.lower() for h in digest_schemes] # TODO: move this outside so we don't define it every invoke
if normalize:
x = re.sub('(-|crypt|\s+)', '', x.lower())
if x.lower() in sorted(list(set(_hashes))):
hashes.append(x)
return(hashes)
@ -83,8 +135,7 @@ class detect(object):
return(salt)

def remote_files(self, url_base, ptrn = None, flags = []):
with urlopen(url_base) as u:
soup = BeautifulSoup(u.read(), 'lxml')
soup = BeautifulSoup(Download(url_base, progress = False).bytes_obj, 'lxml')
urls = []
if 'regex' in flags:
if not isinstance(ptrn, str):
@ -113,8 +164,7 @@ class detect(object):
return(urls)

def gpgkeyID_from_url(self, url):
with urlopen(url) as u:
data = u.read()
data = Download(url, progress = False).bytes_obj
g = GPG.GPGHandler()
key_ids = g.get_sigs(data)
del(g)
@ -166,7 +216,7 @@ class detect(object):
# Get any easy ones out of the way first.
if name in digest_schemes:
return(name)
# Otherwise grab the first one that matches, in order from the .
# Otherwise grab the first one that matches
_digest_re = re.compile('^{0}$'.format(name.strip()), re.IGNORECASE)
for h in digest_schemes:
if _digest_re.search(h):
@ -774,14 +824,19 @@ class valid(object):
return(True)

def salt_hash(self, salthash):
_idents = ''.join([i.ident for i in crypt_map if i.ident])
_idents = ''.join([i.ident for i in crypt_map.values() if i.ident])
# noinspection PyStringFormat
_regex = re.compile('^(\$[{0}]\$)?[./0-9A-Za-z]{{0,16}}\$?'.format(
_idents))
_regex = re.compile('^(\$[{0}]\$)?[./0-9A-Za-z]{{0,16}}\$?'.format(_idents))
if not _regex.search(salthash):
return(False)
return(True)

def salt_hash_full(self, salthash, hash_type):
h = [re.sub('-', '', i.lower()).split()[0] for i in detect.any_hash(self, salthash, normalize = True)]
if hash_type.lower() not in h:
return(False)
return(True)

def plugin_name(self, name):
if len(name) == 0:
return(False)