ParseDomain

#!/usr/bin/env python3
# TODO
# [.] search for unusual user fields (implemented in PowerView as Find-...Outlier
# [.] wordlist generator
# [ ] view: show ACL relationships
# [ ] view: show GPO/GPP objects
# [ ] view: show SPNs
# [x] quick filter for DCs
# [ ] quick filters from goddi
# [ ] filter shortcut for "in a group"
# [ ] filter shortcut for "in an OU"
# [ ] OU trees
# [ ] json output
# [ ] quick filters from https://social.technet.microsoft.com/wiki/contents/articles/5392.active-directory-ldap-syntax-filters.aspx?Sort=MostRecent
# [ ] parse DNs

"""
notes for converting to bloodhound

user json format:
{
    users: [
        {
            Properties: {'domain', 'objectsid', 'highvalue', 'enabled',
                         'lastlogon', 'pwdlastset', 'serviceprincipalnames', 'hasspn',
                         'displayname', 'email', 'title', 'homedirectory', 'description',
                         'userpassword', 'sensitive', 'dontreqpreauth', 'admincount'}
            Name: name@domain.fqdn
            PrimaryGroup: group@domain.fqdn
            Aces: [{'PrincipalName', 'PrincipalType', 'RightName', 'AceType'}]
            AllowedToDelegate: []
        }
    ]
    meta: {
       count: <int>
       type: users, groups, ...
    }
}
"""

import sys
import os
import re
import argparse
import json
import collections

import utils

# output settings
tree_indent = 4

# wordlist settings
wordlist_minimum_word_length = 3
wordlist_maximum_word_length = 10
wordlist_splits = (None, '/', '=', '.', ',', ';', ':', '-', '@', '#', '$', '&', '\\', '+', '?', '{', '}', '*')

# category and objectclass aliases
class_aliases = {'gpo': 'groupPolicyContainer',
                 'ou': 'organizationalUnit',
                 'scp': 'serviceConnectionPoint',
                 'td': 'trustedDomain',
                 'u': 'user',
                 'g': 'group',
                 'c': 'computer'}
category_aliases = {'gpo': 'Group-Policy-Container',
                    'ou': 'Organizational-Unit',
                    'scp': 'Service-Connection-Point',
                    'td': 'Trusted-Domain',
                    'p': 'Person',
                    'g': 'Group',
                    'c': 'Computer'}

property_aliases = {'dn': 'distinguishedname',
                    'spn': 'serviceprincipalname'}

quick_filters = {'laps': '"ms-mcs-admpwdexpirationtime" in item',
                 'admin': 'item["admincount"] == 1',
                 'dc': 'item["x-baseCategory"] == "Computer" and "OU=Domain Controllers" in item["distinguishedname"]',
                 'da': '"CN=Domain Admins," in str(item["memberOf"])',
                 'server': 'item["x-baseCategory"] == "Computer" and "Server" in item["operatingSystem"]',
                }

boring_user_fields = (
    'accountExpires'
    'adminCount'
    'badPasswordTime'
    'badPwdCount'
    'cn'
    'codePage'
    'company'
    'countryCode'
    'dSCorePropagationData'
    'description'
    'distinguishedName'
    'instanceType'
    'isCriticalSystemObject'
    'lastLogoff'
    'lastLogon'
    'logonCount'
    'memberOf'
    'msDS-SupportedEncryptionTypes'
    'name'
    'objectCategory'
    'objectClass'
    'objectGUID'
    'objectSid'
    'primaryGroupID'
    'pwdLastSet'
    'sAMAccountName'
    'sAMAccountType'
    'servicePrincipalName'
    'uSNChanged'
    'uSNCreated'
    'userAccountControl'
    'whenChanged'
    'whenCreated'
)

def parse_powershell_enumeration(enumeration, allow_incomplete=True):
    """
    Parse a PowerShell enumeration in the '{item1, item2}' format.

    :param enumeration: Enumeration to parse.
    :param allow_incomplete: Allow incomplete enumerations
    :return: Items in the enumeration
    """

    enumeration = enumeration.strip()

    match = re.match('{(.*)}', enumeration)
    if match:
        items = match.group(1)
        items = [item.strip() for item in items.split(', ')]

        # deal with ...
        if items[-1].endswith('...'):
            if allow_incomplete:
                items[-1] = items[-1][:-3]
            else:
                raise RuntimeError('Incomplete enumeration: {}'.format(enumeration))

        return items
    else:
        # single item
        return [enumeration]

def dn_to_fqdn(dn):
    return '.'.join(parse_dn(dn))

def parse_dn(dn):
    ret = []
    parts = dn.split(',')
    for part in parts:
        if part[0:3] not in ('CN=', 'DC=', 'OU='):
            raise RuntimeError('not a DN: {}'.format(dn))
        ret.append(part[3:])
    return ret

def find_item(dn, items):
    """
    Find an item based on its distinguishedname field

    :param dn: DN to search for
    :param items: Items to search in
    :return: Found item or None
    """

    for item in items:
        if 'distinguishedname' not in item:
            continue

        if item['distinguishedname'].strip().lower() == dn.strip().lower():
            return item

    return None

def print_group_tree(root, items, emails=False, full=False, level=0):
    """
    Print a tree by linking member fields with their items recursively

    :param root: Item to start at
    :param items: Items to base tree on
    :param level: Current level for indentation
    """

    global tree_indent

    # non-full tree only shows groups and persons
    if not full and root['x-baseCategory'].lower() not in ['group', 'person']:
        return

    content = root['name']
    if emails and 'mail' in root:
        content += ' <{}>'.format(root['mail'])
    if full:
        content = root['x-baseCategory'] + ': ' + content

    print(' ' * (level * tree_indent) + content)

    if 'member' in root:
        members = root['member']
        if not isinstance(members, list):
            # single item
            members = [members]
        for member in members:
            found = find_item(member, items)
            if found:
                # avoid finding the root object
                if found['distinguishedname'].lower() == root['distinguishedname'].lower():
                    continue

                print_group_tree(found, items, emails=emails, full=full, level=level + 1)
            else:
                print(' ' * ((level + 1) * tree_indent) + 'missing item: ' + member)

def read_powerview(fname):
    """
    Parse PowerView formatted data

    :param fname: File containing data in Powerview output format
    :param generator: Produce a generator?
    :return: Parsed data in format [{key: value,...}...]
    """

    #contents = fp.read().decode('utf-16').encode('utf-8').decode()
    #lines = contents.decode().splitlines()

    # read in file
    with open(fname, 'r') as fp:
        data = fp.read()

    items = []
    item = {}
    last_key = None

    def finish_item(item):
        # parse enumerations
        new_item = {}
        for key, value in item.items():
            if value.startswith('{'):
                value = parse_powershell_enumeration(value)
            new_item[key] = value
        item = new_item

        if item:
            items.append(item)

    for line in data.splitlines():
        if not line.strip():
            # next object
            finish_item(item)
            item = {}
            last_key = None
        elif ':' in line and line[0] != ' ':
            # add field to item
            parts = line.split(':')
            key = parts[0].strip().lower()
            value = ':'.join(parts[1:]).strip()
            item[key] = value
            last_key = key
        elif last_key:
            # append to last item
            item[last_key] += line.strip()

    finish_item(item)

    return items

def read_json(fname, generator=True):
    """
    Parse JSON from ldap3

    :param data: File containing data in JSON format
    :param generator: Produce a generator?
    :return: Parsed data in format [{key: value,...}...]
    """

    if generator:
        # produce a generator
        with open(fname, 'r') as fp:
            try:
                for item in utils.lazy_json_strict(fp):
                    yield item['attributes']
            except json.JSONDecodeError as e:
                utils.die('Failed to read the JSON. Try --no-generator? {}'.format(e))
    else:
        # produce a list
        with open(fname, 'r') as fp:
            data = fp.read()
        items = json.loads(data)
        ret = []
        for item in items:
            ret.append(item['attributes'])
        return ret

def read_files(files, generator=True):
    """
    Read in files

    :param files: Files to read in
    :param generator: Produce a generator?
    """

    for file_ in files:
        with open(file_, 'r') as fp:
            if fp.read(1) == '[':
                results = read_json(file_)
            else:
                results = read_powerview(file_)

            if generator:
                yield from results
            else:
                return list(results)

def item_categories(item):
    """
    Parse objectCategory for an item into a list

    :param item: LDAP item (dict)
    :return: List of categories
    """

    categories = []

    for category in item['objectCategory'].split(','):
        category = category.split('=')[1]
        if category == 'Schema':
            break
        categories.append(category)

    return categories

def fix_value(item, remove_binary=True):
    """
    Fix up a value. Recurses lists. Just removes binary values for now. Replaces
    binary values with '<binary>'

    :param itme: Item to fix.
    :param remove_binary: Remove binary item?
    :return: Fixed up item
    """

    if remove_binary and isinstance(item, dict) and 'encoded' in item:
        return '<binary>'
    elif isinstance(item, list):
        new_item = []
        for sub in item:
            new_item.append(fix_value(sub, remove_binary=remove_binary))
        return new_item
    else:
        return item

def fix_items(items, remove_binary=True):
    """
    Fix up a list of LDAP items. This:
    - flattens top-level single-item lists
    - fixes up each value with `fix_value`
    - adds a x-baseCategory field to the item (first category in the
      objectCategory field)

    :param items: List of LDAP items (dicts) to fix.
    :param remove_binary: Passed to fix_value (see fix_value).
    :return: Generator yielding fixed up new items.
    """

    for item in items:
        if not item:
            continue

        if 'objectCategory' not in item:
            utils.bad('item missing objectCategory: {}'.format(str(item)), status=True)
            continue

        new_item = utils.CaselessDictionary()
        for key, value in item.items():
            # flatten top level single-item lists
            if isinstance(value, list) and len(value) == 1:
                value = value[0]

            value = fix_value(value, remove_binary=remove_binary)
            new_item[key] = value

        # add x-baseCategory
        categories = item_categories(new_item)
        #new_item['x-categories'] = categories
        new_item['x-baseCategory'] = categories[0]

        # add x-groups
        if 'memberOf' in item:
            if isinstance(item['memberOf'], str):
                groups = [item['memberOf']]
            else:
                groups = item['memberOf']
        else:
            groups = []
        #new_item['x-groups'] = [dn_to_fqdn(group) for group in groups]

        yield new_item

def run_python_filter(item, filter):
    """
    Run a Python filter against an LDAP item.

    :param item: Item to filter
    :param filter: Filter to run
    :return: True if filter passes. False otherwise.
    """

    try:
        if eval(filter):
            return True
    except KeyError:
        pass
    except Exception as e:
        utils.die('Exception in your --python-filter {}: {}'.format(filter, e))
        utils.debug_exception(e)
    
    return False

def run_basic_filter(item, filter):
    """
    Run a basic (rudimentary) filter against an LDAP item. The format of this
    filter is key=value, key=*, or *=value.

    :param item: Item to filter
    :param filter: Filter to run
    :return: True if filter passes. False otherwise.
    """

    if '=' not in filter:
        raise RuntimeError('Bad filter: ' + filter)

    key, _, value = filter.partition('=')
    key = key.strip().lower()
    value = value.strip().lower()

    if not key or key == '*':
        # all keys
        keys = item.keys()
    else:
        # one key
        if key not in item:
            return False
        keys = [key]

    for key in keys:
        if not value or value == '*' or str(item[key]).lower() == value:
            return True

    return False

def run_regex_filter(item, filter):
    """
    Run a regex filter against an LDAP item. The format of this
    filter is key=regex or *=regex.

    :param item: Item to filter
    :param filter: Regex filter to run
    :return: True if filter passes. False otherwise.
    """

    if '=' not in filter:
        raise RuntimeError('bad filter: ' + filter)
    
    key, _, value = filter.partition('=')
    key = key.strip().lower()
    value = value.strip().lower()

    if not key or key == '*':
        # all keys
        keys = item.keys()
    else:
        # one key
        if key not in item:
            return False
        keys = [key]

    for key in keys:
        match = re.match(value, str(item[key]), re.IGNORECASE)
        if match:
            return True

    return False

def run_filters(item, filter_pairs, and_=False):
    """
    Run filters against an item

    :param item: Item to filter
    :param filter_pairs: List of (filter, method) tuples
    :param and_: All filters must pass (and instead of or)
    """

    for filter, method in filter_pairs:
        filter_match = method(item, filter)

        if not filter_match and and_:
            return False
        elif filter_match and not and_:
            return True
    
    return and_

def filter_items(items, basic_filters=None, regex_filters=None,
        python_filters=None, not_=False, and_=False):
    """
    Filter items

    :param items: Items to filter
    :param basic_filters: Basic/rudimentary filters
    :param regex_filters: Regex filters
    :param python_filters: Python filters
    :param not_: Invert results
    :param and_: All filters must pass (and instead of or)
    """

    filter_pairs = []

    if basic_filters:
        for filter in basic_filters:
            filter_pairs.append((filter, run_basic_filter))

    if python_filters:
        for filter in python_filters:
            filter_pairs.append((filter, run_python_filter))

    if regex_filters:
        for filter in regex_filters:
            filter_pairs.append((filter, run_regex_filter))

    for item in items:
        match = run_filters(item, filter_pairs, and_)

        # invert all the filters?
        if not_:
            match = not match

        if match:
            yield item


def print_spns():
    pass


def main():
    # info for argparse help
    category_alias_info = ', '.join(['{}={}'.format(short, full) for short, full in category_aliases.items()])
    quick_filter_info = ', '.join(quick_filters.keys())

    parser = argparse.ArgumentParser()
    group = parser.add_argument_group('parsing')
    group.add_argument('--no-generator', action='store_true',
            help='Do not use generator-producing lazy-loading JSON performance hack that works most of the time but not always')

    group = parser.add_argument_group('filters')
    group.add_argument('-f', '--filter', action='append',
            help='Filter by property (property=value)')
    group.add_argument('-F', '--filter-file',
            help='Filter by properties in file (property=value)')
    group.add_argument('-r', '--regex', action='append',
            help='Filter by property (property=regex)')
    group.add_argument('-P', '--python-filter', action='append',
            help='Filter with an evaled Python boolean expression. Each run exposes "item" which is {key: value, ...}')
    group.add_argument('-n', '--not', action='store_true', dest='not_',
            help='Invert the filters')
    group.add_argument('-a', '--and', action='store_true', dest='and_',
            help='Treat filters as an "and" instead of an "or". Narrow search down for search filter applied.')
    group.add_argument('-c', '--category', action='append',
            help='Filter objects by x-baseCategory. Aliases: {}'.format(category_aliases))
    group.add_argument('-q', '--quick-filter', action='append', choices=quick_filters.keys(),
            help="Add a quick filter. Options: {}".format(quick_filter_info))

    group = parser.add_argument_group('regular output')
    group.add_argument('-p', '--property', action='append',
            help='Select property to show')
    group.add_argument('-s', '--sort',
            help='Sort by field')
    group.add_argument('--pretty-print', action='store_true',
            help="Show pretty human-readable headers")
    group.add_argument('--no-keys', action='store_true',
            help="Do not show parameter keys")
    group.add_argument('--show-binary', action='store_true',
            help="Show binary blobs in their encoded formats")
    group.add_argument('-D', '--debug', action='store_true',
            help="Enable debug output")

    group = parser.add_argument_group('special output')
    group.add_argument('--spns', action='store_true',
            help="Print SPNs")
    group.add_argument('--emails', action='store_true',
            help="Produce a list of user emails")
    group.add_argument('--group-tree', action='store_true',
            help="Produce a user-group tree (for Get-DomainGroup)")
    group.add_argument('--full-tree', action='store_true',
            help="Show all objects in tree (instead of just people)")
    group.add_argument('--tree-emails', action='store_true',
            help="Show emails in tree")
    group.add_argument('--count', action='store_true',
            help="Produce tally of items")
    group.add_argument('--wordlist', action='store_true',
            help="Produce a wordlist from field values")
    group.add_argument('files', nargs='+',
            help='Files to parse')
    args = parser.parse_args()

    # -D/--debug
    if args.debug:
        utils.enable_debug()

    # read in files
    items = read_files(args.files, generator=not args.no_generator)

    basic_filters = []
    python_filters = []
    regex_filters = []

    # deal with filters
    # -P/--python-filter
    if args.python_filter:
        python_filters += args.python_filter

    # -f/--filter
    if args.filter:
        basic_filters += args.filter

    # -F/--filter-file
    if args.filter_file:
        with open(args.filter_file, 'r') as fp:
            basic_filters += list(fp.readlines())

    # -r/--regex
    if args.regex:
        regex_filters += args.regex

    # -q/--quick-filter
    if args.quick_filter:
        for name in args.quick_filter:
            python_filters.append(quick_filters[name])

    # -c/--category
    if args.category:
        for category in args.category:
            category = utils.alias(category, category_aliases, lower=True)
            basic_filters.append('x-baseCategory={}'.format(category))

    # fix items
    items = fix_items(items, remove_binary=not args.show_binary)

    # filter items
    if basic_filters or args.regex or python_filters:
        items = filter_items(items, basic_filters=basic_filters, regex_filters=regex_filters,
                python_filters=python_filters, not_=args.not_, and_=args.and_)

    # -s/--sort
    if args.sort:
        all_int = True
        for item in items:
            if args.sort in item:
                if not utils.is_int(item[args.sort]):
                    all_int = False

        def sort_helper(item):
            if args.sort in item:
                value = item[args.sort]

                if all_int:
                    return int(value)
                else:
                    return value
            else:
                if all_int:
                    return -999999999
                else:
                    return ''

        items = sorted(items, key=sort_helper, reverse=True)

    # output options
    # --group-tree and --full-tree
    if args.group_tree or args.full_tree:
        # print tree view
        # we need to iterate over items multiple times
        items = list(items)
        for item in items:
            if 'memberOf' not in item:
                # root item
                print_group_tree(item, items, emails=args.tree_emails, full=args.full_tree)
    # --emails
    elif args.emails:
        # print email view
        for item in items:
            if item['x-baseCategory'].lower() != 'person':
                continue

            if 'mail' not in item:
                continue

            if 'name' in item:
                name = item['name']
            else:
                name = item['mail']

            print('{} <{}>'.format(name, item['mail']))
    # --count
    elif args.count:
        total = 0
        counts = collections.defaultdict(int)
        for item in items:
            total += 1
            counts[item['x-baseCategory']] += 1

        for category in sorted(counts, key=lambda key: counts[key]):
            count = counts[category]
            print('{}: {}'.format(category, count))

        print()
        print('Total: {}'.format(total))
    # --wordlist
    elif args.wordlist:
        words = []

        def get_values(collection):
            for value in collection:
                if isinstance(value, str):
                    value = value.strip()

                    if value.startswith('{') and value.endswith('}'):
                        # skip UID
                        continue
                    elif value.startswith('S-'):
                        # skip SID
                        continue

                    yield value
                elif utils.is_iterable(value):
                    yield from get_values(value)
        
        for item in items:
            for value in get_values(item.values()):
                # split values (one split each)
                #for split in wordlist_splits:
                #    words += value.split(split)

                # split values (one split each)
                #split_values = [value]
                #for split in wordlist_splits:
                #    split_values += [v2 for v2 in [v.split(split) for v in split_values][0]]

                # split words
                split_values = [value]
                for split in wordlist_splits:
                    new_split_values = []
                    for split_value in split_values:
                        new_split_values += split_value.split(split)
                    split_values = new_split_values
                words += split_values

        # cleanup pass
        new_words = []
        for word in words:
            word = word.lower()
            if utils.is_int(word):
                continue
            new_words.append(word)
        words = new_words

        # sort and group by occurence
        words.sort()
        words.sort(key=collections.Counter(words).get, reverse=True)
        for word in utils.remove_grouped_duplicates(words):
            if len(word) < wordlist_minimum_word_length:
                # skip short words
                continue
            if len(word) > wordlist_maximum_word_length:
                # skip long words
                continue

            print(word)
    else:
        for number, item in enumerate(items):
            # print default property view
            # -p/--property
            if args.property:
                properties = []
                for prop in args.property:
                    properties += [p.lower() for p in prop.split(',')]

                properties = list(map(lambda p: utils.alias(p, property_aliases, lower=True), properties))

                if set(properties) & set((key.lower() for key in item.keys())) == set():
                    # skip objects without any properties we care about
                    continue
            else:
                properties = item.keys()

            # --pretty-print
            if args.pretty_print:
                print('{}:'.format(item['x-baseCategory']))

            # print fields
            for prop in sorted(properties):
                if prop in item:
                    value = item[prop]

                    indent = ' ' * 4
                    if args.no_keys:
                        if args.pretty_print:
                            print(indent + '- {}'.format(value))
                        else:
                            print('{}'.format(value))
                    else:
                        if args.pretty_print:
                            if isinstance(value, list):
                                print(indent + '- {}:'.format(prop))
                                for piece in value:
                                    print(indent * 2 + '- {}'.format(piece))
                            else:
                                print(indent + '- {}: {}'.format(prop, value))
                        else:
                            print('{}: {}'.format(prop, value))

            print()

if __name__ == '__main__':
    main()