diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 40fe8a4..2a7e06c 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -1,19 +1,15 @@ #!/usr/bin/env python3 import os -import re import sys import getopt -import requests import comanage_utils as utils +import osg_project_usermap as usermap SCRIPT = os.path.basename(__file__) ENDPOINT = "https://registry-test.cilogon.org/registry/" -TOPOLOGY_ENDPOINT = "https://topology.opensciencegrid.org/" OSG_CO_ID = 8 -CACHE_FILENAME = "COmanage_Projects_cache.txt" -CACHE_LIFETIME_HOURS = 0.5 _usage = f"""\ @@ -50,16 +46,11 @@ def usage(msg=None): sys.exit() -class Options: +class Options(usermap.Options): endpoint = ENDPOINT user = "co_7.project_script" osg_co_id = OSG_CO_ID - outfile = None authstr = None - filtergrp = None - ldap_config = None - min_users = 100 # Bail out before updating the file if we have fewer than this many users - localmaps = [] options = Options() @@ -76,7 +67,7 @@ def get_osg_co_groups__map(): def parse_options(args): try: - ops, args = getopt.getopt(args, 'u:c:l:d:f:g:e:o:h:n:m:') + ops, args = getopt.getopt(args, 'u:c:d:f:e:' + usermap.COMMON_OPTSTRING) except getopt.GetoptError: usage() @@ -85,105 +76,45 @@ def parse_options(args): passfd = None passfile = None - ldap_auth_path = None for op, arg in ops: + if usermap.parse_common_option(options, op, arg): continue if op == '-h': usage() if op == '-u': options.user = arg if op == '-c': options.osg_co_id = int(arg) - if op == '-l': ldap_config_path = arg if op == '-d': passfd = int(arg) if op == '-f': passfile = arg if op == '-e': options.endpoint = arg - if op == '-o': options.outfile = arg - if op == '-g': options.filtergrp = arg - if op == '-m': options.localmaps = arg.split(",") - if op == '-n': options.min_users = int(arg) try: user, passwd = utils.getpw(options.user, passfd, passfile) options.authstr = utils.mkauthstr(user, passwd) except PermissionError: usage("PASS required") - + try: - options.ldap_config = utils.read_ldap_conffile(ldap_config_path) + usermap.read_ldap_config(options) except utils.EmptyConfiguration: usage("LDAP Config File Required. Was empty or lacked a valid server configuration.") -def _deduplicate_list(items): - """ Deduplicate a list while maintaining order by converting it to a dictionary and then back to a list. - Used to ensure a consistent ordering for output group lists, since sets are unordered. + +def sort_groups_by_co_group_id(osguser_groups): + """ Order each user's groups by COmanage group ID, preserving the output ordering + of the pre-LDAP migration version of this script. """ - return list(dict.fromkeys(items)) - -def get_osguser_groups(filter_group_name=None): - ldap_users = utils.get_ldap_active_users_and_groups(filter_group_name=filter_group_name, config=options.ldap_config) - topology_projects = requests.get(f"{TOPOLOGY_ENDPOINT}/miscproject/json").json() - project_names = topology_projects.keys() - - # Get COManage group IDs to preserve ordering from pre-LDAP migration script behavior groups_ids = get_osg_co_groups__map() return { - user: sorted([g for g in groups if g in project_names], key = lambda g: groups_ids.get(g, 0)) - for user, groups in ldap_users.items() - if any(g in project_names for g in groups) + user: sorted(groups, key = lambda g: groups_ids.get(g, 0)) + for user, groups in osguser_groups.items() } -def parse_localmap(inputfile): - user_groupmap = dict() - with open(inputfile, 'r', encoding='utf-8') as file: - for line in file: - # Split up 3 semantic columns - split_line = line.strip().split(maxsplit=2) - if split_line[0] == "*" and len(split_line) == 3: - line_groups = re.split(r'[ ,]+', split_line[2]) - if split_line[1] in user_groupmap: - user_groupmap[split_line[1]] = _deduplicate_list(user_groupmap[split_line[1]] + line_groups) - else: - user_groupmap[split_line[1]] = line_groups - return user_groupmap - - -def merge_maps(maps): - merged_map = dict() - for projectmap in maps: - for key in projectmap.keys(): - if key in merged_map: - merged_map[key] = _deduplicate_list(merged_map[key] + projectmap[key]) - else: - merged_map[key] = projectmap[key] - return merged_map - - -def print_usermap_to_file(osguser_groups, file): - for osguser, groups in sorted(osguser_groups.items()): - print("* {} {}".format(osguser, ",".join(group.strip() for group in groups)), file=file) - - -def print_usermap(osguser_groups): - if options.outfile: - with open(options.outfile, "w") as w: - print_usermap_to_file(osguser_groups, w) - else: - print_usermap_to_file(osguser_groups, sys.stdout) - - def main(args): parse_options(args) - osguser_groups = get_osguser_groups(options.filtergrp) - - maps = [osguser_groups] - for localmap in options.localmaps: - maps.append(parse_localmap(localmap)) - osguser_groups_merged = merge_maps(maps) - - # Sanity check, confirm we have generated a "sane" amount of user -> group mappings - if len(osguser_groups_merged) < options.min_users: - raise RuntimeError(f"Refusing to update output file: only {len(osguser_groups_merged)} users found") - print_usermap(osguser_groups_merged) + osguser_groups = usermap.get_osguser_groups(options.ldap_config, options.filtergrp) + osguser_groups = sort_groups_by_co_group_id(osguser_groups) + usermap.write_usermap(osguser_groups, options.localmaps, options.min_users, options.outfile) if __name__ == "__main__": diff --git a/osg_project_usermap.py b/osg_project_usermap.py new file mode 100755 index 0000000..010d600 --- /dev/null +++ b/osg_project_usermap.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 + +import os +import re +import sys +import getopt +import requests +import comanage_utils as utils + + +SCRIPT = os.path.basename(__file__) +TOPOLOGY_ENDPOINT = "https://topology.opensciencegrid.org/" + + +_usage = f"""\ +usage: {SCRIPT} [OPTIONS] + +OPTIONS: + -l LDAP_CONFIG_PATH specify path to LDAP Config file for fallback-search servers + -o outfile specify output file (default: write to stdout) + -g filter_group filter users by group name (eg, 'ap1-login') + -m localmaps specify a comma-delimited list of local HTCondor mapfiles to merge into outfile + -n min_users Specify minimum number of users required to update the output file (default: 100) + -h display this help text + +{utils.LDAP_CONFIG_USAGE_MESSAGE} + +""" + +def usage(msg=None): + if msg: + print(msg + "\n", file=sys.stderr) + + print(_usage, file=sys.stderr) + sys.exit() + + +class Options: + outfile = None + filtergrp = None + ldap_config_path = None + ldap_config = None + min_users = 100 # Bail out before updating the file if we have fewer than this many users + localmaps = [] + + +options = Options() + + +# Options shared with osg-comanage-project-usermap.py, which extends this script. +COMMON_OPTSTRING = 'l:o:g:m:n:h' + + +def parse_common_option(opts, op, arg): + """ Apply one of the COMMON_OPTSTRING options (other than -h) to opts. + Returns True if op was handled, False if the caller should handle it. + """ + if op == '-l': opts.ldap_config_path = arg + elif op == '-o': opts.outfile = arg + elif op == '-g': opts.filtergrp = arg + elif op == '-m': opts.localmaps = arg.split(",") + elif op == '-n': opts.min_users = int(arg) + else: + return False + return True + + +def read_ldap_config(opts): + """ Load the LDAP config file named by -l into opts.ldap_config. + Raises utils.EmptyConfiguration if no path was given or the file has no usable server section. + """ + if opts.ldap_config_path is None: + raise utils.EmptyConfiguration("No LDAP config file path given.") + opts.ldap_config = utils.read_ldap_conffile(opts.ldap_config_path) + + +def parse_options(args): + try: + ops, args = getopt.getopt(args, COMMON_OPTSTRING) + except getopt.GetoptError: + usage() + + if args: + usage("Extra arguments: %s" % repr(args)) + + for op, arg in ops: + if op == '-h': usage() + parse_common_option(options, op, arg) + + try: + read_ldap_config(options) + except utils.EmptyConfiguration: + usage("LDAP Config File Required. Was empty or lacked a valid server configuration.") + + +def _deduplicate_list(items): + """ Deduplicate a list while maintaining order by converting it to a dictionary and then back to a list. + Used to ensure a consistent ordering for output group lists, since sets are unordered. + """ + return list(dict.fromkeys(items)) + + +def get_topology_project_names(): + topology_projects = requests.get(f"{TOPOLOGY_ENDPOINT}/miscproject/json").json() + return set(topology_projects.keys()) + + +def get_osguser_groups(ldap_config, filter_group_name=None): + """ Map each active LDAP user to the list of their groups that are Topology projects. + Users with no project groups are omitted. Groups are listed in the order LDAP returns them. + """ + ldap_users = utils.get_ldap_active_users_and_groups(filter_group_name=filter_group_name, config=ldap_config) + project_names = get_topology_project_names() + return { + user: [g for g in groups if g in project_names] + for user, groups in ldap_users.items() + if any(g in project_names for g in groups) + } + + +def parse_localmap(inputfile): + user_groupmap = dict() + with open(inputfile, 'r', encoding='utf-8') as file: + for line in file: + # Split up 3 semantic columns + split_line = line.strip().split(maxsplit=2) + if split_line[0] == "*" and len(split_line) == 3: + line_groups = re.split(r'[ ,]+', split_line[2]) + if split_line[1] in user_groupmap: + user_groupmap[split_line[1]] = _deduplicate_list(user_groupmap[split_line[1]] + line_groups) + else: + user_groupmap[split_line[1]] = line_groups + return user_groupmap + + +def merge_maps(maps): + merged_map = dict() + for projectmap in maps: + for key in projectmap.keys(): + if key in merged_map: + merged_map[key] = _deduplicate_list(merged_map[key] + projectmap[key]) + else: + merged_map[key] = projectmap[key] + return merged_map + + +def print_usermap_to_file(osguser_groups, file): + for osguser, groups in sorted(osguser_groups.items()): + print("* {} {}".format(osguser, ",".join(group.strip() for group in groups)), file=file) + + +def print_usermap(osguser_groups, outfile=None): + if outfile: + with open(outfile, "w") as w: + print_usermap_to_file(osguser_groups, w) + else: + print_usermap_to_file(osguser_groups, sys.stdout) + + +def write_usermap(osguser_groups, localmaps=(), min_users=0, outfile=None): + """ Merge the local HTCondor mapfiles into osguser_groups, refuse to continue with fewer + than min_users users, and write the result to outfile (or stdout). + """ + maps = [osguser_groups] + [parse_localmap(localmap) for localmap in localmaps] + osguser_groups_merged = merge_maps(maps) + + # Sanity check, confirm we have generated a "sane" amount of user -> group mappings + if len(osguser_groups_merged) < min_users: + raise RuntimeError(f"Refusing to update output file: only {len(osguser_groups_merged)} users found") + print_usermap(osguser_groups_merged, outfile) + + +def main(args): + parse_options(args) + + osguser_groups = get_osguser_groups(options.ldap_config, options.filtergrp) + write_usermap(osguser_groups, options.localmaps, options.min_users, options.outfile) + + +if __name__ == "__main__": + try: + main(sys.argv[1:]) + except Exception as e: + sys.exit(e)