From be69fcf3d4c08b836f31102016d4eb91f4c612c0 Mon Sep 17 00:00:00 2001 From: clock Date: Tue, 22 Sep 2026 11:54:22 -0500 Subject: [PATCH] Add COmanage-agnostic osg-project-usermap.py and share its logic Move the LDAP/Topology lookup, localmap merging, sanity check and output code into a new osg-project-usermap.py that needs only LDAP access. osg-comanage-project-usermap.py now imports that script, sorts each user's groups by COmanage group ID, and reuses the shared writer, so its output is unchanged. This lets the usermap be generated without COmanage REST API credentials. --- osg-comanage-project-usermap.py | 99 +++-------------- osg_project_usermap.py | 184 ++++++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+), 84 deletions(-) create mode 100755 osg_project_usermap.py diff --git a/osg-comanage-project-usermap.py b/osg-comanage-project-usermap.py index 40fe8a4..2a7e06c 100755 --- a/osg-comanage-project-usermap.py +++ b/osg-comanage-project-usermap.py @@ -1,19 +1,15 @@ #!/usr/bin/env python3 import os -import re import sys import getopt -import requests import comanage_utils as utils +import osg_project_usermap as usermap SCRIPT = os.path.basename(__file__) ENDPOINT = "https://registry-test.cilogon.org/registry/" -TOPOLOGY_ENDPOINT = "https://topology.opensciencegrid.org/" OSG_CO_ID = 8 -CACHE_FILENAME = "COmanage_Projects_cache.txt" -CACHE_LIFETIME_HOURS = 0.5 _usage = f"""\ @@ -50,16 +46,11 @@ def usage(msg=None): sys.exit() -class Options: +class Options(usermap.Options): endpoint = ENDPOINT user = "co_7.project_script" osg_co_id = OSG_CO_ID - outfile = None authstr = None - filtergrp = None - ldap_config = None - min_users = 100 # Bail out before updating the file if we have fewer than this many users - localmaps = [] options = Options() @@ -76,7 +67,7 @@ def get_osg_co_groups__map(): def parse_options(args): try: - ops, args = getopt.getopt(args, 'u:c:l:d:f:g:e:o:h:n:m:') + ops, args = getopt.getopt(args, 'u:c:d:f:e:' + usermap.COMMON_OPTSTRING) except getopt.GetoptError: usage() @@ -85,105 +76,45 @@ def parse_options(args): passfd = None passfile = None - ldap_auth_path = None for op, arg in ops: + if usermap.parse_common_option(options, op, arg): continue if op == '-h': usage() if op == '-u': options.user = arg if op == '-c': options.osg_co_id = int(arg) - if op == '-l': ldap_config_path = arg if op == '-d': passfd = int(arg) if op == '-f': passfile = arg if op == '-e': options.endpoint = arg - if op == '-o': options.outfile = arg - if op == '-g': options.filtergrp = arg - if op == '-m': options.localmaps = arg.split(",") - if op == '-n': options.min_users = int(arg) try: user, passwd = utils.getpw(options.user, passfd, passfile) options.authstr = utils.mkauthstr(user, passwd) except PermissionError: usage("PASS required") - + try: - options.ldap_config = utils.read_ldap_conffile(ldap_config_path) + usermap.read_ldap_config(options) except utils.EmptyConfiguration: usage("LDAP Config File Required. Was empty or lacked a valid server configuration.") -def _deduplicate_list(items): - """ Deduplicate a list while maintaining order by converting it to a dictionary and then back to a list. - Used to ensure a consistent ordering for output group lists, since sets are unordered. + +def sort_groups_by_co_group_id(osguser_groups): + """ Order each user's groups by COmanage group ID, preserving the output ordering + of the pre-LDAP migration version of this script. """ - return list(dict.fromkeys(items)) - -def get_osguser_groups(filter_group_name=None): - ldap_users = utils.get_ldap_active_users_and_groups(filter_group_name=filter_group_name, config=options.ldap_config) - topology_projects = requests.get(f"{TOPOLOGY_ENDPOINT}/miscproject/json").json() - project_names = topology_projects.keys() - - # Get COManage group IDs to preserve ordering from pre-LDAP migration script behavior groups_ids = get_osg_co_groups__map() return { - user: sorted([g for g in groups if g in project_names], key = lambda g: groups_ids.get(g, 0)) - for user, groups in ldap_users.items() - if any(g in project_names for g in groups) + user: sorted(groups, key = lambda g: groups_ids.get(g, 0)) + for user, groups in osguser_groups.items() } -def parse_localmap(inputfile): - user_groupmap = dict() - with open(inputfile, 'r', encoding='utf-8') as file: - for line in file: - # Split up 3 semantic columns - split_line = line.strip().split(maxsplit=2) - if split_line[0] == "*" and len(split_line) == 3: - line_groups = re.split(r'[ ,]+', split_line[2]) - if split_line[1] in user_groupmap: - user_groupmap[split_line[1]] = _deduplicate_list(user_groupmap[split_line[1]] + line_groups) - else: - user_groupmap[split_line[1]] = line_groups - return user_groupmap - - -def merge_maps(maps): - merged_map = dict() - for projectmap in maps: - for key in projectmap.keys(): - if key in merged_map: - merged_map[key] = _deduplicate_list(merged_map[key] + projectmap[key]) - else: - merged_map[key] = projectmap[key] - return merged_map - - -def print_usermap_to_file(osguser_groups, file): - for osguser, groups in sorted(osguser_groups.items()): - print("* {} {}".format(osguser, ",".join(group.strip() for group in groups)), file=file) - - -def print_usermap(osguser_groups): - if options.outfile: - with open(options.outfile, "w") as w: - print_usermap_to_file(osguser_groups, w) - else: - print_usermap_to_file(osguser_groups, sys.stdout) - - def main(args): parse_options(args) - osguser_groups = get_osguser_groups(options.filtergrp) - - maps = [osguser_groups] - for localmap in options.localmaps: - maps.append(parse_localmap(localmap)) - osguser_groups_merged = merge_maps(maps) - - # Sanity check, confirm we have generated a "sane" amount of user -> group mappings - if len(osguser_groups_merged) < options.min_users: - raise RuntimeError(f"Refusing to update output file: only {len(osguser_groups_merged)} users found") - print_usermap(osguser_groups_merged) + osguser_groups = usermap.get_osguser_groups(options.ldap_config, options.filtergrp) + osguser_groups = sort_groups_by_co_group_id(osguser_groups) + usermap.write_usermap(osguser_groups, options.localmaps, options.min_users, options.outfile) if __name__ == "__main__": diff --git a/osg_project_usermap.py b/osg_project_usermap.py new file mode 100755 index 0000000..010d600 --- /dev/null +++ b/osg_project_usermap.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 + +import os +import re +import sys +import getopt +import requests +import comanage_utils as utils + + +SCRIPT = os.path.basename(__file__) +TOPOLOGY_ENDPOINT = "https://topology.opensciencegrid.org/" + + +_usage = f"""\ +usage: {SCRIPT} [OPTIONS] + +OPTIONS: + -l LDAP_CONFIG_PATH specify path to LDAP Config file for fallback-search servers + -o outfile specify output file (default: write to stdout) + -g filter_group filter users by group name (eg, 'ap1-login') + -m localmaps specify a comma-delimited list of local HTCondor mapfiles to merge into outfile + -n min_users Specify minimum number of users required to update the output file (default: 100) + -h display this help text + +{utils.LDAP_CONFIG_USAGE_MESSAGE} + +""" + +def usage(msg=None): + if msg: + print(msg + "\n", file=sys.stderr) + + print(_usage, file=sys.stderr) + sys.exit() + + +class Options: + outfile = None + filtergrp = None + ldap_config_path = None + ldap_config = None + min_users = 100 # Bail out before updating the file if we have fewer than this many users + localmaps = [] + + +options = Options() + + +# Options shared with osg-comanage-project-usermap.py, which extends this script. +COMMON_OPTSTRING = 'l:o:g:m:n:h' + + +def parse_common_option(opts, op, arg): + """ Apply one of the COMMON_OPTSTRING options (other than -h) to opts. + Returns True if op was handled, False if the caller should handle it. + """ + if op == '-l': opts.ldap_config_path = arg + elif op == '-o': opts.outfile = arg + elif op == '-g': opts.filtergrp = arg + elif op == '-m': opts.localmaps = arg.split(",") + elif op == '-n': opts.min_users = int(arg) + else: + return False + return True + + +def read_ldap_config(opts): + """ Load the LDAP config file named by -l into opts.ldap_config. + Raises utils.EmptyConfiguration if no path was given or the file has no usable server section. + """ + if opts.ldap_config_path is None: + raise utils.EmptyConfiguration("No LDAP config file path given.") + opts.ldap_config = utils.read_ldap_conffile(opts.ldap_config_path) + + +def parse_options(args): + try: + ops, args = getopt.getopt(args, COMMON_OPTSTRING) + except getopt.GetoptError: + usage() + + if args: + usage("Extra arguments: %s" % repr(args)) + + for op, arg in ops: + if op == '-h': usage() + parse_common_option(options, op, arg) + + try: + read_ldap_config(options) + except utils.EmptyConfiguration: + usage("LDAP Config File Required. Was empty or lacked a valid server configuration.") + + +def _deduplicate_list(items): + """ Deduplicate a list while maintaining order by converting it to a dictionary and then back to a list. + Used to ensure a consistent ordering for output group lists, since sets are unordered. + """ + return list(dict.fromkeys(items)) + + +def get_topology_project_names(): + topology_projects = requests.get(f"{TOPOLOGY_ENDPOINT}/miscproject/json").json() + return set(topology_projects.keys()) + + +def get_osguser_groups(ldap_config, filter_group_name=None): + """ Map each active LDAP user to the list of their groups that are Topology projects. + Users with no project groups are omitted. Groups are listed in the order LDAP returns them. + """ + ldap_users = utils.get_ldap_active_users_and_groups(filter_group_name=filter_group_name, config=ldap_config) + project_names = get_topology_project_names() + return { + user: [g for g in groups if g in project_names] + for user, groups in ldap_users.items() + if any(g in project_names for g in groups) + } + + +def parse_localmap(inputfile): + user_groupmap = dict() + with open(inputfile, 'r', encoding='utf-8') as file: + for line in file: + # Split up 3 semantic columns + split_line = line.strip().split(maxsplit=2) + if split_line[0] == "*" and len(split_line) == 3: + line_groups = re.split(r'[ ,]+', split_line[2]) + if split_line[1] in user_groupmap: + user_groupmap[split_line[1]] = _deduplicate_list(user_groupmap[split_line[1]] + line_groups) + else: + user_groupmap[split_line[1]] = line_groups + return user_groupmap + + +def merge_maps(maps): + merged_map = dict() + for projectmap in maps: + for key in projectmap.keys(): + if key in merged_map: + merged_map[key] = _deduplicate_list(merged_map[key] + projectmap[key]) + else: + merged_map[key] = projectmap[key] + return merged_map + + +def print_usermap_to_file(osguser_groups, file): + for osguser, groups in sorted(osguser_groups.items()): + print("* {} {}".format(osguser, ",".join(group.strip() for group in groups)), file=file) + + +def print_usermap(osguser_groups, outfile=None): + if outfile: + with open(outfile, "w") as w: + print_usermap_to_file(osguser_groups, w) + else: + print_usermap_to_file(osguser_groups, sys.stdout) + + +def write_usermap(osguser_groups, localmaps=(), min_users=0, outfile=None): + """ Merge the local HTCondor mapfiles into osguser_groups, refuse to continue with fewer + than min_users users, and write the result to outfile (or stdout). + """ + maps = [osguser_groups] + [parse_localmap(localmap) for localmap in localmaps] + osguser_groups_merged = merge_maps(maps) + + # Sanity check, confirm we have generated a "sane" amount of user -> group mappings + if len(osguser_groups_merged) < min_users: + raise RuntimeError(f"Refusing to update output file: only {len(osguser_groups_merged)} users found") + print_usermap(osguser_groups_merged, outfile) + + +def main(args): + parse_options(args) + + osguser_groups = get_osguser_groups(options.ldap_config, options.filtergrp) + write_usermap(osguser_groups, options.localmaps, options.min_users, options.outfile) + + +if __name__ == "__main__": + try: + main(sys.argv[1:]) + except Exception as e: + sys.exit(e)