Add COmanage-agnostic osg-project-usermap.py and share its logic

osg-comanage-project-usermap.py

@@ -1,19 +1,15 @@
 #!/usr/bin/env python3
 
 import os
-import re
 import sys
 import getopt
-import requests
 import comanage_utils as utils
+import osg_project_usermap as usermap
 
 
 SCRIPT = os.path.basename(__file__)
 ENDPOINT = "https://registry-test.cilogon.org/registry/"
-TOPOLOGY_ENDPOINT = "https://topology.opensciencegrid.org/"
 OSG_CO_ID = 8
-CACHE_FILENAME = "COmanage_Projects_cache.txt"
-CACHE_LIFETIME_HOURS = 0.5
 
 
 _usage = f"""\
@@ -50,16 +46,11 @@ def usage(msg=None):
     sys.exit()
 
 
-class Options:
+class Options(usermap.Options):
     endpoint = ENDPOINT
     user = "co_7.project_script"
     osg_co_id = OSG_CO_ID
-    outfile = None
     authstr = None
-    filtergrp = None
-    ldap_config = None
-    min_users = 100 # Bail out before updating the file if we have fewer than this many users
-    localmaps = []
 
 
 options = Options()
@@ -76,7 +67,7 @@ def get_osg_co_groups__map():
 
 def parse_options(args):
     try:
-        ops, args = getopt.getopt(args, 'u:c:l:d:f:g:e:o:h:n:m:')
+        ops, args = getopt.getopt(args, 'u:c:d:f:e:' + usermap.COMMON_OPTSTRING)
     except getopt.GetoptError:
         usage()
 
@@ -85,105 +76,45 @@ def parse_options(args):
 
     passfd = None
     passfile = None
-    ldap_auth_path = None
 
     for op, arg in ops:
+        if usermap.parse_common_option(options, op, arg): continue
         if op == '-h': usage()
         if op == '-u': options.user       = arg
         if op == '-c': options.osg_co_id  = int(arg)
-        if op == '-l': ldap_config_path   = arg
         if op == '-d': passfd             = int(arg)
         if op == '-f': passfile           = arg
         if op == '-e': options.endpoint   = arg
-        if op == '-o': options.outfile    = arg
-        if op == '-g': options.filtergrp  = arg
-        if op == '-m': options.localmaps  = arg.split(",")
-        if op == '-n': options.min_users  = int(arg)
 
     try:
         user, passwd = utils.getpw(options.user, passfd, passfile)
         options.authstr = utils.mkauthstr(user, passwd)
     except PermissionError:
         usage("PASS required")
-    
+
     try:
-        options.ldap_config = utils.read_ldap_conffile(ldap_config_path)
+        usermap.read_ldap_config(options)
     except utils.EmptyConfiguration:
         usage("LDAP Config File Required. Was empty or lacked a valid server configuration.")
 
-def _deduplicate_list(items):
-    """ Deduplicate a list while maintaining order by converting it to a dictionary and then back to a list. 
-    Used to ensure a consistent ordering for output group lists, since sets are unordered.
+
+def sort_groups_by_co_group_id(osguser_groups):
+    """ Order each user's groups by COmanage group ID, preserving the output ordering
+    of the pre-LDAP migration version of this script.
     """
-    return list(dict.fromkeys(items))
-
-def get_osguser_groups(filter_group_name=None):
-    ldap_users = utils.get_ldap_active_users_and_groups(filter_group_name=filter_group_name, config=options.ldap_config)
-    topology_projects = requests.get(f"{TOPOLOGY_ENDPOINT}/miscproject/json").json()
-    project_names = topology_projects.keys()
-
-    # Get COManage group IDs to preserve ordering from pre-LDAP migration script behavior
     groups_ids = get_osg_co_groups__map()
     return {
-        user: sorted([g for g in groups if g in project_names], key = lambda g: groups_ids.get(g, 0)) 
-        for user, groups in ldap_users.items()
-        if any(g in project_names for g in groups)
+        user: sorted(groups, key = lambda g: groups_ids.get(g, 0))
+        for user, groups in osguser_groups.items()
     }
 
 
-def parse_localmap(inputfile):
-    user_groupmap = dict()
-    with open(inputfile, 'r', encoding='utf-8') as file:
-        for line in file:
-            # Split up 3 semantic columns
-            split_line = line.strip().split(maxsplit=2)
-            if split_line[0] == "*" and len(split_line) == 3:
-                line_groups = re.split(r'[ ,]+', split_line[2])
-                if split_line[1] in user_groupmap:
-                    user_groupmap[split_line[1]] = _deduplicate_list(user_groupmap[split_line[1]] + line_groups)
-                else:
-                    user_groupmap[split_line[1]] = line_groups
-    return user_groupmap
-
-
-def merge_maps(maps):
-    merged_map = dict()
-    for projectmap in maps:
-        for key in projectmap.keys():
-            if key in merged_map:
-                merged_map[key] = _deduplicate_list(merged_map[key] + projectmap[key])
-            else:
-                merged_map[key] = projectmap[key]
-    return merged_map
-
-
-def print_usermap_to_file(osguser_groups, file):
-    for osguser, groups in sorted(osguser_groups.items()):
-        print("* {} {}".format(osguser, ",".join(group.strip() for group in groups)), file=file)
-
-
-def print_usermap(osguser_groups):
-    if options.outfile:
-        with open(options.outfile, "w") as w:
-            print_usermap_to_file(osguser_groups, w)
-    else:
-        print_usermap_to_file(osguser_groups, sys.stdout)
-
-
 def main(args):
     parse_options(args)
 
-    osguser_groups = get_osguser_groups(options.filtergrp)
-
-    maps = [osguser_groups]
-    for localmap in options.localmaps:
-        maps.append(parse_localmap(localmap))
-    osguser_groups_merged = merge_maps(maps)
-
-    # Sanity check, confirm we have generated a "sane" amount of user -> group mappings
-    if len(osguser_groups_merged) < options.min_users:
-        raise RuntimeError(f"Refusing to update output file: only {len(osguser_groups_merged)} users found")
-    print_usermap(osguser_groups_merged)
+    osguser_groups = usermap.get_osguser_groups(options.ldap_config, options.filtergrp)
+    osguser_groups = sort_groups_by_co_group_id(osguser_groups)
+    usermap.write_usermap(osguser_groups, options.localmaps, options.min_users, options.outfile)
 
 
 if __name__ == "__main__":

osg_project_usermap.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import sys
+import getopt
+import requests
+import comanage_utils as utils
+
+
+SCRIPT = os.path.basename(__file__)
+TOPOLOGY_ENDPOINT = "https://topology.opensciencegrid.org/"
+
+
+_usage = f"""\
+usage: {SCRIPT} [OPTIONS]
+
+OPTIONS:
+  -l LDAP_CONFIG_PATH    specify path to LDAP Config file for fallback-search servers
+  -o outfile             specify output file (default: write to stdout)
+  -g filter_group        filter users by group name (eg, 'ap1-login')
+  -m localmaps           specify a comma-delimited list of local HTCondor mapfiles to merge into outfile
+  -n min_users           Specify minimum number of users required to update the output file (default: 100)
+  -h                     display this help text
+
+{utils.LDAP_CONFIG_USAGE_MESSAGE}
+
+"""
+
+def usage(msg=None):
+    if msg:
+        print(msg + "\n", file=sys.stderr)
+
+    print(_usage, file=sys.stderr)
+    sys.exit()
+
+
+class Options:
+    outfile = None
+    filtergrp = None
+    ldap_config_path = None
+    ldap_config = None
+    min_users = 100 # Bail out before updating the file if we have fewer than this many users
+    localmaps = []
+
+
+options = Options()
+
+
+# Options shared with osg-comanage-project-usermap.py, which extends this script.
+COMMON_OPTSTRING = 'l:o:g:m:n:h'
+
+
+def parse_common_option(opts, op, arg):
+    """ Apply one of the COMMON_OPTSTRING options (other than -h) to opts.
+    Returns True if op was handled, False if the caller should handle it.
+    """
+    if   op == '-l': opts.ldap_config_path = arg
+    elif op == '-o': opts.outfile          = arg
+    elif op == '-g': opts.filtergrp        = arg
+    elif op == '-m': opts.localmaps        = arg.split(",")
+    elif op == '-n': opts.min_users        = int(arg)
+    else:
+        return False
+    return True
+
+
+def read_ldap_config(opts):
+    """ Load the LDAP config file named by -l into opts.ldap_config.
+    Raises utils.EmptyConfiguration if no path was given or the file has no usable server section.
+    """
+    if opts.ldap_config_path is None:
+        raise utils.EmptyConfiguration("No LDAP config file path given.")
+    opts.ldap_config = utils.read_ldap_conffile(opts.ldap_config_path)
+
+
+def parse_options(args):
+    try:
+        ops, args = getopt.getopt(args, COMMON_OPTSTRING)
+    except getopt.GetoptError:
+        usage()
+
+    if args:
+        usage("Extra arguments: %s" % repr(args))
+
+    for op, arg in ops:
+        if op == '-h': usage()
+        parse_common_option(options, op, arg)
+
+    try:
+        read_ldap_config(options)
+    except utils.EmptyConfiguration:
+        usage("LDAP Config File Required. Was empty or lacked a valid server configuration.")
+
+
+def _deduplicate_list(items):
+    """ Deduplicate a list while maintaining order by converting it to a dictionary and then back to a list. 
+    Used to ensure a consistent ordering for output group lists, since sets are unordered.
+    """
+    return list(dict.fromkeys(items))
+
+
+def get_topology_project_names():
+    topology_projects = requests.get(f"{TOPOLOGY_ENDPOINT}/miscproject/json").json()
+    return set(topology_projects.keys())
+
+
+def get_osguser_groups(ldap_config, filter_group_name=None):
+    """ Map each active LDAP user to the list of their groups that are Topology projects.
+    Users with no project groups are omitted.  Groups are listed in the order LDAP returns them.
+    """
+    ldap_users = utils.get_ldap_active_users_and_groups(filter_group_name=filter_group_name, config=ldap_config)
+    project_names = get_topology_project_names()
+    return {
+        user: [g for g in groups if g in project_names]
+        for user, groups in ldap_users.items()
+        if any(g in project_names for g in groups)
+    }
+
+
+def parse_localmap(inputfile):
+    user_groupmap = dict()
+    with open(inputfile, 'r', encoding='utf-8') as file:
+        for line in file:
+            # Split up 3 semantic columns
+            split_line = line.strip().split(maxsplit=2)
+            if split_line[0] == "*" and len(split_line) == 3:
+                line_groups = re.split(r'[ ,]+', split_line[2])
+                if split_line[1] in user_groupmap:
+                    user_groupmap[split_line[1]] = _deduplicate_list(user_groupmap[split_line[1]] + line_groups)
+                else:
+                    user_groupmap[split_line[1]] = line_groups
+    return user_groupmap
+
+
+def merge_maps(maps):
+    merged_map = dict()
+    for projectmap in maps:
+        for key in projectmap.keys():
+            if key in merged_map:
+                merged_map[key] = _deduplicate_list(merged_map[key] + projectmap[key])
+            else:
+                merged_map[key] = projectmap[key]
+    return merged_map
+
+
+def print_usermap_to_file(osguser_groups, file):
+    for osguser, groups in sorted(osguser_groups.items()):
+        print("* {} {}".format(osguser, ",".join(group.strip() for group in groups)), file=file)
+
+
+def print_usermap(osguser_groups, outfile=None):
+    if outfile:
+        with open(outfile, "w") as w:
+            print_usermap_to_file(osguser_groups, w)
+    else:
+        print_usermap_to_file(osguser_groups, sys.stdout)
+
+
+def write_usermap(osguser_groups, localmaps=(), min_users=0, outfile=None):
+    """ Merge the local HTCondor mapfiles into osguser_groups, refuse to continue with fewer
+    than min_users users, and write the result to outfile (or stdout).
+    """
+    maps = [osguser_groups] + [parse_localmap(localmap) for localmap in localmaps]
+    osguser_groups_merged = merge_maps(maps)
+
+    # Sanity check, confirm we have generated a "sane" amount of user -> group mappings
+    if len(osguser_groups_merged) < min_users:
+        raise RuntimeError(f"Refusing to update output file: only {len(osguser_groups_merged)} users found")
+    print_usermap(osguser_groups_merged, outfile)
+
+
+def main(args):
+    parse_options(args)
+
+    osguser_groups = get_osguser_groups(options.ldap_config, options.filtergrp)
+    write_usermap(osguser_groups, options.localmaps, options.min_users, options.outfile)
+
+
+if __name__ == "__main__":
+    try:
+        main(sys.argv[1:])
+    except Exception as e:
+        sys.exit(e)