branch-4.1: [fix](fe) Use service-specific Ranger select access types #67206

fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/RangerAccessController.java

@@ -95,9 +95,8 @@ public List<? extends RowFilterPolicy> evalRowFilterPolicies(UserIdentity curren
         RangerAccessRequestImpl request = createRequest(currentUser);
         // If the access type is not set here, it defaults to ANY1 ACCESS.
         // The internal logic of the ranger is to traverse all permission items.
-        // Since the ranger UI will set the access type to 'SELECT',
-        // we will keep it consistent with the UI here to avoid performance issues
-        request.setAccessType(DorisAccessType.SELECT.name());
+        // Keep the access type consistent with the service definition to avoid traversing all policy items.
+        request.setAccessType(getSelectAccessType());
         request.setResource(resource);
 
         if (LOG.isDebugEnabled()) {
@@ -125,7 +124,7 @@ public Optional<DataMaskPolicy> evalDataMaskPolicy(UserIdentity currentUser, Str
             String col) {
         RangerAccessResourceImpl resource = createResource(ctl, db, tbl, col);
         RangerAccessRequestImpl request = createRequest(currentUser);
-        request.setAccessType(DorisAccessType.SELECT.name());
+        request.setAccessType(getSelectAccessType());
         request.setResource(resource);
 
         if (LOG.isDebugEnabled()) {
@@ -167,6 +166,10 @@ public Optional<DataMaskPolicy> evalDataMaskPolicy(UserIdentity currentUser, Str
 
     protected abstract RangerAccessRequestImpl createRequest(UserIdentity currentUser);
 
+    protected String getSelectAccessType() {
+        return DorisAccessType.SELECT.name();
+    }
+
     protected abstract RangerAccessResourceImpl createResource(String ctl, String db, String tbl);
 
     protected abstract RangerAccessResourceImpl createResource(String ctl, String db, String tbl, String col);

fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/hive/RangerHiveAccessController.java

@@ -42,6 +42,7 @@
 import java.util.Collection;
 import java.util.Date;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
@@ -70,14 +71,16 @@ public RangerHiveAccessController(Map<String, String> properties,
 
     private RangerAccessRequestImpl createRequest(UserIdentity currentUser, HiveAccessType accessType) {
         RangerAccessRequestImpl request = createRequest(currentUser);
-        if (accessType == HiveAccessType.USE) {
-            request.setAccessType(RangerPolicyEngine.ANY_ACCESS);
-        } else {
-            request.setAccessType(accessType.name().toLowerCase());
-        }
+        request.setAccessType(toRangerAccessType(accessType));
         return request;
     }
 
+    static String toRangerAccessType(HiveAccessType accessType) {
+        return accessType == HiveAccessType.USE
+                ? RangerPolicyEngine.ANY_ACCESS
+                : accessType.name().toLowerCase(Locale.ROOT);
+    }
+
     @Override
     protected RangerAccessRequestImpl createRequest(UserIdentity currentUser) {
         RangerAccessRequestImpl request = new RangerAccessRequestImpl();
@@ -94,6 +97,11 @@ protected RangerAccessRequestImpl createRequest(UserIdentity currentUser) {
         return request;
     }
 
+    @Override
+    protected String getSelectAccessType() {
+        return toRangerAccessType(HiveAccessType.SELECT);
+    }
+
     private void checkPrivileges(UserIdentity currentUser, HiveAccessType accessType,
             List<RangerHiveResource> hiveResources) throws AuthorizationException {
         List<RangerAccessRequest> requests = new ArrayList<>();

fe/fe-core/src/test/java/org/apache/doris/catalog/authorizer/ranger/hive/RangerHiveAccessControllerTest.java

@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog.authorizer.ranger.hive;
+
+import org.apache.doris.analysis.UserIdentity;
+
+import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl;
+import org.apache.ranger.plugin.policyengine.RangerPolicyEngine;
+import org.apache.ranger.plugin.service.RangerBasePlugin;
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.lang.reflect.Field;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+public class RangerHiveAccessControllerTest {
+    @Test
+    public void testRangerAccessTypeMapping() {
+        Assert.assertEquals("select", RangerHiveAccessController.toRangerAccessType(HiveAccessType.SELECT));
+        Assert.assertEquals("update", RangerHiveAccessController.toRangerAccessType(HiveAccessType.UPDATE));
+        Assert.assertEquals(RangerPolicyEngine.ANY_ACCESS,
+                RangerHiveAccessController.toRangerAccessType(HiveAccessType.USE));
+    }
+
+    @Test
+    public void testPolicyRequestsUseLowerCaseSelect() throws Exception {
+        RangerHiveAccessController controller = Mockito.mock(
+                RangerHiveAccessController.class, Mockito.CALLS_REAL_METHODS);
+        Field lifecycleLock = RangerHiveAccessController.class.getDeclaredField("lifecycleLock");
+        lifecycleLock.setAccessible(true);
+        lifecycleLock.set(controller, new ReentrantReadWriteLock());
+        RangerBasePlugin plugin = Mockito.mock(RangerBasePlugin.class);
+        UserIdentity currentUser = UserIdentity.createAnalyzedUserIdentWithIp("user", "%");
+        RangerAccessRequestImpl rowFilterRequest = new RangerAccessRequestImpl();
+        RangerAccessRequestImpl dataMaskRequest = new RangerAccessRequestImpl();
+
+        Mockito.doReturn(rowFilterRequest, dataMaskRequest).when(controller).createRequest(currentUser);
+        Mockito.doReturn(plugin).when(controller).getPlugin();
+
+        controller.evalRowFilterPolicies(currentUser, "catalog", "database", "table");
+        controller.evalDataMaskPolicy(currentUser, "catalog", "database", "table", "column");
+
+        Assert.assertEquals("select", rowFilterRequest.getAccessType());
+        Assert.assertEquals("select", dataMaskRequest.getAccessType());
+    }
+}

regression-test/data/ranger_p2/test_ranger_hive_lowercase_access_type.out

@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !lowercase_access_type_first --
+2	\N
+
+-- !lowercase_access_type_warmed_up --
+2	\N
+

regression-test/suites/ranger_p2/test_ranger_hive_lowercase_access_type.groovy

@@ -0,0 +1,188 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.ranger.RangerClient
+import org.apache.ranger.plugin.model.RangerPolicy
+import org.apache.ranger.plugin.model.RangerService
+
+
+suite("test_ranger_hive_lowercase_access_type", "p2,ranger,external") {
+    String enableRangerTest = context.config.otherConfigs.get("enableRangerTest")
+    String enableHiveTest = context.config.otherConfigs.get("enableHiveTest")
+
+    if (!enableRangerTest?.equalsIgnoreCase("true") || !enableHiveTest?.equalsIgnoreCase("true")) {
+        logger.info("skip Ranger-Hive lowercase access type case because Ranger or Hive test is not enabled")
+        return
+    }
+
+    String rangerEndpoint = context.config.otherConfigs.get("rangerEndpoint")
+    String rangerUser = context.config.otherConfigs.get("rangerUser")
+    String rangerPassword = context.config.otherConfigs.get("rangerPassword")
+    String rangerServiceName = context.config.otherConfigs.get("rangerServiceName")
+    String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+    String hmsPort = context.config.otherConfigs.get("hive3HmsPort")
+    String hiveServerPort = context.config.otherConfigs.get("hive3ServerPort")
+    String rangerHiveServiceName = "${rangerServiceName}_hive"
+
+    String catalog = "ranger_hive_lowercase_catalog"
+    String database = "ranger_hive_lowercase_db"
+    String table = "ranger_hive_lowercase_tbl"
+    String user = "ranger_hive_lowercase_user"
+    String password = "C123_567p"
+    String accessPolicyName = "doris_ranger_hive_lowercase_access"
+    String rowFilterPolicyName = "doris_ranger_hive_lowercase_row_filter"
+    String dataMaskPolicyName = "doris_ranger_hive_lowercase_data_mask"
+    List<String> policyNames = [accessPolicyName, rowFilterPolicyName, dataMaskPolicyName]
+
+    RangerClient rangerClient = new RangerClient(
+            "http://${rangerEndpoint}", "simple", rangerUser, rangerPassword, null)
+    boolean createdHiveService = false
+
+    def deletePolicyIfExists = { String policyName ->
+        try {
+            rangerClient.deletePolicy(rangerHiveServiceName, policyName)
+        } catch (Exception e) {
+            logger.info("Ranger policy ${policyName} does not exist: ${e.getMessage()}")
+        }
+    }
+
+    def tableResources = {
+        Map<String, RangerPolicy.RangerPolicyResource> resources = new HashMap<>()
+        resources.put("database", new RangerPolicy.RangerPolicyResource(database))
+        resources.put("table", new RangerPolicy.RangerPolicyResource(table))
+        return resources
+    }
+
+    def accessResources = {
+        Map<String, RangerPolicy.RangerPolicyResource> resources = tableResources()
+        resources.put("column", new RangerPolicy.RangerPolicyResource("*"))
+        return resources
+    }
+
+    try {
+        try {
+            RangerService hiveService = rangerClient.getService(rangerHiveServiceName)
+            assertEquals("hive", hiveService.getType())
+        } catch (Exception e) {
+            logger.info("Create Ranger-Hive service ${rangerHiveServiceName}: ${e.getMessage()}")
+            RangerService hiveService = new RangerService()
+            hiveService.setType("hive")
+            hiveService.setName(rangerHiveServiceName)
+            hiveService.setDisplayName(rangerHiveServiceName)
+            hiveService.setConfigs([
+                    "username": "hive",
+                    "password": "hive",
+                    "jdbc.driverClassName": "org.apache.hive.jdbc.HiveDriver",
+                    "jdbc.url": "jdbc:hive2://${externalEnvIp}:${hiveServerPort}".toString()
+            ])
+            rangerClient.createService(hiveService)
+            createdHiveService = true
+        }
+
+        // Prepare the Hive data through a catalog without Ranger, so setup does not need policies.
+        sql """DROP CATALOG IF EXISTS `${catalog}`"""
+        sql """CREATE CATALOG `${catalog}` PROPERTIES (
+                'type' = 'hms',
+                'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}'
+        )"""
+        try_sql """DROP TABLE IF EXISTS `${catalog}`.`${database}`.`${table}`"""
+        try_sql """DROP DATABASE IF EXISTS `${catalog}`.`${database}`"""
+        sql """CREATE DATABASE `${catalog}`.`${database}`"""
+        sql """CREATE TABLE `${catalog}`.`${database}`.`${table}` (
+                id BIGINT,
+                secret VARCHAR(20)
+        ) ENGINE=hive
+        PROPERTIES ('file_format' = 'parquet')"""
+        sql """INSERT INTO `${catalog}`.`${database}`.`${table}` VALUES
+                (1, 'first'), (2, 'second')"""
+        sql """DROP CATALOG `${catalog}`"""
+
+        sql """DROP USER IF EXISTS '${user}'"""
+        sql """CREATE USER '${user}' IDENTIFIED BY '${password}'"""
+
+        policyNames.each { deletePolicyIfExists(it) }
+
+        RangerPolicy accessPolicy = new RangerPolicy()
+        accessPolicy.setService(rangerHiveServiceName)
+        accessPolicy.setName(accessPolicyName)
+        accessPolicy.setResources(accessResources())
+        RangerPolicy.RangerPolicyItem accessPolicyItem = new RangerPolicy.RangerPolicyItem()
+        accessPolicyItem.setUsers([user])
+        accessPolicyItem.setAccesses([new RangerPolicy.RangerPolicyItemAccess("select")])
+        accessPolicy.setPolicyItems([accessPolicyItem])
+        rangerClient.createPolicy(accessPolicy)
+
+        RangerPolicy rowFilterPolicy = new RangerPolicy()
+        rowFilterPolicy.setService(rangerHiveServiceName)
+        rowFilterPolicy.setName(rowFilterPolicyName)
+        rowFilterPolicy.setPolicyType(RangerPolicy.POLICY_TYPE_ROWFILTER)
+        rowFilterPolicy.setResources(tableResources())
+        RangerPolicy.RangerRowFilterPolicyItem rowFilterPolicyItem = new RangerPolicy.RangerRowFilterPolicyItem()
+        rowFilterPolicyItem.setUsers([user])
+        rowFilterPolicyItem.setAccesses([new RangerPolicy.RangerPolicyItemAccess("select")])
+        rowFilterPolicyItem.setRowFilterInfo(new RangerPolicy.RangerPolicyItemRowFilterInfo("id >= 2"))
+        rowFilterPolicy.setRowFilterPolicyItems([rowFilterPolicyItem])
+        rangerClient.createPolicy(rowFilterPolicy)
+
+        RangerPolicy dataMaskPolicy = new RangerPolicy()
+        dataMaskPolicy.setService(rangerHiveServiceName)
+        dataMaskPolicy.setName(dataMaskPolicyName)
+        dataMaskPolicy.setPolicyType(RangerPolicy.POLICY_TYPE_DATAMASK)
+        Map<String, RangerPolicy.RangerPolicyResource> dataMaskResources = tableResources()
+        dataMaskResources.put("column", new RangerPolicy.RangerPolicyResource("secret"))
+        dataMaskPolicy.setResources(dataMaskResources)
+        RangerPolicy.RangerDataMaskPolicyItem dataMaskPolicyItem = new RangerPolicy.RangerDataMaskPolicyItem()
+        dataMaskPolicyItem.setUsers([user])
+        dataMaskPolicyItem.setAccesses([new RangerPolicy.RangerPolicyItemAccess("select")])
+        dataMaskPolicyItem.setDataMaskInfo(
+                new RangerPolicy.RangerPolicyItemDataMaskInfo("MASK_NULL", "", ""))
+        dataMaskPolicy.setDataMaskPolicyItems([dataMaskPolicyItem])
+        rangerClient.createPolicy(dataMaskPolicy)
+
+        // Recreate the catalog with Ranger-Hive enabled after policies are ready.
+        sql """CREATE CATALOG `${catalog}` PROPERTIES (
+                'type' = 'hms',
+                'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}',
+                'access_controller.properties.ranger.service.name' = '${rangerHiveServiceName}',
+                'access_controller.class' =
+                        'org.apache.doris.catalog.authorizer.ranger.hive.RangerHiveAccessControllerFactory'
+        )"""
+        waitPolicyEffect()
+
+        def tokens = context.config.jdbcUrl.split('/')
+        def defaultJdbcUrl = tokens[0] + "//" + tokens[2] + "/?"
+        connect(user, password, defaultJdbcUrl) {
+            String query = "SELECT id, secret FROM `${catalog}`.`${database}`.`${table}` ORDER BY id"
+
+            // The first query initializes Ranger's optimized policy evaluator. The second query verifies
+            // that lowercase Hive access types still match after that evaluator has been warmed up.
+            order_qt_lowercase_access_type_first(query)
+            order_qt_lowercase_access_type_warmed_up(query)
+        }
+    } finally {
+        try_sql "DROP CATALOG IF EXISTS `${catalog}`"
+        policyNames.each { deletePolicyIfExists(it) }
+        try_sql "DROP USER IF EXISTS '${user}'"
+        if (createdHiveService) {
+            try {
+                rangerClient.deleteService(rangerHiveServiceName)
+            } catch (Exception e) {
+                logger.warn("Failed to delete Ranger-Hive service ${rangerHiveServiceName}: ${e.getMessage()}")
+            }
+        }
+    }
+}