diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/RangerAccessController.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/RangerAccessController.java index 7a2779b43b1c6b..eda20d58d3701a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/RangerAccessController.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/RangerAccessController.java @@ -95,9 +95,8 @@ public List evalRowFilterPolicies(UserIdentity curren RangerAccessRequestImpl request = createRequest(currentUser); // If the access type is not set here, it defaults to ANY1 ACCESS. // The internal logic of the ranger is to traverse all permission items. - // Since the ranger UI will set the access type to 'SELECT', - // we will keep it consistent with the UI here to avoid performance issues - request.setAccessType(DorisAccessType.SELECT.name()); + // Keep the access type consistent with the service definition to avoid traversing all policy items. + request.setAccessType(getSelectAccessType()); request.setResource(resource); if (LOG.isDebugEnabled()) { @@ -125,7 +124,7 @@ public Optional evalDataMaskPolicy(UserIdentity currentUser, Str String col) { RangerAccessResourceImpl resource = createResource(ctl, db, tbl, col); RangerAccessRequestImpl request = createRequest(currentUser); - request.setAccessType(DorisAccessType.SELECT.name()); + request.setAccessType(getSelectAccessType()); request.setResource(resource); if (LOG.isDebugEnabled()) { @@ -167,6 +166,10 @@ public Optional evalDataMaskPolicy(UserIdentity currentUser, Str protected abstract RangerAccessRequestImpl createRequest(UserIdentity currentUser); + protected String getSelectAccessType() { + return DorisAccessType.SELECT.name(); + } + protected abstract RangerAccessResourceImpl createResource(String ctl, String db, String tbl); protected abstract RangerAccessResourceImpl createResource(String ctl, String db, String tbl, String col); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/hive/RangerHiveAccessController.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/hive/RangerHiveAccessController.java index 74be9f24bfe2f5..88fa2c0c9ac8b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/hive/RangerHiveAccessController.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/authorizer/ranger/hive/RangerHiveAccessController.java @@ -42,6 +42,7 @@ import java.util.Collection; import java.util.Date; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.ScheduledThreadPoolExecutor; @@ -70,14 +71,16 @@ public RangerHiveAccessController(Map properties, private RangerAccessRequestImpl createRequest(UserIdentity currentUser, HiveAccessType accessType) { RangerAccessRequestImpl request = createRequest(currentUser); - if (accessType == HiveAccessType.USE) { - request.setAccessType(RangerPolicyEngine.ANY_ACCESS); - } else { - request.setAccessType(accessType.name().toLowerCase()); - } + request.setAccessType(toRangerAccessType(accessType)); return request; } + static String toRangerAccessType(HiveAccessType accessType) { + return accessType == HiveAccessType.USE + ? RangerPolicyEngine.ANY_ACCESS + : accessType.name().toLowerCase(Locale.ROOT); + } + @Override protected RangerAccessRequestImpl createRequest(UserIdentity currentUser) { RangerAccessRequestImpl request = new RangerAccessRequestImpl(); @@ -94,6 +97,11 @@ protected RangerAccessRequestImpl createRequest(UserIdentity currentUser) { return request; } + @Override + protected String getSelectAccessType() { + return toRangerAccessType(HiveAccessType.SELECT); + } + private void checkPrivileges(UserIdentity currentUser, HiveAccessType accessType, List hiveResources) throws AuthorizationException { List requests = new ArrayList<>(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/authorizer/ranger/hive/RangerHiveAccessControllerTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/authorizer/ranger/hive/RangerHiveAccessControllerTest.java new file mode 100644 index 00000000000000..95c3cc2a690c01 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/authorizer/ranger/hive/RangerHiveAccessControllerTest.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog.authorizer.ranger.hive; + +import org.apache.doris.analysis.UserIdentity; + +import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl; +import org.apache.ranger.plugin.policyengine.RangerPolicyEngine; +import org.apache.ranger.plugin.service.RangerBasePlugin; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.lang.reflect.Field; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class RangerHiveAccessControllerTest { + @Test + public void testRangerAccessTypeMapping() { + Assert.assertEquals("select", RangerHiveAccessController.toRangerAccessType(HiveAccessType.SELECT)); + Assert.assertEquals("update", RangerHiveAccessController.toRangerAccessType(HiveAccessType.UPDATE)); + Assert.assertEquals(RangerPolicyEngine.ANY_ACCESS, + RangerHiveAccessController.toRangerAccessType(HiveAccessType.USE)); + } + + @Test + public void testPolicyRequestsUseLowerCaseSelect() throws Exception { + RangerHiveAccessController controller = Mockito.mock( + RangerHiveAccessController.class, Mockito.CALLS_REAL_METHODS); + Field lifecycleLock = RangerHiveAccessController.class.getDeclaredField("lifecycleLock"); + lifecycleLock.setAccessible(true); + lifecycleLock.set(controller, new ReentrantReadWriteLock()); + RangerBasePlugin plugin = Mockito.mock(RangerBasePlugin.class); + UserIdentity currentUser = UserIdentity.createAnalyzedUserIdentWithIp("user", "%"); + RangerAccessRequestImpl rowFilterRequest = new RangerAccessRequestImpl(); + RangerAccessRequestImpl dataMaskRequest = new RangerAccessRequestImpl(); + + Mockito.doReturn(rowFilterRequest, dataMaskRequest).when(controller).createRequest(currentUser); + Mockito.doReturn(plugin).when(controller).getPlugin(); + + controller.evalRowFilterPolicies(currentUser, "catalog", "database", "table"); + controller.evalDataMaskPolicy(currentUser, "catalog", "database", "table", "column"); + + Assert.assertEquals("select", rowFilterRequest.getAccessType()); + Assert.assertEquals("select", dataMaskRequest.getAccessType()); + } +} diff --git a/regression-test/data/ranger_p2/test_ranger_hive_lowercase_access_type.out b/regression-test/data/ranger_p2/test_ranger_hive_lowercase_access_type.out new file mode 100644 index 00000000000000..0a48252701f19e --- /dev/null +++ b/regression-test/data/ranger_p2/test_ranger_hive_lowercase_access_type.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !lowercase_access_type_first -- +2 \N + +-- !lowercase_access_type_warmed_up -- +2 \N + diff --git a/regression-test/suites/ranger_p2/test_ranger_hive_lowercase_access_type.groovy b/regression-test/suites/ranger_p2/test_ranger_hive_lowercase_access_type.groovy new file mode 100644 index 00000000000000..876bd0b70193bc --- /dev/null +++ b/regression-test/suites/ranger_p2/test_ranger_hive_lowercase_access_type.groovy @@ -0,0 +1,188 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.ranger.RangerClient +import org.apache.ranger.plugin.model.RangerPolicy +import org.apache.ranger.plugin.model.RangerService + + +suite("test_ranger_hive_lowercase_access_type", "p2,ranger,external") { + String enableRangerTest = context.config.otherConfigs.get("enableRangerTest") + String enableHiveTest = context.config.otherConfigs.get("enableHiveTest") + + if (!enableRangerTest?.equalsIgnoreCase("true") || !enableHiveTest?.equalsIgnoreCase("true")) { + logger.info("skip Ranger-Hive lowercase access type case because Ranger or Hive test is not enabled") + return + } + + String rangerEndpoint = context.config.otherConfigs.get("rangerEndpoint") + String rangerUser = context.config.otherConfigs.get("rangerUser") + String rangerPassword = context.config.otherConfigs.get("rangerPassword") + String rangerServiceName = context.config.otherConfigs.get("rangerServiceName") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String hmsPort = context.config.otherConfigs.get("hive3HmsPort") + String hiveServerPort = context.config.otherConfigs.get("hive3ServerPort") + String rangerHiveServiceName = "${rangerServiceName}_hive" + + String catalog = "ranger_hive_lowercase_catalog" + String database = "ranger_hive_lowercase_db" + String table = "ranger_hive_lowercase_tbl" + String user = "ranger_hive_lowercase_user" + String password = "C123_567p" + String accessPolicyName = "doris_ranger_hive_lowercase_access" + String rowFilterPolicyName = "doris_ranger_hive_lowercase_row_filter" + String dataMaskPolicyName = "doris_ranger_hive_lowercase_data_mask" + List policyNames = [accessPolicyName, rowFilterPolicyName, dataMaskPolicyName] + + RangerClient rangerClient = new RangerClient( + "http://${rangerEndpoint}", "simple", rangerUser, rangerPassword, null) + boolean createdHiveService = false + + def deletePolicyIfExists = { String policyName -> + try { + rangerClient.deletePolicy(rangerHiveServiceName, policyName) + } catch (Exception e) { + logger.info("Ranger policy ${policyName} does not exist: ${e.getMessage()}") + } + } + + def tableResources = { + Map resources = new HashMap<>() + resources.put("database", new RangerPolicy.RangerPolicyResource(database)) + resources.put("table", new RangerPolicy.RangerPolicyResource(table)) + return resources + } + + def accessResources = { + Map resources = tableResources() + resources.put("column", new RangerPolicy.RangerPolicyResource("*")) + return resources + } + + try { + try { + RangerService hiveService = rangerClient.getService(rangerHiveServiceName) + assertEquals("hive", hiveService.getType()) + } catch (Exception e) { + logger.info("Create Ranger-Hive service ${rangerHiveServiceName}: ${e.getMessage()}") + RangerService hiveService = new RangerService() + hiveService.setType("hive") + hiveService.setName(rangerHiveServiceName) + hiveService.setDisplayName(rangerHiveServiceName) + hiveService.setConfigs([ + "username": "hive", + "password": "hive", + "jdbc.driverClassName": "org.apache.hive.jdbc.HiveDriver", + "jdbc.url": "jdbc:hive2://${externalEnvIp}:${hiveServerPort}".toString() + ]) + rangerClient.createService(hiveService) + createdHiveService = true + } + + // Prepare the Hive data through a catalog without Ranger, so setup does not need policies. + sql """DROP CATALOG IF EXISTS `${catalog}`""" + sql """CREATE CATALOG `${catalog}` PROPERTIES ( + 'type' = 'hms', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}' + )""" + try_sql """DROP TABLE IF EXISTS `${catalog}`.`${database}`.`${table}`""" + try_sql """DROP DATABASE IF EXISTS `${catalog}`.`${database}`""" + sql """CREATE DATABASE `${catalog}`.`${database}`""" + sql """CREATE TABLE `${catalog}`.`${database}`.`${table}` ( + id BIGINT, + secret VARCHAR(20) + ) ENGINE=hive + PROPERTIES ('file_format' = 'parquet')""" + sql """INSERT INTO `${catalog}`.`${database}`.`${table}` VALUES + (1, 'first'), (2, 'second')""" + sql """DROP CATALOG `${catalog}`""" + + sql """DROP USER IF EXISTS '${user}'""" + sql """CREATE USER '${user}' IDENTIFIED BY '${password}'""" + + policyNames.each { deletePolicyIfExists(it) } + + RangerPolicy accessPolicy = new RangerPolicy() + accessPolicy.setService(rangerHiveServiceName) + accessPolicy.setName(accessPolicyName) + accessPolicy.setResources(accessResources()) + RangerPolicy.RangerPolicyItem accessPolicyItem = new RangerPolicy.RangerPolicyItem() + accessPolicyItem.setUsers([user]) + accessPolicyItem.setAccesses([new RangerPolicy.RangerPolicyItemAccess("select")]) + accessPolicy.setPolicyItems([accessPolicyItem]) + rangerClient.createPolicy(accessPolicy) + + RangerPolicy rowFilterPolicy = new RangerPolicy() + rowFilterPolicy.setService(rangerHiveServiceName) + rowFilterPolicy.setName(rowFilterPolicyName) + rowFilterPolicy.setPolicyType(RangerPolicy.POLICY_TYPE_ROWFILTER) + rowFilterPolicy.setResources(tableResources()) + RangerPolicy.RangerRowFilterPolicyItem rowFilterPolicyItem = new RangerPolicy.RangerRowFilterPolicyItem() + rowFilterPolicyItem.setUsers([user]) + rowFilterPolicyItem.setAccesses([new RangerPolicy.RangerPolicyItemAccess("select")]) + rowFilterPolicyItem.setRowFilterInfo(new RangerPolicy.RangerPolicyItemRowFilterInfo("id >= 2")) + rowFilterPolicy.setRowFilterPolicyItems([rowFilterPolicyItem]) + rangerClient.createPolicy(rowFilterPolicy) + + RangerPolicy dataMaskPolicy = new RangerPolicy() + dataMaskPolicy.setService(rangerHiveServiceName) + dataMaskPolicy.setName(dataMaskPolicyName) + dataMaskPolicy.setPolicyType(RangerPolicy.POLICY_TYPE_DATAMASK) + Map dataMaskResources = tableResources() + dataMaskResources.put("column", new RangerPolicy.RangerPolicyResource("secret")) + dataMaskPolicy.setResources(dataMaskResources) + RangerPolicy.RangerDataMaskPolicyItem dataMaskPolicyItem = new RangerPolicy.RangerDataMaskPolicyItem() + dataMaskPolicyItem.setUsers([user]) + dataMaskPolicyItem.setAccesses([new RangerPolicy.RangerPolicyItemAccess("select")]) + dataMaskPolicyItem.setDataMaskInfo( + new RangerPolicy.RangerPolicyItemDataMaskInfo("MASK_NULL", "", "")) + dataMaskPolicy.setDataMaskPolicyItems([dataMaskPolicyItem]) + rangerClient.createPolicy(dataMaskPolicy) + + // Recreate the catalog with Ranger-Hive enabled after policies are ready. + sql """CREATE CATALOG `${catalog}` PROPERTIES ( + 'type' = 'hms', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}', + 'access_controller.properties.ranger.service.name' = '${rangerHiveServiceName}', + 'access_controller.class' = + 'org.apache.doris.catalog.authorizer.ranger.hive.RangerHiveAccessControllerFactory' + )""" + waitPolicyEffect() + + def tokens = context.config.jdbcUrl.split('/') + def defaultJdbcUrl = tokens[0] + "//" + tokens[2] + "/?" + connect(user, password, defaultJdbcUrl) { + String query = "SELECT id, secret FROM `${catalog}`.`${database}`.`${table}` ORDER BY id" + + // The first query initializes Ranger's optimized policy evaluator. The second query verifies + // that lowercase Hive access types still match after that evaluator has been warmed up. + order_qt_lowercase_access_type_first(query) + order_qt_lowercase_access_type_warmed_up(query) + } + } finally { + try_sql "DROP CATALOG IF EXISTS `${catalog}`" + policyNames.each { deletePolicyIfExists(it) } + try_sql "DROP USER IF EXISTS '${user}'" + if (createdHiveService) { + try { + rangerClient.deleteService(rangerHiveServiceName) + } catch (Exception e) { + logger.warn("Failed to delete Ranger-Hive service ${rangerHiveServiceName}: ${e.getMessage()}") + } + } + } +}