diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSecretKeySnapshot.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSecretKeySnapshot.java index bd379a8d86e..b799f648edc 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSecretKeySnapshot.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSecretKeySnapshot.java @@ -17,60 +17,40 @@ package org.apache.hadoop.hdds.scm; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED; -import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_KERBEROS_KEYTAB_FILE_KEY; -import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_EXPIRY_DURATION; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_ROTATE_CHECK_DURATION; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_ROTATE_DURATION; -import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY; -import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY; -import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY; -import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.DELEGATION_REMOVER_SCAN_INTERVAL_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY; -import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.io.File; import java.io.IOException; -import java.net.InetAddress; import java.util.ArrayList; import java.util.List; -import java.util.Properties; import java.util.concurrent.TimeoutException; import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor; import org.apache.hadoop.hdds.scm.container.ContainerInfo; import org.apache.hadoop.hdds.scm.ha.SCMStateMachine; -import org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey; import org.apache.hadoop.hdds.security.symmetric.SecretKeyManager; import org.apache.hadoop.hdds.utils.IOUtils; -import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.ozone.AbstractKerberosTest; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.ozone.test.GenericTestUtils; import org.apache.ratis.util.ExitUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,7 +58,7 @@ * Integration test to verify that symmetric secret keys are correctly * synchronized from leader to follower during snapshot installation. */ -public final class TestSecretKeySnapshot { +public class TestSecretKeySnapshot extends AbstractKerberosTest { private static final Logger LOG = LoggerFactory .getLogger(TestSecretKeySnapshot.class); private static final long SNAPSHOT_THRESHOLD = 100; @@ -87,38 +67,41 @@ public final class TestSecretKeySnapshot { public static final int ROTATE_DURATION_MS = 30_000; public static final int EXPIRY_DURATION_MS = 61_000; - private MiniKdc miniKdc; - private OzoneConfiguration conf; - @TempDir - private File workDir; - private File ozoneKeytab; - private File spnegoKeytab; private MiniOzoneHAClusterImpl cluster; + @Override + protected boolean createTestUserPrincipal() { + return false; + } + + @Override + protected boolean enableSecurityAuthorizationByDefault() { + return false; + } + @BeforeEach public void init() throws Exception { - conf = new OzoneConfiguration(); - conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "localhost"); + setConf(new OzoneConfiguration()); + getConf().set(OZONE_SCM_CLIENT_ADDRESS_KEY, "localhost"); ExitUtils.disableSystemExit(); - startMiniKdc(); - setSecureConfig(); - createCredentialsInKDC(); + initKerberos(); + getConf().setBoolean(HDDS_BLOCK_TOKEN_ENABLED, true); - conf.setBoolean(ScmConfigKeys.OZONE_SCM_HA_RAFT_LOG_PURGE_ENABLED, true); - conf.setInt(ScmConfigKeys.OZONE_SCM_HA_RAFT_LOG_PURGE_GAP, LOG_PURGE_GAP); - conf.setLong(ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_THRESHOLD, + getConf().setBoolean(ScmConfigKeys.OZONE_SCM_HA_RAFT_LOG_PURGE_ENABLED, true); + getConf().setInt(ScmConfigKeys.OZONE_SCM_HA_RAFT_LOG_PURGE_GAP, LOG_PURGE_GAP); + getConf().setLong(ScmConfigKeys.OZONE_SCM_HA_RATIS_SNAPSHOT_THRESHOLD, SNAPSHOT_THRESHOLD); - conf.set(HDDS_SECRET_KEY_ROTATE_CHECK_DURATION, + getConf().set(HDDS_SECRET_KEY_ROTATE_CHECK_DURATION, ROTATE_CHECK_DURATION_MS + "ms"); - conf.set(HDDS_SECRET_KEY_ROTATE_DURATION, ROTATE_DURATION_MS + "ms"); - conf.set(HDDS_SECRET_KEY_EXPIRY_DURATION, EXPIRY_DURATION_MS + "ms"); - conf.set(DELEGATION_TOKEN_MAX_LIFETIME_KEY, ROTATE_DURATION_MS + "ms"); - conf.set(DELEGATION_REMOVER_SCAN_INTERVAL_KEY, ROTATE_CHECK_DURATION_MS + "ms"); + getConf().set(HDDS_SECRET_KEY_ROTATE_DURATION, ROTATE_DURATION_MS + "ms"); + getConf().set(HDDS_SECRET_KEY_EXPIRY_DURATION, EXPIRY_DURATION_MS + "ms"); + getConf().set(DELEGATION_TOKEN_MAX_LIFETIME_KEY, ROTATE_DURATION_MS + "ms"); + getConf().set(DELEGATION_REMOVER_SCAN_INTERVAL_KEY, ROTATE_CHECK_DURATION_MS + "ms"); - MiniOzoneHAClusterImpl.Builder builder = MiniOzoneCluster.newHABuilder(conf); + MiniOzoneHAClusterImpl.Builder builder = MiniOzoneCluster.newHABuilder(getConf()); builder .setSCMServiceId("TestSecretKeySnapshot") .setSCMServiceId("SCMServiceId") @@ -133,64 +116,10 @@ public void init() throws Exception { @AfterEach public void stop() { - miniKdc.stop(); + stopMiniKdc(); IOUtils.closeQuietly(cluster); } - private void createCredentialsInKDC() throws Exception { - ScmConfig scmConfig = conf.getObject(ScmConfig.class); - SCMHTTPServerConfig httpServerConfig = - conf.getObject(SCMHTTPServerConfig.class); - createPrincipal(ozoneKeytab, scmConfig.getKerberosPrincipal()); - createPrincipal(spnegoKeytab, httpServerConfig.getKerberosPrincipal()); - } - - private void createPrincipal(File keytab, String... principal) - throws Exception { - miniKdc.createPrincipal(keytab, principal); - } - - private void startMiniKdc() throws Exception { - Properties securityProperties = MiniKdc.createConf(); - miniKdc = new MiniKdc(securityProperties, workDir); - miniKdc.start(); - } - - private void setSecureConfig() throws IOException { - conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true); - String host = InetAddress.getLocalHost().getCanonicalHostName() - .toLowerCase(); - - conf.set(HADOOP_SECURITY_AUTHENTICATION, KERBEROS.name()); - - String curUser = UserGroupInformation.getCurrentUser().getUserName(); - conf.set(OZONE_ADMINISTRATORS, curUser); - - String realm = miniKdc.getRealm(); - String hostAndRealm = host + "@" + realm; - conf.set(HDDS_SCM_KERBEROS_PRINCIPAL_KEY, "scm/" + hostAndRealm); - conf.set(HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY, "HTTP_SCM/" + hostAndRealm); - conf.set(OZONE_OM_KERBEROS_PRINCIPAL_KEY, "scm/" + hostAndRealm); - conf.set(OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY, "HTTP_OM/" + hostAndRealm); - conf.set(HDDS_DATANODE_KERBEROS_PRINCIPAL_KEY, "scm/" + hostAndRealm); - - ozoneKeytab = new File(workDir, "scm.keytab"); - spnegoKeytab = new File(workDir, "http.keytab"); - - conf.set(HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, - ozoneKeytab.getAbsolutePath()); - conf.set(HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY, - spnegoKeytab.getAbsolutePath()); - conf.set(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY, - ozoneKeytab.getAbsolutePath()); - conf.set(OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE, - spnegoKeytab.getAbsolutePath()); - conf.set(HDDS_DATANODE_KERBEROS_KEYTAB_FILE_KEY, - ozoneKeytab.getAbsolutePath()); - - conf.setBoolean(HDDS_BLOCK_TOKEN_ENABLED, true); - } - @Test public void testInstallSnapshot() throws Exception { // Get the leader SCM diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSecretKeysApi.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSecretKeysApi.java index 59021773bb1..0b10f90a85c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSecretKeysApi.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestSecretKeysApi.java @@ -17,28 +17,15 @@ package org.apache.hadoop.hdds.scm; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_BLOCK_TOKEN_ENABLED; -import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_KERBEROS_KEYTAB_FILE_KEY; -import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_EXPIRY_DURATION; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_ROTATE_CHECK_DURATION; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SECRET_KEY_ROTATE_DURATION; -import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY; -import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY; -import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY; -import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.hdds.utils.HddsServerUtil.getSecretKeyClientForDatanode; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.DELEGATION_REMOVER_SCAN_INTERVAL_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -49,20 +36,17 @@ import jakarta.annotation.Nonnull; import java.io.File; import java.io.IOException; -import java.net.InetAddress; import java.util.List; -import java.util.Properties; import java.util.UUID; import java.util.concurrent.TimeoutException; import org.apache.hadoop.hdds.annotation.InterfaceAudience; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.SecretKeyProtocol; -import org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.security.symmetric.ManagedSecretKey; import org.apache.hadoop.hdds.utils.IOUtils; import org.apache.hadoop.ipc_.RemoteException; -import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.ozone.AbstractKerberosTest; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl; import org.apache.hadoop.ozone.om.OzoneManager; @@ -75,7 +59,6 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -84,96 +67,28 @@ */ @InterfaceAudience.Private -public final class TestSecretKeysApi { +public class TestSecretKeysApi extends AbstractKerberosTest { private static final Logger LOG = LoggerFactory .getLogger(TestSecretKeysApi.class); - private MiniKdc miniKdc; - private OzoneConfiguration conf; - @TempDir - private File workDir; - private File ozoneKeytab; - private File spnegoKeytab; - private File testUserKeytab; - private String testUserPrincipal; - private String ozonePrincipal; private MiniOzoneHAClusterImpl cluster; @BeforeEach public void init() throws Exception { - conf = new OzoneConfiguration(); - conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "localhost"); + setConf(new OzoneConfiguration()); + getConf().set(OZONE_SCM_CLIENT_ADDRESS_KEY, "localhost"); ExitUtils.disableSystemExit(); ExitUtil.disableSystemExit(); - startMiniKdc(); - setSecureConfig(); - createCredentialsInKDC(); + initKerberos(); } @AfterEach public void stop() { - miniKdc.stop(); + stopMiniKdc(); IOUtils.closeQuietly(cluster); } - private void createCredentialsInKDC() throws Exception { - SCMHTTPServerConfig httpServerConfig = - conf.getObject(SCMHTTPServerConfig.class); - createPrincipal(ozoneKeytab, ozonePrincipal); - createPrincipal(spnegoKeytab, httpServerConfig.getKerberosPrincipal()); - createPrincipal(testUserKeytab, testUserPrincipal); - } - - private void createPrincipal(File keytab, String... principal) - throws Exception { - miniKdc.createPrincipal(keytab, principal); - } - - private void startMiniKdc() throws Exception { - Properties securityProperties = MiniKdc.createConf(); - miniKdc = new MiniKdc(securityProperties, workDir); - miniKdc.start(); - } - - private void setSecureConfig() throws IOException { - conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true); - String host = InetAddress.getLocalHost().getCanonicalHostName() - .toLowerCase(); - - conf.set(HADOOP_SECURITY_AUTHENTICATION, KERBEROS.name()); - - String curUser = UserGroupInformation.getCurrentUser().getUserName(); - conf.set(OZONE_ADMINISTRATORS, curUser); - - String realm = miniKdc.getRealm(); - String hostAndRealm = host + "@" + realm; - ozonePrincipal = "scm/" + hostAndRealm; - conf.set(HDDS_SCM_KERBEROS_PRINCIPAL_KEY, ozonePrincipal); - conf.set(HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY, "HTTP_SCM/" + hostAndRealm); - conf.set(OZONE_OM_KERBEROS_PRINCIPAL_KEY, ozonePrincipal); - conf.set(OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY, "HTTP_OM/" + hostAndRealm); - conf.set(HDDS_DATANODE_KERBEROS_PRINCIPAL_KEY, ozonePrincipal); - - ozoneKeytab = new File(workDir, "scm.keytab"); - spnegoKeytab = new File(workDir, "http.keytab"); - testUserKeytab = new File(workDir, "testuser.keytab"); - testUserPrincipal = "test@" + realm; - - conf.set(HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, - ozoneKeytab.getAbsolutePath()); - conf.set(HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY, - spnegoKeytab.getAbsolutePath()); - conf.set(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY, - ozoneKeytab.getAbsolutePath()); - conf.set(OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE, - spnegoKeytab.getAbsolutePath()); - conf.set(HDDS_DATANODE_KERBEROS_KEYTAB_FILE_KEY, - ozoneKeytab.getAbsolutePath()); - - conf.setBoolean(HADOOP_SECURITY_AUTHORIZATION, true); - } - /** * Test secret key apis in happy case. */ @@ -183,11 +98,11 @@ public void testSecretKeyApiSuccess() throws Exception { enableBlockToken(); // set a low rotation period, of 1s, expiry is 3s, expect 3 active keys // at any moment. - conf.set(HDDS_SECRET_KEY_ROTATE_CHECK_DURATION, "100ms"); - conf.set(HDDS_SECRET_KEY_ROTATE_DURATION, "1s"); - conf.set(HDDS_SECRET_KEY_EXPIRY_DURATION, "3000ms"); - conf.set(DELEGATION_TOKEN_MAX_LIFETIME_KEY, "1500ms"); - conf.set(DELEGATION_REMOVER_SCAN_INTERVAL_KEY, "100ms"); + getConf().set(HDDS_SECRET_KEY_ROTATE_CHECK_DURATION, "100ms"); + getConf().set(HDDS_SECRET_KEY_ROTATE_DURATION, "1s"); + getConf().set(HDDS_SECRET_KEY_EXPIRY_DURATION, "3000ms"); + getConf().set(DELEGATION_TOKEN_MAX_LIFETIME_KEY, "1500ms"); + getConf().set(DELEGATION_REMOVER_SCAN_INTERVAL_KEY, "100ms"); startCluster(3); SecretKeyProtocol secretKeyProtocol = getSecretKeyProtocol(); @@ -258,10 +173,10 @@ public void testSecretKeyAfterSCMFailover() throws Exception { enableBlockToken(); // set a long duration period, so that no rotation happens during SCM // leader change. - conf.set(HDDS_SECRET_KEY_ROTATE_CHECK_DURATION, "10m"); - conf.set(HDDS_SECRET_KEY_ROTATE_DURATION, "1d"); - conf.set(HDDS_SECRET_KEY_EXPIRY_DURATION, "7d"); - conf.set(DELEGATION_TOKEN_MAX_LIFETIME_KEY, "5d"); + getConf().set(HDDS_SECRET_KEY_ROTATE_CHECK_DURATION, "10m"); + getConf().set(HDDS_SECRET_KEY_ROTATE_DURATION, "1d"); + getConf().set(HDDS_SECRET_KEY_EXPIRY_DURATION, "7d"); + getConf().set(DELEGATION_TOKEN_MAX_LIFETIME_KEY, "5d"); startCluster(3); SecretKeyProtocol securityProtocol = getSecretKeyProtocol(); @@ -288,7 +203,7 @@ private void testSecretKeyAuthorization() throws Exception { // is only available for Datanode and OM, any other authenticated user // can't access the protocol. SecretKeyProtocol secretKeyProtocol = - getSecretKeyProtocol(testUserPrincipal, testUserKeytab); + getSecretKeyProtocol(getTestUserPrincipal(), getTestUserKeytab()); RemoteException ex = assertThrows(RemoteException.class, secretKeyProtocol::getCurrentSecretKey); @@ -301,20 +216,20 @@ private void testSecretKeyAuthorization() throws Exception { @Test public void testSecretKeyWithoutAuthorization() throws Exception { enableBlockToken(); - conf.setBoolean(HADOOP_SECURITY_AUTHORIZATION, false); + getConf().setBoolean(HADOOP_SECURITY_AUTHORIZATION, false); startCluster(1); // When HADOOP_SECURITY_AUTHORIZATION is not enabled, any other // authenticated user can access the protocol. SecretKeyProtocol secretKeyProtocol = - getSecretKeyProtocol(testUserPrincipal, testUserKeytab); + getSecretKeyProtocol(getTestUserPrincipal(), getTestUserKeytab()); assertNotNull(secretKeyProtocol.getCurrentSecretKey()); } private void startCluster(int numSCMs) throws IOException, TimeoutException, InterruptedException { OzoneManager.setTestSecureOmFlag(true); - MiniOzoneHAClusterImpl.Builder builder = MiniOzoneCluster.newHABuilder(conf) + MiniOzoneHAClusterImpl.Builder builder = MiniOzoneCluster.newHABuilder(getConf()) .setSCMServiceId("TestSecretKey") .setNumOfStorageContainerManagers(numSCMs) .setNumOfOzoneManagers(1); @@ -325,7 +240,7 @@ private void startCluster(int numSCMs) @Nonnull private SecretKeyProtocol getSecretKeyProtocol() throws IOException { - return getSecretKeyProtocol(ozonePrincipal, ozoneKeytab); + return getSecretKeyProtocol(getOzonePrincipal(), getOzoneKeytab()); } @Nonnull @@ -335,10 +250,10 @@ private SecretKeyProtocol getSecretKeyProtocol( UserGroupInformation.loginUserFromKeytabAndReturnUGI( user, keyTab.getCanonicalPath()); ugi.setAuthenticationMethod(KERBEROS); - return getSecretKeyClientForDatanode(conf, ugi); + return getSecretKeyClientForDatanode(getConf(), ugi); } private void enableBlockToken() { - conf.setBoolean(HDDS_BLOCK_TOKEN_ENABLED, true); + getConf().setBoolean(HDDS_BLOCK_TOKEN_ENABLED, true); } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/AbstractKerberosTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/AbstractKerberosTest.java new file mode 100644 index 00000000000..ab908c0d436 --- /dev/null +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/AbstractKerberosTest.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone; + +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION; +import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_KERBEROS_KEYTAB_FILE_KEY; +import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_KERBEROS_PRINCIPAL_KEY; +import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY; +import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_PRINCIPAL_KEY; +import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY; +import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS; +import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY; +import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY; +import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS; + +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.nio.file.Files; +import java.util.Properties; +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig; +import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.UserGroupInformation; + +/** + * Shared MiniKdc / Kerberos setup for secure cluster integration tests (HDDS-15913). + */ +public abstract class AbstractKerberosTest { + + private MiniKdc miniKdc; + private OzoneConfiguration conf; + + private File workDir; + + /** Service keytab (scm.keytab); shared by SCM/OM/DN when useSharedServicePrincipal(). */ + private File ozoneKeytab; + private File spnegoKeytab; + private File testUserKeytab; + private String testUserPrincipal; + /** e.g. scm/host@REALM when useSharedServicePrincipal(). */ + private String ozonePrincipal; + + /** Separate OM keytab, only used when useSharedServicePrincipal() is false. */ + private File omKeytab; + + protected OzoneConfiguration getConf() { + return conf; + } + + protected void setConf(OzoneConfiguration conf) { + this.conf = conf; + } + + protected File getOzoneKeytab() { + return ozoneKeytab; + } + + protected String getOzonePrincipal() { + return ozonePrincipal; + } + + protected File getTestUserKeytab() { + return testUserKeytab; + } + + protected String getTestUserPrincipal() { + return testUserPrincipal; + } + + protected void initKerberos() throws Exception { + startMiniKdc(); + setSecureConfig(); + createCredentialsInKDC(); + } + + protected void startMiniKdc() throws Exception { + if (workDir == null) { + workDir = Files.createTempDirectory("kerberos").toFile(); + } + Properties securityProperties = MiniKdc.createConf(); + miniKdc = new MiniKdc(securityProperties, workDir); + miniKdc.start(); + } + + protected void stopMiniKdc() { + if (miniKdc != null) { + miniKdc.stop(); + } + FileUtils.deleteQuietly(workDir); + } + + protected void createPrincipal(File keytab, String... principal) + throws Exception { + miniKdc.createPrincipal(keytab, principal); + } + + protected void setSecureConfig() throws IOException { + conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true); + String host = InetAddress.getLocalHost().getCanonicalHostName() + .toLowerCase(); + + conf.set(HADOOP_SECURITY_AUTHENTICATION, kerberosAuthenticationValue()); + + String curUser = UserGroupInformation.getCurrentUser().getUserName(); + conf.set(OZONE_ADMINISTRATORS, curUser); + + String realm = miniKdc.getRealm(); + String hostAndRealm = host + "@" + realm; + + ozoneKeytab = new File(workDir, "scm.keytab"); + spnegoKeytab = new File(workDir, "http.keytab"); + + if (useSharedServicePrincipal()) { + ozonePrincipal = "scm/" + hostAndRealm; + conf.set(HDDS_SCM_KERBEROS_PRINCIPAL_KEY, ozonePrincipal); + conf.set(OZONE_OM_KERBEROS_PRINCIPAL_KEY, ozonePrincipal); + conf.set(HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, ozoneKeytab.getAbsolutePath()); + conf.set(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY, ozoneKeytab.getAbsolutePath()); + conf.set(HDDS_DATANODE_KERBEROS_PRINCIPAL_KEY, ozonePrincipal); + conf.set(HDDS_DATANODE_KERBEROS_KEYTAB_FILE_KEY, + ozoneKeytab.getAbsolutePath()); + } else { + ozonePrincipal = "scm/" + hostAndRealm; + conf.set(HDDS_SCM_KERBEROS_PRINCIPAL_KEY, ozonePrincipal); + conf.set(HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, ozoneKeytab.getAbsolutePath()); + omKeytab = new File(workDir, "om.keytab"); + conf.set(OZONE_OM_KERBEROS_PRINCIPAL_KEY, "om/" + hostAndRealm); + conf.set(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY, omKeytab.getAbsolutePath()); + } + + conf.set(HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY, "HTTP_SCM/" + hostAndRealm); + conf.set(OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY, "HTTP_OM/" + hostAndRealm); + conf.set(HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY, + spnegoKeytab.getAbsolutePath()); + conf.set(OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE, + spnegoKeytab.getAbsolutePath()); + + if (createTestUserPrincipal()) { + testUserKeytab = new File(workDir, "testuser.keytab"); + testUserPrincipal = "test@" + realm; + } + + if (enableSecurityAuthorizationByDefault()) { + conf.setBoolean(HADOOP_SECURITY_AUTHORIZATION, true); + } + } + + protected void createCredentialsInKDC() throws Exception { + SCMHTTPServerConfig httpServerConfig = + conf.getObject(SCMHTTPServerConfig.class); + createPrincipal(ozoneKeytab, conf.get(HDDS_SCM_KERBEROS_PRINCIPAL_KEY)); + if (!useSharedServicePrincipal()) { + createPrincipal(omKeytab, conf.get(OZONE_OM_KERBEROS_PRINCIPAL_KEY)); + } + createPrincipal(spnegoKeytab, httpServerConfig.getKerberosPrincipal()); + if (createTestUserPrincipal()) { + createPrincipal(testUserKeytab, testUserPrincipal); + } + } + + /** Whether SCM/OM (and optionally DN) share a single "scm/..." principal and keytab. */ + protected boolean useSharedServicePrincipal() { + return true; + } + + protected boolean createTestUserPrincipal() { + return true; + } + + protected boolean enableSecurityAuthorizationByDefault() { + return true; + } + + protected String kerberosAuthenticationValue() { + return KERBEROS.name(); + } +} diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDelegationToken.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDelegationToken.java index 444b6c313fa..c3346f7d901 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDelegationToken.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestDelegationToken.java @@ -17,10 +17,7 @@ package org.apache.hadoop.ozone; -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION; import static org.apache.hadoop.hdds.HddsConfigKeys.OZONE_METADATA_DIRS; -import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY; -import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY; @@ -30,45 +27,34 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_KEY; -import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY; -import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.net.ServerSocketUtil.getPort; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.DELEGATION_TOKEN_MAX_LIFETIME_KEY; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.INVALID_AUTH_METHOD; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_ERROR_OTHER; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.TOKEN_EXPIRED; import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.VOLUME_NOT_FOUND; -import static org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.slf4j.event.Level.INFO; -import java.io.File; import java.io.IOException; import java.net.InetAddress; import java.nio.file.Path; import java.nio.file.Paths; import java.security.KeyPair; import java.security.PrivilegedExceptionAction; -import java.util.Properties; import java.util.UUID; import java.util.stream.Stream; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.HddsTestUtils; -import org.apache.hadoop.hdds.scm.ScmConfig; import org.apache.hadoop.hdds.scm.client.ScmTopologyClient; import org.apache.hadoop.hdds.scm.ha.HASecurityUtils; -import org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.security.SecurityConfig; @@ -78,7 +64,6 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc_.Server; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; -import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.ozone.om.OMStorage; import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.ScmBlockLocationTestingClient; @@ -105,7 +90,7 @@ /** * Test class to for security enabled Ozone cluster. */ -public final class TestDelegationToken { +public class TestDelegationToken extends AbstractKerberosTest { private static final String TEST_USER = "testUgiUser@EXAMPLE.COM"; private static final String COMPONENT = "test"; @@ -116,23 +101,28 @@ public final class TestDelegationToken { @TempDir private Path folder; - @TempDir - private File workDir; - - private MiniKdc miniKdc; - private OzoneConfiguration conf; - private File scmKeytab; - private File spnegoKeytab; - private File omKeyTab; - private File testUserKeytab; - private String testUserPrincipal; + private StorageContainerManager scm; private OzoneManager om; - private String host; private String clusterId = UUID.randomUUID().toString(); private String scmId = UUID.randomUUID().toString(); private OzoneManagerProtocolClientSideTranslatorPB omClient; + @Override + protected boolean useSharedServicePrincipal() { + return false; + } + + @Override + protected boolean enableSecurityAuthorizationByDefault() { + return false; + } + + @Override + protected String kerberosAuthenticationValue() { + return "kerberos"; + } + public static Stream options() { return Stream.of(false, true); } @@ -145,28 +135,24 @@ public static void setup() { @BeforeEach public void init() { try { - conf = new OzoneConfiguration(); - conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "localhost"); + setConf(new OzoneConfiguration()); + getConf().set(OZONE_SCM_CLIENT_ADDRESS_KEY, "localhost"); - conf.setInt(OZONE_SCM_CLIENT_PORT_KEY, + getConf().setInt(OZONE_SCM_CLIENT_PORT_KEY, getPort(OZONE_SCM_CLIENT_PORT_DEFAULT, 100)); - conf.setInt(OZONE_SCM_DATANODE_PORT_KEY, + getConf().setInt(OZONE_SCM_DATANODE_PORT_KEY, getPort(OZONE_SCM_DATANODE_PORT_DEFAULT, 100)); - conf.setInt(OZONE_SCM_BLOCK_CLIENT_PORT_KEY, + getConf().setInt(OZONE_SCM_BLOCK_CLIENT_PORT_KEY, getPort(OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT, 100)); - conf.setInt(OZONE_SCM_SECURITY_SERVICE_PORT_KEY, + getConf().setInt(OZONE_SCM_SECURITY_SERVICE_PORT_KEY, getPort(OZONE_SCM_SECURITY_SERVICE_PORT_DEFAULT, 100)); DefaultMetricsSystem.setMiniClusterMode(true); final String path = folder.resolve("om-meta").toString(); Path metaDirPath = Paths.get(path, "om-meta"); - conf.set(OZONE_METADATA_DIRS, metaDirPath.toString()); - conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true); - conf.set(HADOOP_SECURITY_AUTHENTICATION, KERBEROS.name()); + getConf().set(OZONE_METADATA_DIRS, metaDirPath.toString()); - startMiniKdc(); - setSecureConfig(); - createCredentialsInKDC(); + initKerberos(); generateKeyPair(); } catch (Exception e) { LOG.error("Failed to initialize TestSecureOzoneCluster", e); @@ -187,70 +173,11 @@ public void stop() { } } - private void createCredentialsInKDC() throws Exception { - ScmConfig scmConfig = conf.getObject(ScmConfig.class); - SCMHTTPServerConfig httpServerConfig = - conf.getObject(SCMHTTPServerConfig.class); - createPrincipal(scmKeytab, scmConfig.getKerberosPrincipal()); - createPrincipal(spnegoKeytab, httpServerConfig.getKerberosPrincipal()); - createPrincipal(testUserKeytab, testUserPrincipal); - createPrincipal(omKeyTab, - conf.get(OZONE_OM_KERBEROS_PRINCIPAL_KEY)); - } - - private void createPrincipal(File keytab, String... principal) - throws Exception { - miniKdc.createPrincipal(keytab, principal); - } - - private void startMiniKdc() throws Exception { - Properties securityProperties = MiniKdc.createConf(); - miniKdc = new MiniKdc(securityProperties, workDir); - miniKdc.start(); - } - - private void stopMiniKdc() { - miniKdc.stop(); - } - - private void setSecureConfig() throws IOException { - conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true); - host = InetAddress.getLocalHost().getCanonicalHostName() - .toLowerCase(); - - conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos"); - - String curUser = UserGroupInformation.getCurrentUser().getUserName(); - conf.set(OZONE_ADMINISTRATORS, curUser); - - String realm = miniKdc.getRealm(); - String hostAndRealm = host + "@" + realm; - conf.set(HDDS_SCM_KERBEROS_PRINCIPAL_KEY, "scm/" + hostAndRealm); - conf.set(HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY, "HTTP_SCM/" + hostAndRealm); - conf.set(OZONE_OM_KERBEROS_PRINCIPAL_KEY, "om/" + hostAndRealm); - conf.set(OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY, "HTTP_OM/" + hostAndRealm); - - scmKeytab = new File(workDir, "scm.keytab"); - spnegoKeytab = new File(workDir, "http.keytab"); - omKeyTab = new File(workDir, "om.keytab"); - testUserKeytab = new File(workDir, "testuser.keytab"); - testUserPrincipal = "test@" + realm; - - conf.set(HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, - scmKeytab.getAbsolutePath()); - conf.set(HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY, - spnegoKeytab.getAbsolutePath()); - conf.set(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY, - omKeyTab.getAbsolutePath()); - conf.set(OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE, - spnegoKeytab.getAbsolutePath()); - } - private void initSCM() throws IOException { - SCMStorageConfig scmStore = new SCMStorageConfig(conf); + SCMStorageConfig scmStore = new SCMStorageConfig(getConf()); scmStore.setClusterId(clusterId); scmStore.setScmId(scmId); - HASecurityUtils.initializeSecurity(scmStore, conf, + HASecurityUtils.initializeSecurity(scmStore, getConf(), InetAddress.getLocalHost().getHostName(), true); scmStore.setPrimaryScmNodeId(scmId); // writes the version file properties @@ -272,7 +199,7 @@ private void initSCM() throws IOException { @MethodSource("options") public void testDelegationToken(boolean useIp) throws Exception { initSCM(); - scm = HddsTestUtils.getScmSimple(conf); + scm = HddsTestUtils.getScmSimple(getConf()); scm.start(); // Capture logs for assertions @@ -283,10 +210,10 @@ public void testDelegationToken(boolean useIp) throws Exception { // Generous token lifetime so the renewal-failure cases below do not race // token expiry. - conf.setLong(DELEGATION_TOKEN_MAX_LIFETIME_KEY, 60 * 1000L); + getConf().setLong(DELEGATION_TOKEN_MAX_LIFETIME_KEY, 60 * 1000L); // Setup secure OM for start - setupOm(conf); + setupOm(getConf()); //These are two very important lines: ProtobufRpcEngine uses ClientCache //which caches clients until no more references. Cache key is the @@ -312,7 +239,7 @@ public void testDelegationToken(boolean useIp) throws Exception { try { // Start OM - om.setCertClient(new CertificateClientTestImpl(conf)); + om.setCertClient(new CertificateClientTestImpl(getConf())); om.setScmTopologyClient(new ScmTopologyClient( new ScmBlockLocationTestingClient(null, null, 0))); om.start(); @@ -322,7 +249,7 @@ public void testDelegationToken(boolean useIp) throws Exception { // Get first OM client which will authenticate via Kerberos omClient = new OzoneManagerProtocolClientSideTranslatorPB( - OmTransportFactory.create(conf, ugi, null), + OmTransportFactory.create(getConf(), ugi, null), RandomStringUtils.secure().nextAscii(5)); // Assert if auth was successful via Kerberos @@ -355,7 +282,7 @@ public void testDelegationToken(boolean useIp) throws Exception { // Get Om client, this time authentication should happen via Token testUser.doAs((PrivilegedExceptionAction) () -> { omClient = new OzoneManagerProtocolClientSideTranslatorPB( - OmTransportFactory.create(conf, testUser, null), + OmTransportFactory.create(getConf(), testUser, null), RandomStringUtils.secure().nextAscii(5)); return null; }); @@ -383,7 +310,7 @@ public void testDelegationToken(boolean useIp) throws Exception { omClient.close(); UserGroupInformation.setLoginUser(ugi); omClient = new OzoneManagerProtocolClientSideTranslatorPB( - OmTransportFactory.create(conf, ugi, null), + OmTransportFactory.create(getConf(), ugi, null), RandomStringUtils.secure().nextAscii(5)); // Case 5: Test success of token cancellation. @@ -399,7 +326,7 @@ public void testDelegationToken(boolean useIp) throws Exception { // Get Om client, this time authentication using Token will fail as // token is not in cache anymore. omClient = new OzoneManagerProtocolClientSideTranslatorPB( - OmTransportFactory.create(conf, testUser, null), + OmTransportFactory.create(getConf(), testUser, null), RandomStringUtils.secure().nextAscii(5)); ex = assertThrows(OMException.class, () -> omClient.cancelDelegationToken(token)); @@ -425,7 +352,7 @@ private void assertDelegationTokenRenewalFailures( omClient.close(); UserGroupInformation.setLoginUser(ugi); omClient = new OzoneManagerProtocolClientSideTranslatorPB( - OmTransportFactory.create(conf, ugi, null), + OmTransportFactory.create(getConf(), ugi, null), RandomStringUtils.secure().nextAscii(5)); Token seedToken = omClient.getDelegationToken(new Text("om")); @@ -469,7 +396,7 @@ private void assertDelegationTokenRenewalFailures( } private void generateKeyPair() throws Exception { - SecurityConfig securityConfig = new SecurityConfig(conf); + SecurityConfig securityConfig = new SecurityConfig(getConf()); HDDSKeyGenerator keyGenerator = new HDDSKeyGenerator(securityConfig); KeyPair keyPair = keyGenerator.generateKey(); KeyStorage keyStorage = new KeyStorage(securityConfig, COMPONENT); @@ -484,10 +411,9 @@ private void setupOm(OzoneConfiguration config) throws Exception { omStore.initialize(); // OM uses scm/host@EXAMPLE.COM to access SCM - config.set(OZONE_OM_KERBEROS_PRINCIPAL_KEY, - "scm/" + host + "@" + miniKdc.getRealm()); - omKeyTab = new File(workDir, "scm.keytab"); - config.set(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY, omKeyTab.getAbsolutePath()); + config.set(OZONE_OM_KERBEROS_PRINCIPAL_KEY, getOzonePrincipal()); + config.set(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY, + getOzoneKeytab().getAbsolutePath()); OzoneManager.setTestSecureOmFlag(true); om = OzoneManager.createOm(config); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java index bb9fb9f6180..56e314f4669 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestSecureOzoneCluster.java @@ -35,18 +35,13 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_GRPC_PORT_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_RATIS_PORT_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SECURITY_SERVICE_PORT_KEY; -import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY; -import static org.apache.hadoop.hdds.scm.server.SCMHTTPServerConfig.ConfigStrings.HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.hdds.security.x509.exception.CertificateException.ErrorCode.ROLLBACK_ERROR; import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmSecurityClient; import static org.apache.hadoop.hdds.utils.HddsServerUtil.getValidInetsForCurrentHost; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ADMINISTRATORS; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_FAILOVER_MAX_ATTEMPTS_KEY; import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_SECURITY_ENABLED_KEY; import static org.apache.hadoop.ozone.OzoneConsts.SCM_SUB_CA; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE; -import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_KEYTAB_FILE_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_KERBEROS_PRINCIPAL_KEY; import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_S3_GPRC_SERVER_ENABLED; @@ -83,7 +78,6 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; -import java.util.Properties; import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; @@ -125,7 +119,6 @@ import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc_.Client; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; -import org.apache.hadoop.minikdc.MiniKdc; import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.common.Storage; @@ -163,6 +156,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.io.TempDir; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -170,7 +164,8 @@ /** * Test class to for security enabled Ozone cluster. */ -final class TestSecureOzoneCluster { +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class TestSecureOzoneCluster extends AbstractKerberosTest { private static final String COMPONENT = "om"; private static final String OM_CERT_SERIAL_ID = "9879877970576"; @@ -180,18 +175,6 @@ final class TestSecureOzoneCluster { private static final int CERT_GRACE_TIME_MS = 10 * 1000; // 10s private static final int DELEGATION_TOKEN_MAX_TIME_MS = 9 * 1000; // 9s - @TempDir - private static File workDir; - private static MiniKdc miniKdc; - private static OzoneConfiguration kdcConf; - private static File scmKeytab; - private static File spnegoKeytab; - private static File omKeytab; - private static File testUserKeytab; - private static String testUserPrincipal; - private static String host; - private static String realm; - @TempDir private File tempDir; private OzoneConfiguration conf; @@ -206,32 +189,38 @@ final class TestSecureOzoneCluster { private KeyPair keyPair; private Path omMetaDirPath; + @Override + protected boolean useSharedServicePrincipal() { + return false; + } + + @Override + protected boolean enableSecurityAuthorizationByDefault() { + return false; + } + + @Override + protected String kerberosAuthenticationValue() { + return "kerberos"; + } + @BeforeAll - static void setupKdc() throws Exception { + void setupKdc() throws Exception { ExitUtils.disableSystemExit(); DefaultMetricsSystem.setMiniClusterMode(true); - host = InetAddress.getLocalHost().getCanonicalHostName().toLowerCase(); - startMiniKdc(); - realm = miniKdc.getRealm(); - testUserPrincipal = "test@" + realm; - scmKeytab = new File(workDir, "scm.keytab"); - spnegoKeytab = new File(workDir, "http.keytab"); - omKeytab = new File(workDir, "om.keytab"); - testUserKeytab = new File(workDir, "testuser.keytab"); - kdcConf = new OzoneConfiguration(); - setSecureConfig(kdcConf); - createCredentialsInKDC(kdcConf); + setConf(new OzoneConfiguration()); + initKerberos(); } @AfterAll - static void tearDownKdc() { + void tearDownKdc() { stopMiniKdc(); } @BeforeEach void init() { try { - conf = new OzoneConfiguration(kdcConf); + conf = new OzoneConfiguration(getConf()); conf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, "localhost"); conf.setInt(OZONE_SCM_CLIENT_PORT_KEY, getFreePort()); @@ -284,49 +273,6 @@ void stop() throws Exception { } } - private static void createCredentialsInKDC(OzoneConfiguration conf) throws Exception { - createPrincipal(scmKeytab, conf.get(HDDS_SCM_KERBEROS_PRINCIPAL_KEY)); - createPrincipal(spnegoKeytab, conf.get(HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY)); - createPrincipal(omKeytab, conf.get(OZONE_OM_KERBEROS_PRINCIPAL_KEY)); - createPrincipal(testUserKeytab, testUserPrincipal); - } - - private static void createPrincipal(File keytab, String... principal) - throws Exception { - miniKdc.createPrincipal(keytab, principal); - } - - private static void startMiniKdc() throws Exception { - Properties securityProperties = MiniKdc.createConf(); - miniKdc = new MiniKdc(securityProperties, workDir); - miniKdc.start(); - } - - private static void stopMiniKdc() { - if (miniKdc != null) { - miniKdc.stop(); - } - } - - private static void setSecureConfig(OzoneConfiguration conf) throws IOException { - conf.setBoolean(OZONE_SECURITY_ENABLED_KEY, true); - conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos"); - - String curUser = UserGroupInformation.getCurrentUser().getUserName(); - conf.set(OZONE_ADMINISTRATORS, curUser); - - String hostAndRealm = host + "@" + realm; - conf.set(HDDS_SCM_KERBEROS_PRINCIPAL_KEY, "scm/" + hostAndRealm); - conf.set(HDDS_SCM_HTTP_KERBEROS_PRINCIPAL_KEY, "HTTP_SCM/" + hostAndRealm); - conf.set(OZONE_OM_KERBEROS_PRINCIPAL_KEY, "om/" + hostAndRealm); - conf.set(OZONE_OM_HTTP_KERBEROS_PRINCIPAL_KEY, "HTTP_OM/" + hostAndRealm); - - conf.set(HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, scmKeytab.getAbsolutePath()); - conf.set(HDDS_SCM_HTTP_KERBEROS_KEYTAB_FILE_KEY, spnegoKeytab.getAbsolutePath()); - conf.set(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY, omKeytab.getAbsolutePath()); - conf.set(OZONE_OM_HTTP_KERBEROS_KEYTAB_FILE, spnegoKeytab.getAbsolutePath()); - } - /** * Exercises a secure SCM and OM sharing a single SCM instance to avoid paying * the SCM startup cost twice. Covers SCM startup and cert trust chain, SCM @@ -367,7 +313,7 @@ private void assertScmExposesClusterInfoAndTrustChain() throws IOException { private void assertScmSecurityProtocolAllowsKerberosUser() throws Exception { UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI( - testUserPrincipal, testUserKeytab.getCanonicalPath()); + getTestUserPrincipal(), getTestUserKeytab().getCanonicalPath()); ugi.setAuthenticationMethod(KERBEROS); try (SCMSecurityProtocolClientSideTranslatorPB securityClient = getScmSecurityClient(conf, ugi)) { @@ -403,7 +349,7 @@ private void assertScmSecurityProtocolRejectsNonKerberosUser() throws Exception /** SCM admin protocol - authenticated non-admin user is denied. */ private void assertScmAdminProtocolDeniesNonAdminUser() throws IOException { UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI( - testUserPrincipal, testUserKeytab.getCanonicalPath()); + getTestUserPrincipal(), getTestUserKeytab().getCanonicalPath()); StorageContainerLocationProtocol scmRpcClient = HAUtils.getScmContainerClient(conf, ugi); IOException adminException = assertThrows(IOException.class, @@ -474,7 +420,8 @@ private void assertSecureOmInitFailsWithNonExistentPrincipal() { */ private void assertSecureOmInitWhenAlreadyInitialized() throws Exception { conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos"); - conf.set(OZONE_OM_KERBEROS_PRINCIPAL_KEY, "om/" + host + "@" + realm); + conf.set(OZONE_OM_KERBEROS_PRINCIPAL_KEY, + getConf().get(OZONE_OM_KERBEROS_PRINCIPAL_KEY)); conf.set(OZONE_OM_ADDRESS_KEY, InetAddress.getLocalHost().getCanonicalHostName() + ":" + getFreePort()); LogCapturer omLogs = LogCapturer.captureLogs(OMCertificateClient.class); @@ -534,9 +481,9 @@ private void initSCM() throws IOException { * */ conf.set(HDDS_SCM_KERBEROS_PRINCIPAL_KEY, - "om/" + host + "@" + miniKdc.getRealm()); - File keyTab = new File(workDir, "om.keytab"); - conf.set(HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, keyTab.getAbsolutePath()); + conf.get(OZONE_OM_KERBEROS_PRINCIPAL_KEY)); + conf.set(HDDS_SCM_KERBEROS_KEYTAB_FILE_KEY, + conf.get(OZONE_OM_KERBEROS_KEYTAB_FILE_KEY)); } @Test @@ -596,7 +543,7 @@ void testAccessControlExceptionOnClient() throws Exception { // positive case (happy-path) UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI( - testUserPrincipal, testUserKeytab.getCanonicalPath()); + getTestUserPrincipal(), getTestUserKeytab().getCanonicalPath()); ugi.setAuthenticationMethod(KERBEROS); OzoneManagerProtocolClientSideTranslatorPB secureClient = new OzoneManagerProtocolClientSideTranslatorPB( @@ -725,7 +672,7 @@ void testGetSetRevokeS3Secret() throws Exception { // testUser is not an admin final UserGroupInformation ugiNonAdmin = UserGroupInformation.loginUserFromKeytabAndReturnUGI( - testUserPrincipal, testUserKeytab.getCanonicalPath()); + getTestUserPrincipal(), getTestUserKeytab().getCanonicalPath()); final OzoneManagerProtocolClientSideTranslatorPB omClientNonAdmin = new OzoneManagerProtocolClientSideTranslatorPB( OmTransportFactory.create(conf, ugiNonAdmin, null),