PHOENIX-7829 TaskRegionObserver.SelfHealingTask runs unexpectedly during tests

phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java

@@ -4923,9 +4923,9 @@ public MutationState addColumn(PTable table, List<ColumnDef> origColumnDefs,
           /**
            * To check if TTL is defined at any of the child below we are checking it at
            * {@link org.apache.phoenix.coprocessor.MetaDataEndpointImpl#mutateColumn(List, ColumnMutator, int, PTable, PTable, boolean)}
-           * level where in function
-           * {@link org.apache.phoenix.coprocessor.MetaDataEndpointImpl# validateIfMutationAllowedOnParent(PTable, List, PTableType, long, byte[], byte[], byte[], List, int)}
-           * we are already traversing through allDescendantViews.
+           * level where in function {@link org.apache.phoenix.coprocessor.MetaDataEndpointImpl#
+           * validateIfMutationAllowedOnParent(PTable, List, PTableType, long, byte[], byte[],
+           * byte[], List, int)} we are already traversing through allDescendantViews.
            */
         }
 

phoenix-core/src/it/java/org/apache/phoenix/cache/ServerMetadataCacheIT.java

@@ -101,8 +101,6 @@ public static synchronized void doSetup() throws Exception {
     props.put(QueryServices.DEFAULT_UPDATE_CACHE_FREQUENCY_ATRRIB, "NEVER");
     props.put(QueryServices.LAST_DDL_TIMESTAMP_VALIDATION_ENABLED, Boolean.toString(true));
     props.put(PHOENIX_METADATA_INVALIDATE_CACHE_ENABLED, Boolean.toString(true));
-    props.put(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
-    props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
     setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
     assertEquals(1, getUtility().getHBaseCluster().getNumLiveRegionServers());
     serverName = getUtility().getHBaseCluster().getRegionServer(0).getServerName();

phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java

@@ -115,8 +115,6 @@ public static synchronized Collection<MavenCoordinates> data() throws Exception
   public synchronized void doSetup() throws Exception {
     tmpDir = System.getProperty("java.io.tmpdir");
     conf = HBaseConfiguration.create();
-    conf.set(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
-    conf.set(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
     hbaseTestUtil = new HBaseTestingUtility(conf);
     setUpConfigForMiniCluster(conf);
     conf.set(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB,

phoenix-core/src/it/java/org/apache/phoenix/end2end/CDCStreamIT.java

@@ -86,8 +86,6 @@ public static synchronized void doSetup() throws Exception {
     Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
     props.put(PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, Integer.toString(60 * 60)); // An hour
     props.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(false));
-    props.put(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
-    props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
     props.put("hbase.coprocessor.master.classes", PhoenixMasterObserver.class.getName());
     setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
     taskRegionEnvironment =

phoenix-core/src/it/java/org/apache/phoenix/end2end/MetadataGetTableReadLockIT.java

@@ -27,7 +27,6 @@
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.phoenix.coprocessor.MetaDataEndpointImpl;
 import org.apache.phoenix.query.BaseTest;
-import org.apache.phoenix.query.QueryServices;
 import org.apache.phoenix.util.ReadOnlyProps;
 import org.apache.phoenix.util.TestUtil;
 import org.junit.AfterClass;
@@ -48,8 +47,6 @@ public class MetadataGetTableReadLockIT extends BaseTest {
   @BeforeClass
   public static synchronized void doSetup() throws Exception {
     Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
-    // Disable system task handling
-    props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
     setUpTestDriver(new ReadOnlyProps(props));
   }
 

phoenix-core/src/it/java/org/apache/phoenix/end2end/MetadataServerConnectionsIT.java

@@ -52,7 +52,6 @@
 import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
 import org.apache.phoenix.protobuf.ProtobufUtil;
 import org.apache.phoenix.query.BaseTest;
-import org.apache.phoenix.query.QueryServices;
 import org.apache.phoenix.util.ClientUtil;
 import org.apache.phoenix.util.MetaDataUtil;
 import org.apache.phoenix.util.ReadOnlyProps;
@@ -80,7 +79,6 @@ public class MetadataServerConnectionsIT extends BaseTest {
   @BeforeClass
   public static synchronized void doSetup() throws Exception {
     Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
-    props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
     props.put(DISABLE_VIEW_SUBTREE_VALIDATION, "true");
     props.put(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, Boolean.toString(false));
     setUpTestDriver(new ReadOnlyProps(props));

phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsEnabledIT.java

@@ -45,7 +45,6 @@ public static synchronized final void doSetup() throws Exception {
     props.put(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Long.toString(5));
     props.put(QueryServices.MAX_SERVER_METADATA_CACHE_TIME_TO_LIVE_MS_ATTRIB, Long.toString(5));
     props.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(true));
-    props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
     setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
 
     TaskRegionEnvironment =

phoenix-core/src/it/java/org/apache/phoenix/end2end/UCFWithDisabledIndexIT.java

@@ -74,7 +74,6 @@ private static void initCluster() throws Exception {
     props.put(QueryServices.DEFAULT_UPDATE_CACHE_FREQUENCY_ATRRIB, "ALWAYS");
     props.put(QueryServices.LAST_DDL_TIMESTAMP_VALIDATION_ENABLED, Boolean.toString(false));
     props.put(QueryServices.PHOENIX_METADATA_INVALIDATE_CACHE_ENABLED, Boolean.toString(false));
-    props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
     props.put(DISABLE_VIEW_SUBTREE_VALIDATION, "true");
     props.put(INDEX_USE_SERVER_METADATA_ATTRIB, Boolean.toString(false));
     setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));

phoenix-core/src/it/java/org/apache/phoenix/end2end/UCFWithDisabledIndexWithDDLValidationIT.java

@@ -44,7 +44,6 @@ private static void initCluster() throws Exception {
     props.put(QueryServices.DEFAULT_UPDATE_CACHE_FREQUENCY_ATRRIB, "NEVER");
     props.put(QueryServices.LAST_DDL_TIMESTAMP_VALIDATION_ENABLED, Boolean.toString(true));
     props.put(QueryServices.PHOENIX_METADATA_INVALIDATE_CACHE_ENABLED, Boolean.toString(true));
-    props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
     props.put(INDEX_USE_SERVER_METADATA_ATTRIB, Boolean.toString(false));
     setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
   }

phoenix-core/src/it/java/org/apache/phoenix/end2end/UCFWithServerMetadataIT.java

@@ -100,7 +100,6 @@ public void addServerCache(RpcController controller, AddServerCacheRequest reque
   @BeforeClass
   public static synchronized void doSetup() throws Exception {
     Map<String, String> props = new HashMap<>(1);
-    props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
     props.put(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, Boolean.toString(false));
     setUpTestDriver(new ReadOnlyProps(props));
   }

phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectIT.java

@@ -85,10 +85,6 @@ public static synchronized void doSetup() throws Exception {
     // An hour - inherited from ParallelStatsDisabledIT
     props.put(BaseScannerRegionObserverConstants.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY,
       Integer.toString(60 * 60));
-    // Postpone scans of SYSTEM.TASK indefinitely so as to prevent
-    // any addition to GLOBAL_OPEN_PHOENIX_CONNECTIONS
-    props.put(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
-    props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
     props.put(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, Boolean.toString(false));
     setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
   }

phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectWithRegionMovesIT.java

@@ -86,10 +86,6 @@ public static synchronized void doSetup() throws Exception {
     // An hour - inherited from ParallelStatsDisabledIT
     props.put(BaseScannerRegionObserverConstants.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY,
       Integer.toString(60 * 60));
-    // Postpone scans of SYSTEM.TASK indefinitely so as to prevent
-    // any addition to GLOBAL_OPEN_PHOENIX_CONNECTIONS
-    props.put(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
-    props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
     props.put(QueryServices.PHOENIX_SERVER_PAGE_SIZE_MS, Long.toString(0));
     props.put(QueryServices.TESTS_MINI_CLUSTER_NUM_REGION_SERVERS, String.valueOf(2));
     props.put(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, String.valueOf(1));

phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java

@@ -767,8 +767,16 @@ public void testAsyncRebuildAll() throws Exception {
       assertEquals(testTable, rs.getString(TABLE_NAME));
       assertFalse(rs.next());
 
+      // The SelfHealingTask scheduler is quiesced by default in mini-cluster ITs, so drive it
+      // manually to pick up the CREATED INDEX_REBUILD task and launch the rebuild job.
+      TestUtil.runSelfHealingTaskOnce();
+
       TestUtil.waitForIndexState(conn, indexName, PIndexState.ACTIVE);
 
+      // Once the index is ACTIVE the rebuild job has completed, drive the task again so it observes
+      // the finished job and transitions from STARTED to COMPLETED.
+      TestUtil.runSelfHealingTaskOnce();
+
       // Check task status
       String query = "SELECT * " + " FROM " + PhoenixDatabaseMetaData.SYSTEM_TASK_NAME + " WHERE "
         + PhoenixDatabaseMetaData.TASK_TYPE + " = "

phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java

@@ -197,8 +197,6 @@ public Configuration getConfiguration(Configuration confToClone) {
       }
     });
     Configuration conf = HBaseFactoryProvider.getConfigurationFactory().getConfiguration();
-    conf.set(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
-    conf.set(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
     hbaseTestUtil = new HBaseTestingUtility(conf);
     setUpConfigForMiniCluster(conf);
     conf.set(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB,
@@ -214,8 +212,6 @@ public Configuration getConfiguration(Configuration confToClone) {
           Integer.toString(0));
         put(QueryServices.PHOENIX_VIEW_TTL_ENABLED, Boolean.toString(true));
         put(QueryServices.PHOENIX_VIEW_TTL_TENANT_VIEWS_PER_SCAN_LIMIT, String.valueOf(1));
-        put(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
-        put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
       }
     };
 

phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java

@@ -59,8 +59,6 @@ public abstract class BasePhoenixMetricsIT extends BaseTest {
   @BeforeClass
   public static synchronized void doSetup() throws Exception {
     Map<String, String> props = Maps.newHashMapWithExpectedSize(3);
-    // Disable system task handling
-    props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
     // Phoenix Global client metrics are enabled by default
     // Enable request metric collection at the driver level
     props.put(QueryServices.COLLECT_REQUEST_LEVEL_METRICS, String.valueOf(true));

phoenix-core/src/it/java/org/apache/phoenix/monitoring/IndexMetricsIT.java

@@ -47,7 +47,6 @@ public class IndexMetricsIT extends ParallelStatsDisabledIT {
   @BeforeClass
   public static synchronized void doSetup() throws Exception {
     Map<String, String> props = Maps.newHashMapWithExpectedSize(3);
-    props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
     // disable renewing leases as this will force spooling to happen.
     props.put(QueryServices.RENEW_LEASE_ENABLED, String.valueOf(false));
     setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));

phoenix-core/src/it/java/org/apache/phoenix/monitoring/MetadataMetricsIT.java

@@ -37,7 +37,6 @@ public class MetadataMetricsIT extends ParallelStatsDisabledIT {
   @BeforeClass
   public static void setup() throws Exception {
     Map<String, String> props = Maps.newHashMapWithExpectedSize(3);
-    props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
     // disable renewing leases as this will force spooling to happen.
     props.put(QueryServices.RENEW_LEASE_ENABLED, String.valueOf(false));
     setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));

phoenix-core/src/it/java/org/apache/phoenix/monitoring/connectionqueryservice/ConnectionQueryServicesMetricsIT.java

@@ -118,8 +118,6 @@ public Configuration getConfiguration(Configuration confToClone) {
       }
     });
     Configuration conf = HBaseFactoryProvider.getConfigurationFactory().getConfiguration();
-    conf.set(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
-    conf.set(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
     conf.set(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, Boolean.toString(false));
     hbaseTestUtil = new HBaseTestingUtility(conf);
     setUpConfigForMiniCluster(conf);

phoenix-core/src/it/java/org/apache/phoenix/query/MetaDataCachingIT.java

@@ -59,15 +59,12 @@ public class MetaDataCachingIT extends BaseTest {
   @BeforeClass
   public static synchronized void doSetup() throws Exception {
     Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
-    // We set here a tiny cache to verify that even if the total size of the cache is just enough to
-    // hold
-    // system tables and Phoenix is still functional. Please note the cache weight for system tables
-    // is set to
-    // zero to allow insertion of system tables even when the cache reaches its maximum weight.
+    // We set here a tiny cache to verify that even if the total size of the cache is just enough
+    // to hold system tables and Phoenix is still functional. Please note the cache weight for
+    // system tables is set to zero to allow insertion of system tables even when the cache reaches
+    // its maximum weight.
     props.put(QueryServices.MAX_CLIENT_METADATA_CACHE_SIZE_ATTRIB, "50000");
     props.put(QueryServices.CLIENT_CACHE_ENCODING, "object");
-    props.put(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
-    props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
     props.put(PHOENIX_INDEX_CDC_CONSUMER_ENABLED, Boolean.toString(false));
     setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
   }

phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java

@@ -579,6 +579,10 @@ public static Configuration setUpConfigForMiniCluster(Configuration conf,
     conf.set(IndexManagementUtil.WAL_EDIT_CODEC_CLASS_KEY,
       "org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec");
 
+    if (conf.get(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB) == null) {
+      conf.setLong(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.MAX_VALUE);
+    }
+
     // This results in processing one row at a time in each next operation of the aggregate region
     // scanner, i.e., one row pages. In other words, 0ms page allows only one row to be processed
     // within one page; 0ms page is equivalent to one-row page

phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java

@@ -87,6 +87,7 @@
 import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
 import org.apache.hadoop.hbase.client.coprocessor.Batch;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
 import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
 import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils.BlockingRpcCallback;
@@ -104,6 +105,7 @@
 import org.apache.phoenix.compile.SubqueryRewriter;
 import org.apache.phoenix.compile.SubselectRewriter;
 import org.apache.phoenix.coprocessor.CompactionScanner;
+import org.apache.phoenix.coprocessor.TaskRegionObserver;
 import org.apache.phoenix.coprocessor.generated.MetaDataProtos.ClearCacheRequest;
 import org.apache.phoenix.coprocessor.generated.MetaDataProtos.ClearCacheResponse;
 import org.apache.phoenix.coprocessor.generated.MetaDataProtos.MetaDataService;
@@ -139,6 +141,7 @@
 import org.apache.phoenix.parse.LikeParseNode.LikeType;
 import org.apache.phoenix.parse.SQLParser;
 import org.apache.phoenix.parse.SelectStatement;
+import org.apache.phoenix.query.BaseTest;
 import org.apache.phoenix.query.ConnectionQueryServices;
 import org.apache.phoenix.query.KeyRange;
 import org.apache.phoenix.query.QueryConstants;
@@ -1152,6 +1155,20 @@ public static void waitForIndexRebuild(Connection conn, String fullIndexName,
     waitForIndexState(conn, fullIndexName, indexState, 0L);
   }
 
+  /**
+   * Runs a single synchronous tick of {@link TaskRegionObserver.SelfHealingTask} against the
+   * SYSTEM.TASK region on the running mini-cluster.
+   */
+  public static void runSelfHealingTaskOnce() throws Exception {
+    RegionCoprocessorEnvironment taskRegionEnvironment = BaseTest.getUtility()
+      .getRSForFirstRegionInTable(PhoenixDatabaseMetaData.SYSTEM_TASK_HBASE_TABLE_NAME)
+      .getRegions(PhoenixDatabaseMetaData.SYSTEM_TASK_HBASE_TABLE_NAME).get(0).getCoprocessorHost()
+      .findCoprocessorEnvironment(TaskRegionObserver.class.getName());
+    TaskRegionObserver.SelfHealingTask task = new TaskRegionObserver.SelfHealingTask(
+      taskRegionEnvironment, QueryServicesOptions.DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS);
+    task.run();
+  }
+
   private static class IndexStateCheck {
     public final PIndexState indexState;
     public final Long indexDisableTimestamp;