diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
index 0b012eb9234c..288357dc8c9a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java
@@ -322,12 +322,18 @@ private List<ServerName> filterServers(Set<Address> servers, List<ServerName> on
       } catch (IOException exp) {
         LOG.debug("RSGroup information null for region of table " + tableName, exp);
       }
+      List<ServerName> candidateServers = Collections.emptyList();
+      if (targetRSGInfo != null) {
+        List<ServerName> onlineServers = Lists.newArrayList(clusterLoad.keySet());
+        candidateServers = filterOfflineServers(targetRSGInfo, onlineServers);
+        if (isFallbackEnabled() && candidateServers.isEmpty()) {
+          candidateServers = getFallBackCandidates(onlineServers);
+        }
+      }
       for (Map.Entry<ServerName, List<RegionInfo>> serverRegionMap : clusterLoad.entrySet()) {
         ServerName currentHostServer = serverRegionMap.getKey();
         List<RegionInfo> regionInfoList = serverRegionMap.getValue();
-        if (
-          targetRSGInfo == null || !targetRSGInfo.containsServer(currentHostServer.getAddress())
-        ) {
+        if (!candidateServers.contains(currentHostServer)) {
           regionInfoList.forEach(regionInfo -> {
             regionPlansForMisplacedRegions.add(new RegionPlan(regionInfo, currentHostServer, null));
           });
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsFallback.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsFallback.java
index ab1ed04fcf79..20a15382b02e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsFallback.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsFallback.java
@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hbase.rsgroup;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.io.IOException;
@@ -25,6 +26,7 @@
 import org.apache.hadoop.hbase.HBaseTestingUtil;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.BalanceResponse;
 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
 import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
@@ -93,10 +95,20 @@ public void testFallback() throws Exception {
     // server of test group crash, regions move to default group
     crashRsInGroup(groupName);
     assertRegionsInGroup(tableName, RSGroupInfo.DEFAULT_GROUP);
+    assertTrue(MASTER.balance().isBalancerRan());
+    assertRegionsInGroup(tableName, RSGroupInfo.DEFAULT_GROUP);
+    BalanceResponse response = MASTER.balance();
+    assertTrue(response.isBalancerRan());
+    assertEquals(0, response.getMovesCalculated());
 
     // server of default group crash, regions move to any other group
     crashRsInGroup(RSGroupInfo.DEFAULT_GROUP);
     assertRegionsInGroup(tableName, FALLBACK_GROUP);
+    assertTrue(MASTER.balance().isBalancerRan());
+    assertRegionsInGroup(tableName, FALLBACK_GROUP);
+    response = MASTER.balance();
+    assertTrue(response.isBalancerRan());
+    assertEquals(0, response.getMovesCalculated());
 
     // add a new server to default group, regions move to default group
     TEST_UTIL.getMiniHBaseCluster().startRegionServerAndWait(60000);