PHOENIX-7891 Explain the query optimizer's index selection rationale (#2513)

Co-authored-by: Claude Opus 4.8[1m] <noreply@anthropic.com>

Author: apurtell

phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainPlanAttributes.java

@@ -25,6 +25,7 @@
 import java.util.Set;
 import org.apache.hadoop.hbase.HRegionLocation;
 import org.apache.hadoop.hbase.client.Consistency;
+import org.apache.phoenix.optimize.RejectedIndexEntry;
 import org.apache.phoenix.parse.HintNode;
 import org.apache.phoenix.parse.HintNode.Hint;
 import org.apache.phoenix.schema.PColumn;
@@ -35,16 +36,16 @@
  * Strings containing entire plan.
  */
 @JsonPropertyOrder({ "abstractExplainPlan", "hint", "explainScanType", "consistency", "tableName",
-  "keyRanges", "indexName", "indexKind", "saltBuckets", "regionsPlanned", "scanTimeRangeMin",
-  "scanTimeRangeMax", "splitsChunk", "useRoundRobinIterator", "samplingRate", "hexStringRVCOffset",
-  "iteratorTypeAndScanSize", "estimatedRows", "estimatedSizeInBytes", "serverWhereFilter",
-  "serverDistinctFilter", "serverMergeColumns", "serverArrayElementProjection",
-  "serverFirstKeyOnlyProjection", "serverEmptyColumnOnlyProjection", "serverAggregate",
-  "serverGroupByLimit", "serverSortedBy", "serverOffset", "serverRowLimit", "clientFilterBy",
-  "clientAggregate", "clientDistinctFilter", "clientAfterAggregate", "clientSortAlgo",
-  "clientSortedBy", "clientOffset", "clientRowLimit", "clientSequenceCount", "clientCursorName",
-  "clientSteps", "lhsJoinQueryExplainPlan", "rhsJoinQueryExplainPlan", "subPlans",
-  "dynamicServerFilter", "afterJoinFilter", "joinScannerLimit", "sortMergeSkipMerge",
+  "keyRanges", "indexName", "indexKind", "indexRule", "indexRejected", "saltBuckets",
+  "regionsPlanned", "scanTimeRangeMin", "scanTimeRangeMax", "splitsChunk", "useRoundRobinIterator",
+  "samplingRate", "hexStringRVCOffset", "iteratorTypeAndScanSize", "estimatedRows",
+  "estimatedSizeInBytes", "serverWhereFilter", "serverDistinctFilter", "serverMergeColumns",
+  "serverArrayElementProjection", "serverFirstKeyOnlyProjection", "serverEmptyColumnOnlyProjection",
+  "serverAggregate", "serverGroupByLimit", "serverSortedBy", "serverOffset", "serverRowLimit",
+  "clientFilterBy", "clientAggregate", "clientDistinctFilter", "clientAfterAggregate",
+  "clientSortAlgo", "clientSortedBy", "clientOffset", "clientRowLimit", "clientSequenceCount",
+  "clientCursorName", "clientSteps", "lhsJoinQueryExplainPlan", "rhsJoinQueryExplainPlan",
+  "subPlans", "dynamicServerFilter", "afterJoinFilter", "joinScannerLimit", "sortMergeSkipMerge",
   "regionLocations", "regionLocationsTotalSize", "numRegionLocationLookups" })
 public class ExplainPlanAttributes {
 
@@ -57,6 +58,8 @@ public class ExplainPlanAttributes {
   private final String keyRanges;
   private final String indexName;
   private final String indexKind;
+  private final String indexRule;
+  private final List<RejectedIndexEntry> indexRejected;
   private final Integer saltBuckets;
   private final Integer regionsPlanned;
   private final Long scanTimeRangeMin;
@@ -122,6 +125,8 @@ private ExplainPlanAttributes() {
     this.keyRanges = null;
     this.indexName = null;
     this.indexKind = null;
+    this.indexRule = null;
+    this.indexRejected = null;
     this.saltBuckets = null;
     this.regionsPlanned = null;
     this.scanTimeRangeMin = null;
@@ -169,20 +174,20 @@ private ExplainPlanAttributes() {
 
   public ExplainPlanAttributes(String abstractExplainPlan, Hint hint, String explainScanType,
     Consistency consistency, String tableName, String keyRanges, String indexName, String indexKind,
-    Integer saltBuckets, Integer regionsPlanned, Long scanTimeRangeMin, Long scanTimeRangeMax,
-    Integer splitsChunk, boolean useRoundRobinIterator, Double samplingRate,
-    String hexStringRVCOffset, String iteratorTypeAndScanSize, Long estimatedRows,
-    Long estimatedSizeInBytes, String serverWhereFilter, String serverDistinctFilter,
-    Set<PColumn> serverMergeColumns, boolean serverArrayElementProjection,
-    boolean serverFirstKeyOnlyProjection, boolean serverEmptyColumnOnlyProjection,
-    String serverAggregate, Integer serverGroupByLimit, String serverSortedBy, Integer serverOffset,
-    Long serverRowLimit, String clientFilterBy, String clientAggregate, String clientDistinctFilter,
-    String clientAfterAggregate, String clientSortAlgo, String clientSortedBy, Integer clientOffset,
-    Integer clientRowLimit, Integer clientSequenceCount, String clientCursorName,
-    List<String> clientSteps, ExplainPlanAttributes lhsJoinQueryExplainPlan,
-    ExplainPlanAttributes rhsJoinQueryExplainPlan, List<ExplainPlanAttributes> subPlans,
-    String dynamicServerFilter, String afterJoinFilter, Long joinScannerLimit,
-    boolean sortMergeSkipMerge, List<HRegionLocation> regionLocations,
+    String indexRule, List<RejectedIndexEntry> indexRejected, Integer saltBuckets,
+    Integer regionsPlanned, Long scanTimeRangeMin, Long scanTimeRangeMax, Integer splitsChunk,
+    boolean useRoundRobinIterator, Double samplingRate, String hexStringRVCOffset,
+    String iteratorTypeAndScanSize, Long estimatedRows, Long estimatedSizeInBytes,
+    String serverWhereFilter, String serverDistinctFilter, Set<PColumn> serverMergeColumns,
+    boolean serverArrayElementProjection, boolean serverFirstKeyOnlyProjection,
+    boolean serverEmptyColumnOnlyProjection, String serverAggregate, Integer serverGroupByLimit,
+    String serverSortedBy, Integer serverOffset, Long serverRowLimit, String clientFilterBy,
+    String clientAggregate, String clientDistinctFilter, String clientAfterAggregate,
+    String clientSortAlgo, String clientSortedBy, Integer clientOffset, Integer clientRowLimit,
+    Integer clientSequenceCount, String clientCursorName, List<String> clientSteps,
+    ExplainPlanAttributes lhsJoinQueryExplainPlan, ExplainPlanAttributes rhsJoinQueryExplainPlan,
+    List<ExplainPlanAttributes> subPlans, String dynamicServerFilter, String afterJoinFilter,
+    Long joinScannerLimit, boolean sortMergeSkipMerge, List<HRegionLocation> regionLocations,
     Integer regionLocationsTotalSize, int numRegionLocationLookups) {
     this.abstractExplainPlan = abstractExplainPlan;
     this.hint = hint;
@@ -192,6 +197,10 @@ public ExplainPlanAttributes(String abstractExplainPlan, Hint hint, String expla
     this.keyRanges = keyRanges;
     this.indexName = indexName;
     this.indexKind = indexKind;
+    this.indexRule = indexRule;
+    this.indexRejected = (indexRejected == null || indexRejected.isEmpty())
+      ? null
+      : Collections.unmodifiableList(new ArrayList<>(indexRejected));
     this.saltBuckets = saltBuckets;
     this.regionsPlanned = regionsPlanned;
     this.scanTimeRangeMin = scanTimeRangeMin;
@@ -271,6 +280,14 @@ public String getIndexKind() {
     return indexKind;
   }
 
+  public String getIndexRule() {
+    return indexRule;
+  }
+
+  public List<RejectedIndexEntry> getIndexRejected() {
+    return indexRejected;
+  }
+
   public Integer getSaltBuckets() {
     return saltBuckets;
   }
@@ -458,6 +475,8 @@ public static class ExplainPlanAttributesBuilder {
     private String keyRanges;
     private String indexName;
     private String indexKind;
+    private String indexRule;
+    private List<RejectedIndexEntry> indexRejected;
     private Integer saltBuckets;
     private Integer regionsPlanned;
     private Long scanTimeRangeMin;
@@ -515,6 +534,9 @@ public ExplainPlanAttributesBuilder(ExplainPlanAttributes explainPlanAttributes)
       this.keyRanges = explainPlanAttributes.getKeyRanges();
       this.indexName = explainPlanAttributes.getIndexName();
       this.indexKind = explainPlanAttributes.getIndexKind();
+      this.indexRule = explainPlanAttributes.getIndexRule();
+      List<RejectedIndexEntry> srcIndexRejected = explainPlanAttributes.getIndexRejected();
+      this.indexRejected = srcIndexRejected == null ? null : new ArrayList<>(srcIndexRejected);
       this.saltBuckets = explainPlanAttributes.getSaltBuckets();
       this.regionsPlanned = explainPlanAttributes.getRegionsPlanned();
       this.scanTimeRangeMin = explainPlanAttributes.getScanTimeRangeMin();
@@ -602,6 +624,16 @@ public ExplainPlanAttributesBuilder setIndexKind(String indexKind) {
       return this;
     }
 
+    public ExplainPlanAttributesBuilder setIndexRule(String indexRule) {
+      this.indexRule = indexRule;
+      return this;
+    }
+
+    public ExplainPlanAttributesBuilder setIndexRejected(List<RejectedIndexEntry> indexRejected) {
+      this.indexRejected = indexRejected == null ? null : new ArrayList<>(indexRejected);
+      return this;
+    }
+
     public ExplainPlanAttributesBuilder setSaltBuckets(Integer saltBuckets) {
       this.saltBuckets = saltBuckets;
       return this;
@@ -833,16 +865,16 @@ public ExplainPlanAttributesBuilder setNumRegionLocationLookups(int numRegionLoc
 
     public ExplainPlanAttributes build() {
       return new ExplainPlanAttributes(abstractExplainPlan, hint, explainScanType, consistency,
-        tableName, keyRanges, indexName, indexKind, saltBuckets, regionsPlanned, scanTimeRangeMin,
-        scanTimeRangeMax, splitsChunk, useRoundRobinIterator, samplingRate, hexStringRVCOffset,
-        iteratorTypeAndScanSize, estimatedRows, estimatedSizeInBytes, serverWhereFilter,
-        serverDistinctFilter, serverMergeColumns, serverArrayElementProjection,
-        serverFirstKeyOnlyProjection, serverEmptyColumnOnlyProjection, serverAggregate,
-        serverGroupByLimit, serverSortedBy, serverOffset, serverRowLimit, clientFilterBy,
-        clientAggregate, clientDistinctFilter, clientAfterAggregate, clientSortAlgo, clientSortedBy,
-        clientOffset, clientRowLimit, clientSequenceCount, clientCursorName, clientSteps,
-        lhsJoinQueryExplainPlan, rhsJoinQueryExplainPlan, subPlans, dynamicServerFilter,
-        afterJoinFilter, joinScannerLimit, sortMergeSkipMerge, regionLocations,
+        tableName, keyRanges, indexName, indexKind, indexRule, indexRejected, saltBuckets,
+        regionsPlanned, scanTimeRangeMin, scanTimeRangeMax, splitsChunk, useRoundRobinIterator,
+        samplingRate, hexStringRVCOffset, iteratorTypeAndScanSize, estimatedRows,
+        estimatedSizeInBytes, serverWhereFilter, serverDistinctFilter, serverMergeColumns,
+        serverArrayElementProjection, serverFirstKeyOnlyProjection, serverEmptyColumnOnlyProjection,
+        serverAggregate, serverGroupByLimit, serverSortedBy, serverOffset, serverRowLimit,
+        clientFilterBy, clientAggregate, clientDistinctFilter, clientAfterAggregate, clientSortAlgo,
+        clientSortedBy, clientOffset, clientRowLimit, clientSequenceCount, clientCursorName,
+        clientSteps, lhsJoinQueryExplainPlan, rhsJoinQueryExplainPlan, subPlans,
+        dynamicServerFilter, afterJoinFilter, joinScannerLimit, sortMergeSkipMerge, regionLocations,
         regionLocationsTotalSize, numRegionLocationLookups);
     }
   }

phoenix-core-client/src/main/java/org/apache/phoenix/compile/QueryPlan.java

@@ -28,6 +28,7 @@
 import org.apache.phoenix.iterate.ParallelScanGrouper;
 import org.apache.phoenix.iterate.ResultIterator;
 import org.apache.phoenix.optimize.Cost;
+import org.apache.phoenix.optimize.OptimizerDecision;
 import org.apache.phoenix.parse.FilterableStatement;
 import org.apache.phoenix.parse.SelectStatement;
 import org.apache.phoenix.query.KeyRange;
@@ -103,4 +104,17 @@ public interface QueryPlan extends StatementPlan {
    * </pre>
    */
   public List<OrderBy> getOutputOrderBys();
+
+  /**
+   * The optimizer's index selection rationale for this plan, or {@code null} if this plan did not
+   * participate in optimizer index selection.
+   */
+  default OptimizerDecision getOptimizerDecision() {
+    return null;
+  }
+
+  /** Records the optimizer's index selection rationale on this plan. */
+  default void setOptimizerDecision(OptimizerDecision decision) {
+    // no-op
+  }
 }

phoenix-core-client/src/main/java/org/apache/phoenix/execute/BaseQueryPlan.java

@@ -57,6 +57,7 @@
 import org.apache.phoenix.iterate.ResultIterator;
 import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.jdbc.PhoenixStatement.Operation;
+import org.apache.phoenix.optimize.OptimizerDecision;
 import org.apache.phoenix.parse.FilterableStatement;
 import org.apache.phoenix.parse.HintNode.Hint;
 import org.apache.phoenix.parse.ParseNodeFactory;
@@ -110,6 +111,7 @@ public abstract class BaseQueryPlan implements QueryPlan {
   protected Long estimateInfoTimestamp;
   private boolean getEstimatesCalled;
   protected boolean isApplicable = true;
+  private OptimizerDecision optimizerDecision;
 
   protected BaseQueryPlan(StatementContext context, FilterableStatement statement, TableRef table,
     RowProjector projection, ParameterMetaData paramMetaData, Integer limit, Integer offset,
@@ -543,6 +545,11 @@ private Pair<List<String>, ExplainPlanAttributes> getPlanStepsV2(ResultIterator
     List<String> planSteps = Lists.newArrayListWithExpectedSize(5);
     ExplainPlanAttributesBuilder builder = new ExplainPlanAttributesBuilder();
     iterator.explain(planSteps, builder);
+    OptimizerDecision decision = getOptimizerDecision();
+    if (decision != null) {
+      builder.setIndexRule(decision.getRule());
+      builder.setIndexRejected(decision.getRejectedIndexes());
+    }
     return Pair.of(planSteps, builder.build());
   }
 
@@ -585,6 +592,16 @@ public void setApplicable(boolean isApplicable) {
     this.isApplicable = isApplicable;
   }
 
+  @Override
+  public OptimizerDecision getOptimizerDecision() {
+    return optimizerDecision;
+  }
+
+  @Override
+  public void setOptimizerDecision(OptimizerDecision decision) {
+    this.optimizerDecision = decision;
+  }
+
   private void getEstimates() throws SQLException {
     getEstimatesCalled = true;
     // Initialize a dummy iterator to get the estimates based on stats.

phoenix-core-client/src/main/java/org/apache/phoenix/execute/DelegateQueryPlan.java

@@ -32,6 +32,7 @@
 import org.apache.phoenix.iterate.ResultIterator;
 import org.apache.phoenix.jdbc.PhoenixStatement.Operation;
 import org.apache.phoenix.optimize.Cost;
+import org.apache.phoenix.optimize.OptimizerDecision;
 import org.apache.phoenix.parse.FilterableStatement;
 import org.apache.phoenix.query.KeyRange;
 import org.apache.phoenix.schema.TableRef;
@@ -171,4 +172,14 @@ public List<OrderBy> getOutputOrderBys() {
   public boolean isApplicable() {
     return delegate.isApplicable();
   }
+
+  @Override
+  public OptimizerDecision getOptimizerDecision() {
+    return delegate.getOptimizerDecision();
+  }
+
+  @Override
+  public void setOptimizerDecision(OptimizerDecision decision) {
+    delegate.setOptimizerDecision(decision);
+  }
 }

phoenix-core-client/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java

@@ -86,6 +86,7 @@
 import org.apache.phoenix.hbase.index.util.VersionUtil;
 import org.apache.phoenix.join.HashCacheClient;
 import org.apache.phoenix.monitoring.OverAllQueryMetrics;
+import org.apache.phoenix.optimize.OptimizerDecision;
 import org.apache.phoenix.parse.FilterableStatement;
 import org.apache.phoenix.parse.HintNode;
 import org.apache.phoenix.parse.HintNode.Hint;
@@ -645,6 +646,11 @@ protected int getSplitCount() {
     return splits == null ? 0 : splits.size();
   }
 
+  @Override
+  protected OptimizerDecision getOptimizerDecision() {
+    return plan.getOptimizerDecision();
+  }
+
   @Override
   public List<List<Scan>> getScans() {
     if (scans == null) return Collections.emptyList();

phoenix-core-client/src/main/java/org/apache/phoenix/iterate/ExplainTable.java

@@ -43,6 +43,9 @@
 import org.apache.phoenix.filter.BooleanExpressionFilter;
 import org.apache.phoenix.filter.DistinctPrefixFilter;
 import org.apache.phoenix.filter.EmptyColumnOnlyFilter;
+import org.apache.phoenix.optimize.OptimizerDecision;
+import org.apache.phoenix.optimize.OptimizerReasons;
+import org.apache.phoenix.optimize.RejectedIndexEntry;
 import org.apache.phoenix.parse.HintNode;
 import org.apache.phoenix.parse.HintNode.Hint;
 import org.apache.phoenix.query.KeyRange;
@@ -131,6 +134,27 @@ protected int getSplitCount() {
     return 0;
   }
 
+  /**
+   * The optimizer's index selection rationale for the plan this scan belongs to, used to render the
+   * per-scan {@code INDEX} rule comment and the {@code !INDEX} rejection comments. Returns
+   * {@code null} when the plan did not participate in optimizer index selection (DML, DDL, and
+   * non-optimizer plans).
+   * @return the decision, or {@code null} when unavailable
+   */
+  protected OptimizerDecision getOptimizerDecision() {
+    return null;
+  }
+
+  /**
+   * Whether {@code rule} is a default rule whose {@code INDEX} comment is suppressed. The default
+   * rules are {@link OptimizerReasons#RULE_DATA_TABLE} (no candidate indexes considered) and
+   * {@link OptimizerReasons#RULE_ONLY_CANDIDATE} (a single viable candidate).
+   */
+  private static boolean isDefaultRule(String rule) {
+    return OptimizerReasons.RULE_DATA_TABLE.equals(rule)
+      || OptimizerReasons.RULE_ONLY_CANDIDATE.equals(rule);
+  }
+
   /**
    * Logical name used to render a table or index in EXPLAIN output. Shared by both the scan
    * {@code OVER} line's local index decoration and the per scan {@code INDEX} line.
@@ -216,7 +240,21 @@ protected void explain(String prefix, List<String> planSteps,
           indexKind = null;
       }
     }
-    planSteps.add("    INDEX " + explainIndexName + (indexKind == null ? "" : " " + indexKind));
+    OptimizerDecision decision = getOptimizerDecision();
+    StringBuilder indexLine = new StringBuilder("    INDEX ").append(explainIndexName);
+    if (indexKind != null) {
+      indexLine.append(" ").append(indexKind);
+    }
+    if (decision != null && !isDefaultRule(decision.getRule())) {
+      indexLine.append("  /* ").append(decision.getRule()).append(" */");
+    }
+    planSteps.add(indexLine.toString());
+    if (decision != null) {
+      for (RejectedIndexEntry rejected : decision.getRejectedIndexes()) {
+        planSteps
+          .add("    /* !INDEX " + rejected.getName() + " -- " + rejected.getReason() + " */");
+      }
+    }
     Integer bucketNum = tableRef.getTable().getBucketNum();
     if (bucketNum != null) {
       planSteps.add("    SALT BUCKETS " + bucketNum);