[ASTERIXDB-3688][COMP] Improving Index Selection for Index-Only Query Plans

- user model changes: no
- storage format changes: no
- interface changes: no

Details: This change ensures that a secondary index capable of supporting an index-only query plan is selected when available.


Ext-ref: MB-69514


Change-Id: I4f996ee7b6eb03067fb384161fb4eb60f94e8353
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20774
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>

Author: Shahrzadshirazii

asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java

@@ -287,8 +287,15 @@ protected void chooseAllIndexes(Map<IAccessMethod, AccessMethodAnalysisContext>
                         // 2) functions that take keyword as an argument, e.g. edit_distance_check() when the threshold is 1
                         || (chosenAccessMethod == InvertedIndexAccessMethod.INSTANCE && isKeywordIndexChosen
                                 && isSameFullTextConfigInIndexAndQuery(analysisCtx, chosenIndex.getIndexDetails()))) {
-
-                    if (resultVarsToIndexTypesMap.containsKey(indexEntry.getValue())) {
+                    // For Btrees and Rtrees we need to keep all the indexes to be able to find the covering ones.
+                    // therefore we add them without needing to check their index type.
+                    if ((chosenAccessMethod == RTreeAccessMethod.INSTANCE && indexType == IndexType.RTREE)
+                            || (chosenAccessMethod == BTreeAccessMethod.INSTANCE && indexType == IndexType.BTREE)) {
+                        result.add(
+                                new IntroduceSelectAccessMethodRule.IndexAccessInfo(chosenAccessMethod, chosenIndex));
+                    }
+                    // this if seems to be needed for ngram indexes. todo: can we remove this for other types as well?
+                    else if (resultVarsToIndexTypesMap.containsKey(indexEntry.getValue())) {
                         List<IndexType> appliedIndexTypes = resultVarsToIndexTypesMap.get(indexEntry.getValue());
                         if (!appliedIndexTypes.contains(indexType)) {
                             appliedIndexTypes.add(indexType);

asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceSelectAccessMethodRule.java

@@ -19,11 +19,13 @@
 package org.apache.asterix.optimizer.rules.am;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.TreeMap;
+import java.util.stream.IntStream;
 
 import org.apache.asterix.algebra.operators.CommitOperator;
 import org.apache.asterix.common.cluster.PartitioningProperties;
@@ -50,6 +52,7 @@
 import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
 import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
 import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import org.apache.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
 import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
 import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
@@ -383,7 +386,8 @@ protected boolean prefix(List<List<String>> list1, List<List<String>> list2) {
         return true;
     }
 
-    protected void removeSmallerPrefixIndexes(List<IndexAccessInfo> indexes) throws CompilationException {
+    protected void keepBestPrefixIndexes(List<IndexAccessInfo> indexes, List<Integer> numberOfMatchedKeys)
+            throws CompilationException {
         int len = indexes.size();
         int i, j;
         Index indexI, indexJ;
@@ -412,10 +416,19 @@ protected void removeSmallerPrefixIndexes(List<IndexAccessInfo> indexes) throws
                                 fieldNamesJ = findKeyFieldNames(indexJ);
                                 if (fieldNamesI.size() <= fieldNamesJ.size()) {
                                     if (prefix(fieldNamesI, fieldNamesJ)) {
-                                        include[i] = false;
+                                        if (numberOfMatchedKeys.get(j) > numberOfMatchedKeys.get(i)) {
+                                            include[i] = false;
+                                        } else {
+                                            include[j] = false;
+
+                                        }
                                     }
                                 } else if (prefix(fieldNamesJ, fieldNamesI)) {
-                                    include[j] = false;
+                                    if (numberOfMatchedKeys.get(i) > numberOfMatchedKeys.get(j)) {
+                                        include[j] = false;
+                                    } else {
+                                        include[i] = false;
+                                    }
                                 }
                             }
                         }
@@ -548,13 +561,16 @@ protected boolean checkAndApplyTheSelectTransformation(Mutable<ILogicalOperator>
 
                 // Choose all indexes that will be applied.
                 chooseAllIndexes(analyzedAMs, chosenIndexes);
-                removeSmallerPrefixIndexes(chosenIndexes);
-
                 if (chosenIndexes == null || chosenIndexes.isEmpty()) {
                     // We can't apply any index for this SELECT operator
                     context.addToDontApplySet(this, selectRef.getValue());
                     return false;
                 }
+                List<Integer> matchedKeyCountsNonCovering = new ArrayList<>();
+                List<Integer> matchedKeyCountsCovering = new ArrayList<>();
+                List<IndexAccessInfo> nonCoverigIndexes = new ArrayList<>();
+                List<IndexAccessInfo> coverigIndexes = new ArrayList<>();
+                fillFieldNamesInTheSubTree(subTree, context);
 
                 for (IndexAccessInfo indexAccessInfo : chosenIndexes) {
                     AccessMethodAnalysisContext analysisCtx = analyzedAMs.get(indexAccessInfo.getAccessMethod());
@@ -565,7 +581,15 @@ protected boolean checkAndApplyTheSelectTransformation(Mutable<ILogicalOperator>
                     AccessMethodUtils.indexOnlyPlanCheck(afterSelectRefs, selectRef, subTree, null,
                             indexAccessInfo.getIndex(), analysisCtx, context, indexOnlyPlanInfo, false);
                     indexAccessInfo.setIsIndexOnlyPlan(indexOnlyPlanInfo.getFirst());
+                    if (indexAccessInfo.isIndexOnlyPlan) {
+                        matchedKeyCountsCovering.add(analysisCtx.getNumberOfMatchedKeys(indexAccessInfo.getIndex()));
+                        coverigIndexes.add(indexAccessInfo);
+                    } else {
+                        matchedKeyCountsNonCovering.add(analysisCtx.getNumberOfMatchedKeys(indexAccessInfo.getIndex()));
+                        nonCoverigIndexes.add(indexAccessInfo);
+                    }
                 }
+                keepBestPrefixIndexes(nonCoverigIndexes, matchedKeyCountsNonCovering);
 
                 if (checkApplicableOnly) {
                     return true;
@@ -574,28 +598,36 @@ protected boolean checkAndApplyTheSelectTransformation(Mutable<ILogicalOperator>
                 // Apply plan transformation using chosen index.
                 boolean res;
                 // Primary index applicable?
+                // We should look in the list including both covering and non covering.
                 IndexAccessInfo chosenPrimaryIndex = fetchPrimaryIndexAmongChosenIndexes(chosenIndexes);
                 if (chosenPrimaryIndex != null) {
+                    subTree.getVarsToFieldNameMap().clear();
                     AccessMethodAnalysisContext analysisCtx = analyzedAMs.get(chosenPrimaryIndex.getAccessMethod());
                     res = chosenPrimaryIndex.getAccessMethod().applySelectPlanTransformation(afterSelectRefs, selectRef,
                             subTree, chosenPrimaryIndex.getIndex(), analysisCtx, context);
                     context.addToDontApplySet(this, selectRef.getValue());
-                } else if (chosenIndexes.size() == 1) {
-                    // Index-only plan possible?
+                } else if (coverigIndexes.isEmpty() && nonCoverigIndexes.size() == 1) {
                     // Gets the analysis context for the given index.
-                    AccessMethodAnalysisContext analysisCtx = analyzedAMs.get(chosenIndexes.get(0).getAccessMethod());
-
-                    // Finds the field name of each variable in the sub-tree.
-                    fillFieldNamesInTheSubTree(subTree, context);
+                    AccessMethodAnalysisContext analysisCtx =
+                            analyzedAMs.get(nonCoverigIndexes.get(0).getAccessMethod());
 
                     // Finally, try to apply plan transformation using chosen index.
-                    res = chosenIndexes.get(0).getAccessMethod().applySelectPlanTransformation(afterSelectRefs,
-                            selectRef, subTree, chosenIndexes.get(0).getIndex(), analysisCtx, context);
+                    res = nonCoverigIndexes.get(0).getAccessMethod().applySelectPlanTransformation(afterSelectRefs,
+                            selectRef, subTree, nonCoverigIndexes.get(0).getIndex(), analysisCtx, context);
                     context.addToDontApplySet(this, selectRef.getValue());
                 } else {
-                    // Multiple secondary indexes applicable?
-                    res = intersectAllSecondaryIndexes(chosenIndexes, analyzedAMs, context);
-                    context.addToDontApplySet(this, selectRef.getValue());
+                    if (!coverigIndexes.isEmpty()) {
+                        IndexAccessInfo bestCoveringIndex =
+                                chooseBestCoveringIndex(coverigIndexes, matchedKeyCountsCovering, analyzedAMs);
+                        AccessMethodAnalysisContext analysisCtx = analyzedAMs.get(bestCoveringIndex.getAccessMethod());
+                        res = bestCoveringIndex.getAccessMethod().applySelectPlanTransformation(afterSelectRefs,
+                                selectRef, subTree, bestCoveringIndex.getIndex(), analysisCtx, context);
+                        context.addToDontApplySet(this, selectRef.getValue());
+                    } else {
+                        subTree.getVarsToFieldNameMap().clear();
+                        res = intersectAllSecondaryIndexes(nonCoverigIndexes, analyzedAMs, context);
+                        context.addToDontApplySet(this, selectRef.getValue());
+                    }
                 }
 
                 // If the plan transformation is successful, we don't need to traverse
@@ -619,6 +651,68 @@ protected boolean checkAndApplyTheSelectTransformation(Mutable<ILogicalOperator>
 
     }
 
+    private IndexAccessInfo chooseBestCoveringIndex(List<IndexAccessInfo> indexes,
+            List<Integer> numberOfMatchedKeysForCoveringindexes,
+            Map<IAccessMethod, AccessMethodAnalysisContext> analyzedAMs) throws CompilationException {
+
+        if (indexes.size() == 1) {
+            return indexes.get(0);
+        }
+
+        // 1) Keep only indexes with max matched keys
+        int max = numberOfMatchedKeysForCoveringindexes.stream().mapToInt(Integer::intValue).max().orElse(0);
+        List<IndexAccessInfo> indexesWithMaxMatchedKeys = IntStream.range(0, indexes.size())
+                .filter(i -> numberOfMatchedKeysForCoveringindexes.get(i) == max).mapToObj(indexes::get).toList();
+        if (indexesWithMaxMatchedKeys.size() == 1) {
+            return indexesWithMaxMatchedKeys.get(0);
+        }
+
+        // 2) Find earliest equality positions
+        List<IndexInfo> indexesWithMaxMatchedKeysInfo = new ArrayList<>(indexesWithMaxMatchedKeys.size());
+        for (IndexAccessInfo indexAccessInfo : indexesWithMaxMatchedKeys) {
+            List<List<String>> indexFields = findKeyFieldNames(indexAccessInfo.getIndex());
+            List<IOptimizableFuncExpr> funcExprs =
+                    analyzedAMs.get(indexAccessInfo.getAccessMethod()).getMatchedFuncExprs();
+            IndexInfo indexInfo = new IndexInfo();
+            indexInfo.indexExprs = new ArrayList<>(Collections.nCopies(funcExprs.size(), null));
+            indexInfo.numKeys = indexFields.size();
+            indexInfo.indexaccessinfo = indexAccessInfo;
+            for (IOptimizableFuncExpr expr : funcExprs) {
+                for (int i = 0; i < expr.getNumLogicalVars(); i++) {
+                    int i1 = indexFields.indexOf(expr.getFieldName(i));
+                    indexInfo.indexExprs.set(i1, expr.getFuncExpr().getFunctionIdentifier());
+                }
+            }
+            indexesWithMaxMatchedKeysInfo.add(indexInfo);
+        }
+        return indexesWithMaxMatchedKeysInfo.stream().sorted().toList().getFirst().indexaccessinfo;
+    }
+
+    class IndexInfo implements Comparable<IndexInfo> {
+        List<FunctionIdentifier> indexExprs = new ArrayList<>();
+        int numKeys;
+        IndexAccessInfo indexaccessinfo;
+
+        @Override
+        public int compareTo(IndexInfo other) {
+            for (int i = 0; i < this.indexExprs.size() && i < other.indexExprs.size(); i++) {
+                if (this.indexExprs.get(i) == AlgebricksBuiltinFunctions.EQ
+                        && other.indexExprs.get(i) != AlgebricksBuiltinFunctions.EQ) {
+                    return -1;
+                } else if (this.indexExprs.get(i) != AlgebricksBuiltinFunctions.EQ
+                        && other.indexExprs.get(i) == AlgebricksBuiltinFunctions.EQ) {
+                    return 1;
+                }
+            }
+
+            if (this.indexExprs.size() != other.indexExprs.size()) {
+                return 0;
+            }
+
+            return Integer.compare(this.numKeys, other.numKeys);
+        }
+    }
+
     @Override
     public Map<FunctionIdentifier, List<IAccessMethod>> getAccessMethods() {
         return accessMethods;

asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-only-testcases/simple-index-only/simple-index-only.10.query.sqlpp

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+use test;
+
+drop index Orders.idx9_early_eq if exists;
+drop index Orders.idx9_late_eq if exists;
+
+create index idx10_thin on Orders(city,country,storeName);
+create index idx10_thick on Orders(city,country,storeName,extra1,extra2);
+
+
+explain select o.city, o.country, o.storeName
+from Orders o
+where o.city = 'NYC' and o.country > 'USA';

asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-only-testcases/simple-index-only/simple-index-only.11.query.sqlpp

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+use test;
+
+drop index Orders.idx10_thin if exists;
+drop index Orders.idx10_thick if exists;
+
+create index idx11_stops on Orders(city,pp,country,storeName);
+create index idx11_all_searchable on Orders(city,country,storeName);
+
+explain select o.city, o.country, o.storeName
+from Orders o
+where o.city = 'NYC' and o.country > 'USA';

asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-only-testcases/simple-index-only/simple-index-only.5.ddl.sqlpp

@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+
+create index storeName_idx on Orders(storeName);
+create index country_idx2 on Orders(storeName,city,country);
+create index storeName_city_idx2 on Orders(storeName,city);
+create index storeName_city_idx1 on Orders(city,storeName);
+create index country_idx1 on Orders(storeName,city,country);

asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-only-testcases/simple-index-only/simple-index-only.6.query.sqlpp

@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+explain select o.city
+from Orders o
+where o.storeName="big store";
+

asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/index-only-testcases/simple-index-only/simple-index-only.7.query.sqlpp

@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+
+explain select o.storeName
+from Orders o
+where o.storeName="big store";
+