Add index selection pass for KQIR planning (#2278)

CMakeLists.txt

@@ -140,6 +140,7 @@ include(cmake/xxhash.cmake)
 include(cmake/span.cmake)
 include(cmake/trie.cmake)
 include(cmake/pegtl.cmake)
+include(cmake/rangev3.cmake)
 
 if (ENABLE_LUAJIT)
     include(cmake/luajit.cmake)
@@ -173,6 +174,7 @@ list(APPEND EXTERNAL_LIBS xxhash)
 list(APPEND EXTERNAL_LIBS span-lite)
 list(APPEND EXTERNAL_LIBS tsl_hat_trie)
 list(APPEND EXTERNAL_LIBS pegtl)
+list(APPEND EXTERNAL_LIBS range-v3)
 
 # Add git sha to version.h
 find_package(Git REQUIRED)

NOTICE

@@ -66,7 +66,7 @@ The text of each license is also included in licenses/LICENSE-[project].txt
 * LuaJIT(https://github.com/KvrocksLabs/LuaJIT)
 * lua(https://github.com/KvrocksLabs/lua, alternative to LuaJIT)
 * hat-trie(https://github.com/Tessil/hat-trie)
-* pegtl(https://github.com/taocpp/PEGTL, NOTE: changed to BSL-1.0 in main branch)
+* pegtl(https://github.com/taocpp/PEGTL, NOTE: changed to Boost Software License Version 1.0 in main branch)
 
 ================================================================
 Boost Software License Version 1.0
@@ -76,6 +76,7 @@ The text of each license is also included in licenses/LICENSE-[project].txt
 
 * jsoncons(https://github.com/danielaparker/jsoncons)
 * span-lite(https://github.com/martinmoene/span-lite)
+* range-v3(https://github.com/ericniebler/range-v3)
 
 ================================================================
 zlib/libpng licenses

cmake/rangev3.cmake

@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+include_guard()
+
+include(cmake/utils.cmake)
+
+FetchContent_DeclareGitHubWithMirror(rangev3
+  ericniebler/range-v3 0.12.0
+  MD5=e220e3f545fdf46241b4f139822d73a1
+)
+
+if (CMAKE_BUILD_TYPE STREQUAL "Debug")
+    set(WITH_DEBUG_INFO ON)
+elseif(CMAKE_BUILD_TYPE STREQUAL "Release")
+    set(WITH_DEBUG_INFO OFF)
+elseif(CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
+    set(WITH_DEBUG_INFO ON)
+elseif (CMAKE_BUILD_TYPE STREQUAL "MinSizeRel")
+    set(WITH_DEBUG_INFO OFF)
+endif()
+
+if (PORTABLE STREQUAL 0)
+    set(ARG_RANGES_NATIVE ON)
+else()
+    set(ARG_RANGES_NATIVE OFF)
+endif()
+
+FetchContent_MakeAvailableWithArgs(rangev3
+  RANGES_CXX_STD=17
+  RANGES_BUILD_CALENDAR_EXAMPLE=OFF
+  RANGES_DEBUG_INFO=${WITH_DEBUG_INFO}
+  RANGES_NATIVE=${ARG_RANGES_NATIVE}
+)

licenses/LICENSE-range-v3.txt

@@ -0,0 +1,151 @@
+========================================================
+Boost Software License - Version 1.0 - August 17th, 2003
+========================================================
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+==============================================================================
+libc++ License
+==============================================================================
+
+The libc++ library is dual licensed under both the University of Illinois
+"BSD-Like" license and the MIT license.  As a user of this code you may choose
+to use it under either license.  As a contributor, you agree to allow your code
+to be used under both.
+
+Full text of the relevant licenses is included below.
+
+==============================================================================
+
+University of Illinois/NCSA
+Open Source License
+
+Copyright (c) 2009-2014 by the contributors listed in CREDITS.TXT
+http://llvm.org/svn/llvm-project/libcxx/trunk/CREDITS.TXT
+
+All rights reserved.
+
+Developed by:
+
+    LLVM Team
+
+    University of Illinois at Urbana-Champaign
+
+    http://llvm.org
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal with
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimers.
+
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimers in the
+      documentation and/or other materials provided with the distribution.
+
+    * Neither the names of the LLVM Team, University of Illinois at
+      Urbana-Champaign, nor the names of its contributors may be used to
+      endorse or promote products derived from this Software without specific
+      prior written permission.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE
+SOFTWARE.
+
+==============================================================================
+
+Copyright (c) 2009-2014 by the contributors listed in CREDITS.TXT
+  http://llvm.org/svn/llvm-project/libcxx/trunk/CREDITS.TXT
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+==============================================================================
+Stepanov and McJones, "Elements of Programming" license
+==============================================================================
+
+// Copyright (c) 2009 Alexander Stepanov and Paul McJones
+//
+// Permission to use, copy, modify, distribute and sell this software
+// and its documentation for any purpose is hereby granted without
+// fee, provided that the above copyright notice appear in all copies
+// and that both that copyright notice and this permission notice
+// appear in supporting documentation. The authors make no
+// representations about the suitability of this software for any
+// purpose. It is provided "as is" without express or implied
+// warranty.
+//
+// Algorithms from
+// Elements of Programming
+// by Alexander Stepanov and Paul McJones
+// Addison-Wesley Professional, 2009
+
+==============================================================================
+SGI C++ Standard Template Library license
+==============================================================================
+
+// Copyright (c) 1994
+// Hewlett-Packard Company
+//
+// Permission to use, copy, modify, distribute and sell this software
+// and its documentation for any purpose is hereby granted without fee,
+// provided that the above copyright notice appear in all copies and
+// that both that copyright notice and this permission notice appear
+// in supporting documentation.  Hewlett-Packard Company makes no
+// representations about the suitability of this software for any
+// purpose.  It is provided "as is" without express or implied warranty.
+//
+// Copyright (c) 1996
+// Silicon Graphics Computer Systems, Inc.
+//
+// Permission to use, copy, modify, distribute and sell this software
+// and its documentation for any purpose is hereby granted without fee,
+// provided that the above copyright notice appear in all copies and
+// that both that copyright notice and this permission notice appear
+// in supporting documentation.  Silicon Graphics makes no
+// representations about the suitability of this software for any
+// purpose.  It is provided "as is" without express or implied warranty.
+//

src/search/index_info.h

@@ -39,6 +39,12 @@ struct FieldInfo {
       : name(std::move(name)), metadata(std::move(metadata)) {}
 
   bool IsSortable() const { return dynamic_cast<redis::SearchSortableFieldMetadata *>(metadata.get()) != nullptr; }
+  bool HasIndex() const { return !metadata->noindex; }
+
+  template <typename T>
+  const T *MetadataAs() const {
+    return dynamic_cast<const T *>(metadata.get());
+  }
 };
 
 struct IndexInfo {

src/search/interval.h

@@ -34,9 +34,13 @@ namespace kqir {
 struct Interval {
   double l, r;  // [l, r)
 
+  static inline const double inf = std::numeric_limits<double>::infinity();
+  static inline const double minf = -inf;
+
   Interval(double l, double r) : l(l), r(r) {}
 
   bool IsEmpty() const { return l >= r; }
+  static Interval Full() { return {minf, inf}; }
 
   bool operator==(const Interval &other) const { return l == other.l && r == other.r; }
   bool operator!=(const Interval &other) const { return !(*this == other); }
@@ -69,8 +73,8 @@ struct IntervalSet {
   using DataType = std::vector<std::pair<double, double>>;
   DataType intervals;
 
-  static inline const double inf = std::numeric_limits<double>::infinity();
-  static inline const double minf = -inf;
+  static inline const double inf = Interval::inf;
+  static inline const double minf = Interval::minf;
 
   static double NextNum(double val) { return std::nextafter(val, inf); }
 

src/search/ir.h

@@ -256,6 +256,16 @@ struct AndExpr : QueryExpr {
 
   explicit AndExpr(std::vector<std::unique_ptr<QueryExpr>> &&inners) : inners(std::move(inners)) {}
 
+  static std::unique_ptr<QueryExpr> Create(std::vector<std::unique_ptr<QueryExpr>> &&exprs) {
+    CHECK(!exprs.empty());
+
+    if (exprs.size() == 1) {
+      return std::move(exprs.front());
+    }
+
+    return std::make_unique<AndExpr>(std::move(exprs));
+  }
+
   std::string_view Name() const override { return "AndExpr"; }
   std::string Dump() const override {
     return fmt::format("(and {})", util::StringJoin(inners, [](const auto &v) { return v->Dump(); }));
@@ -279,6 +289,16 @@ struct OrExpr : QueryExpr {
 
   explicit OrExpr(std::vector<std::unique_ptr<QueryExpr>> &&inners) : inners(std::move(inners)) {}
 
+  static std::unique_ptr<QueryExpr> Create(std::vector<std::unique_ptr<QueryExpr>> &&exprs) {
+    CHECK(!exprs.empty());
+
+    if (exprs.size() == 1) {
+      return std::move(exprs.front());
+    }
+
+    return std::make_unique<OrExpr>(std::move(exprs));
+  }
+
   std::string_view Name() const override { return "OrExpr"; }
   std::string Dump() const override {
     return fmt::format("(or {})", util::StringJoin(inners, [](const auto &v) { return v->Dump(); }));

src/search/ir_plan.h

@@ -60,15 +60,19 @@ struct FieldScan : PlanOperator {
 
 struct NumericFieldScan : FieldScan {
   Interval range;
+  SortByClause::Order order;
 
-  NumericFieldScan(std::unique_ptr<FieldRef> field, Interval range) : FieldScan(std::move(field)), range(range) {}
+  NumericFieldScan(std::unique_ptr<FieldRef> field, Interval range, SortByClause::Order order = SortByClause::ASC)
+      : FieldScan(std::move(field)), range(range), order(order) {}
 
   std::string_view Name() const override { return "NumericFieldScan"; };
-  std::string Content() const override { return fmt::format("{}, {}", field->name, range.ToString()); };
+  std::string Content() const override {
+    return fmt::format("{}, {}, {}", field->name, range.ToString(), SortByClause::OrderToString(order));
+  };
   std::string Dump() const override { return fmt::format("numeric-scan {}", Content()); }
 
   std::unique_ptr<Node> Clone() const override {
-    return std::make_unique<NumericFieldScan>(field->CloneAs<FieldRef>(), range);
+    return std::make_unique<NumericFieldScan>(field->CloneAs<FieldRef>(), range, order);
   }
 };
 
@@ -110,6 +114,16 @@ struct Merge : PlanOperator {
 
   explicit Merge(std::vector<std::unique_ptr<PlanOperator>> &&ops) : ops(std::move(ops)) {}
 
+  static std::unique_ptr<PlanOperator> Create(std::vector<std::unique_ptr<PlanOperator>> &&ops) {
+    CHECK(!ops.empty());
+
+    if (ops.size() == 1) {
+      return std::move(ops.front());
+    }
+
+    return std::make_unique<Merge>(std::move(ops));
+  }
+
   std::string_view Name() const override { return "Merge"; };
   std::string Dump() const override {
     return fmt::format("(merge {})", util::StringJoin(ops, [](const auto &v) { return v->Dump(); }));

src/search/ir_sema_checker.h

@@ -75,7 +75,7 @@ struct SemaChecker {
     } else if (auto v = dynamic_cast<TagContainExpr *>(node)) {
       if (auto iter = current_index->fields.find(v->field->name); iter == current_index->fields.end()) {
         return {Status::NotOK, fmt::format("field `{}` not found in index `{}`", v->field->name)};
-      } else if (auto meta = dynamic_cast<redis::SearchTagFieldMetadata *>(iter->second.metadata.get()); !meta) {
+      } else if (auto meta = iter->second.MetadataAs<redis::SearchTagFieldMetadata>(); !meta) {
         return {Status::NotOK, fmt::format("field `{}` is not a tag field", v->field->name)};
       } else {
         v->field->info = &iter->second;
@@ -91,7 +91,7 @@ struct SemaChecker {
     } else if (auto v = dynamic_cast<NumericCompareExpr *>(node)) {
       if (auto iter = current_index->fields.find(v->field->name); iter == current_index->fields.end()) {
         return {Status::NotOK, fmt::format("field `{}` not found in index `{}`", v->field->name, current_index->name)};
-      } else if (!dynamic_cast<redis::SearchNumericFieldMetadata *>(iter->second.metadata.get())) {
+      } else if (!iter->second.MetadataAs<redis::SearchNumericFieldMetadata>()) {
         return {Status::NotOK, fmt::format("field `{}` is not a numeric field", v->field->name)};
       } else {
         v->field->info = &iter->second;