Merge branch 'master' into processUpdateFrontend

# Conflicts:
#	src/engine/Server.cpp

src/ServerMain.cpp

@@ -107,6 +107,10 @@ int main(int argc, char** argv) {
       optionFactory.getProgramOption<"default-query-timeout">(),
       "Set the default timeout in seconds after which queries are cancelled"
       "automatically.");
+  add("service-max-value-rows,S",
+      optionFactory.getProgramOption<"service-max-value-rows">(),
+      "The maximal number of result rows to be passed to a SERVICE operation "
+      "as a VALUES clause to optimize its computation.");
   po::variables_map optionsMap;
 
   try {

src/engine/Server.h

@@ -5,6 +5,8 @@
 
 #pragma once
 
+#include <util/http/websocket/MessageSender.h>
+
 #include <string>
 #include <vector>
 
@@ -118,22 +120,52 @@ class Server {
   Awaitable<void> process(
       const ad_utility::httpUtils::HttpRequest auto& request, auto&& send);
 
-  /// Handle a http request that asks for the processing of a query.
+  // Indicates which type of operation is being processed.
+  enum class OperationType { Query, Update };
+
+  /// Handle a http request that asks for the processing of an query or update.
+  /// This is only a wrapper for `processQuery` and `processUpdate` which
+  /// does the error handling.
   /// \param params The key-value-pairs  sent in the HTTP GET request.
-  /// \param query The query.
+  /// \param queryOrUpdate The query or update.
   /// \param requestTimer Timer that measure the total processing
   ///                     time of this request.
   /// \param request The HTTP request.
   /// \param send The action that sends a http:response (see the
   ///             `HttpServer.h` for documentation).
   /// \param timeLimit Duration in seconds after which the query will be
   ///                  cancelled.
+  template <OperationType type>
+  Awaitable<void> processQueryOrUpdate(
+      const ad_utility::url_parser::ParamValueMap& params,
+      const string& queryOrUpdate, ad_utility::Timer& requestTimer,
+      const ad_utility::httpUtils::HttpRequest auto& request, auto&& send,
+      TimeLimit timeLimit);
+  // Do the actual execution of a query.
   Awaitable<void> processQuery(
       const ad_utility::url_parser::ParamValueMap& params, const string& query,
       ad_utility::Timer& requestTimer,
       const ad_utility::httpUtils::HttpRequest auto& request, auto&& send,
       TimeLimit timeLimit);
 
+  // Determine the media type to be used for the result. The media type is
+  // determined (in this order) by the current action (e.g.,
+  // "action=csv_export") and by the "Accept" header of the request.
+  static ad_utility::MediaType determineMediaType(
+      const ad_utility::url_parser::ParamValueMap& params,
+      const ad_utility::httpUtils::HttpRequest auto& request);
+  FRIEND_TEST(ServerTest, determineMediaType);
+  // Determine whether the subtrees and the result should be pinned.
+  static std::pair<bool, bool> determineResultPinning(
+      const ad_utility::url_parser::ParamValueMap& params);
+  FRIEND_TEST(ServerTest, determineResultPinning);
+  // Sets up the PlannedQuery s.t. it is ready to be executed.
+  Awaitable<PlannedQuery> setupPlannedQuery(
+      const ad_utility::url_parser::ParamValueMap& params,
+      const std::string& operation, QueryExecutionContext& qec,
+      SharedCancellationHandle handle, TimeLimit timeLimit,
+      const ad_utility::Timer& requestTimer);
+
   static json composeErrorResponseJson(
       const string& query, const std::string& errorMsg,
       ad_utility::Timer& requestTimer,
@@ -204,19 +236,16 @@ class Server {
   /// HTTP error response.
   bool checkAccessToken(std::optional<std::string_view> accessToken) const;
 
-  /// Checks if a URL parameter exists in the request, if we are allowed to
-  /// access it and it matches the expected `value`. If yes, return the value,
-  /// otherwise return `std::nullopt`. If `value` is `std::nullopt`, only check
-  /// if the key exists. We need this because we have parameters like
-  /// "cmd=stats", where a fixed combination of the key and value determines the
-  /// kind of action, as well as parameters like "index-decription=...", where
-  /// the key determines the kind of action. If the key is not found, always
-  /// return `std::nullopt`. If `accessAllowed` is false and a value is present,
-  /// throw an exception.
+  /// Checks if a URL parameter exists in the request, and it matches the
+  /// expected `value`. If yes, return the value, otherwise return
+  /// `std::nullopt`. If `value` is `std::nullopt`, only check if the key
+  /// exists. We need this because we have parameters like "cmd=stats", where a
+  /// fixed combination of the key and value determines the kind of action, as
+  /// well as parameters like "index-decription=...", where the key determines
+  /// the kind of action. If the key is not found, always return `std::nullopt`.
   static std::optional<std::string> checkParameter(
       const ad_utility::url_parser::ParamValueMap& parameters,
-      std::string_view key, std::optional<std::string> value,
-      bool accessAllowed);
+      std::string_view key, std::optional<std::string> value);
   FRIEND_TEST(ServerTest, checkParameter);
 
   /// Check if user-provided timeout is authorized with a valid access-token or

src/engine/sparqlExpressions/StringExpressions.cpp

@@ -125,9 +125,12 @@ using IriOrUriExpression = NARY<1, FV<std::identity, IriOrUriValueGetter>>;
 
 // STRLEN
 [[maybe_unused]] auto strlen = [](std::string_view s) {
-  return Id::makeFromInt(static_cast<int64_t>(s.size()));
+  // Count UTF-8 characters by skipping continuation bytes (those starting with
+  // "10").
+  auto utf8Len = std::ranges::count_if(
+      s, [](char c) { return (static_cast<unsigned char>(c) & 0xC0) != 0x80; });
+  return Id::makeFromInt(utf8Len);
 };
-
 using StrlenExpression =
     StringExpressionImpl<1, LiftStringFunction<decltype(strlen)>>;
 

src/global/RuntimeParameters.h

@@ -48,7 +48,7 @@ inline auto& RuntimeParameters() {
         Bool<"use-binsearch-transitive-path">{true},
         Bool<"group-by-hash-map-enabled">{false},
         Bool<"group-by-disable-index-scan-optimizations">{false},
-        SizeT<"service-max-value-rows">{100},
+        SizeT<"service-max-value-rows">{10'000},
         SizeT<"query-planning-budget">{1500}};
   }();
   return params;

src/global/ValueIdComparators.h

@@ -349,9 +349,12 @@ inline std::vector<std::pair<RandomIt, RandomIt>> getRangesForIndexTypes(
 // Helper function: Sort the non-overlapping ranges in `input` by the first
 // element, remove the empty ranges, and merge  directly adjacent ranges
 inline auto simplifyRanges =
-    []<typename RandomIt>(std::vector<std::pair<RandomIt, RandomIt>> input) {
-      // Eliminate empty ranges
-      std::erase_if(input, [](const auto& p) { return p.first == p.second; });
+    []<typename RandomIt>(std::vector<std::pair<RandomIt, RandomIt>> input,
+                          bool removeEmptyRanges = true) {
+      if (removeEmptyRanges) {
+        // Eliminate empty ranges
+        std::erase_if(input, [](const auto& p) { return p.first == p.second; });
+      }
       std::sort(input.begin(), input.end());
       if (input.empty()) {
         return input;
@@ -378,9 +381,13 @@ inline auto simplifyRanges =
 // 2. The condition x `comparison` value is fulfilled, where value is the value
 // of `valueId`.
 // 3. The datatype of x and `valueId` are compatible.
+//
+// When setting the flag argument `removeEmptyRanges` to false, empty ranges
+// [`begin`, `end`] where `begin` is equal to `end` will not be discarded.
 template <typename RandomIt>
 inline std::vector<std::pair<RandomIt, RandomIt>> getRangesForId(
-    RandomIt begin, RandomIt end, ValueId valueId, Comparison comparison) {
+    RandomIt begin, RandomIt end, ValueId valueId, Comparison comparison,
+    bool removeEmptyRanges = true) {
   // For the evaluation of FILTERs, comparisons that involve undefined values
   // are always false.
   if (valueId.getDatatype() == Datatype::Undefined) {
@@ -389,11 +396,15 @@ inline std::vector<std::pair<RandomIt, RandomIt>> getRangesForId(
   // This lambda enforces the invariants `non-empty` and `sorted`.
   switch (valueId.getDatatype()) {
     case Datatype::Double:
-      return detail::simplifyRanges(detail::getRangesForIntsAndDoubles(
-          begin, end, valueId.getDouble(), comparison));
+      return detail::simplifyRanges(
+          detail::getRangesForIntsAndDoubles(begin, end, valueId.getDouble(),
+                                             comparison),
+          removeEmptyRanges);
     case Datatype::Int:
-      return detail::simplifyRanges(detail::getRangesForIntsAndDoubles(
-          begin, end, valueId.getInt(), comparison));
+      return detail::simplifyRanges(
+          detail::getRangesForIntsAndDoubles(begin, end, valueId.getInt(),
+                                             comparison),
+          removeEmptyRanges);
     case Datatype::Undefined:
     case Datatype::VocabIndex:
     case Datatype::LocalVocabIndex:
@@ -405,7 +416,8 @@ inline std::vector<std::pair<RandomIt, RandomIt>> getRangesForId(
     case Datatype::BlankNodeIndex:
       // For `Date` the trivial comparison via bits is also correct.
       return detail::simplifyRanges(
-          detail::getRangesForIndexTypes(begin, end, valueId, comparison));
+          detail::getRangesForIndexTypes(begin, end, valueId, comparison),
+          removeEmptyRanges);
   }
   AD_FAIL();
 }

src/index/CMakeLists.txt

@@ -6,5 +6,6 @@ add_library(index
         DocsDB.cpp FTSAlgorithms.cpp
         PrefixHeuristic.cpp CompressedRelation.cpp
         PatternCreator.cpp ScanSpecification.cpp
+        CompressedBlockPrefiltering.cpp
         DeltaTriples.cpp LocalVocabEntry.cpp)
 qlever_target_link_libraries(index util parser vocabulary ${STXXL_LIBRARIES})