From 4a159948af0b56472898e34392754141c68e07cf Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Thu, 12 Dec 2024 15:16:18 +0100 Subject: [PATCH 01/14] Make query planning of index scans fast again (#1674) Since #1619, the size estimate for an index scan always involved one or several copies of the block metadata, which incurred a significant query planning cost for most queries. Now, such a copy is only made for an index scan followed by a `FILTER` and only the metadata of those blocks is copied, which remain after the `FILTER` (in which case the two operations are expensive anyway). --- src/engine/IndexScan.cpp | 32 +++++++++++++++++--------------- src/engine/IndexScan.h | 9 ++++----- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index f56123a42b..5bf47dd4c8 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -339,24 +339,26 @@ IndexScan::getBlockMetadata() const { // _____________________________________________________________________________ std::optional> IndexScan::getBlockMetadataOptionallyPrefiltered() const { + // The code after this is expensive because it always copies the complete + // block metadata, so we do an early return of `nullopt` (which means "use all + // the blocks") if no prefilter is specified. + if (!prefilter_.has_value()) { + return std::nullopt; + } auto optBlockSpan = getBlockMetadata(); - std::optional> optBlocks = std::nullopt; - if (optBlockSpan.has_value()) { - const auto& blockSpan = optBlockSpan.value(); - optBlocks = {blockSpan.begin(), blockSpan.end()}; - applyPrefilterIfPossible(optBlocks.value()); + if (!optBlockSpan.has_value()) { + return std::nullopt; } - return optBlocks; + return applyPrefilter(optBlockSpan.value()); } // _____________________________________________________________________________ -void IndexScan::applyPrefilterIfPossible( - std::vector& blocks) const { - if (prefilter_.has_value()) { - // Apply the prefilter on given blocks. - auto& [prefilterExpr, columnIndex] = prefilter_.value(); - blocks = prefilterExpr->evaluate(blocks, columnIndex); - } +std::vector IndexScan::applyPrefilter( + std::span blocks) const { + AD_CORRECTNESS_CHECK(prefilter_.has_value() && getLimit().isUnconstrained()); + // Apply the prefilter on given blocks. + auto& [prefilterExpr, columnIndex] = prefilter_.value(); + return prefilterExpr->evaluate(blocks, columnIndex); } // _____________________________________________________________________________ @@ -369,12 +371,12 @@ Permutation::IdTableGenerator IndexScan::getLazyScan( auto filteredBlocks = getLimit().isUnconstrained() ? std::optional(std::move(blocks)) : std::nullopt; - if (filteredBlocks.has_value()) { + if (filteredBlocks.has_value() && prefilter_.has_value()) { // Note: The prefilter expression applied with applyPrefilterIfPossible() // is not related to the prefilter procedure mentioned in the comment above. // If this IndexScan owns a pair, it can // be applied. - applyPrefilterIfPossible(filteredBlocks.value()); + filteredBlocks = applyPrefilter(filteredBlocks.value()); } return getScanPermutation().lazyScan(getScanSpecification(), filteredBlocks, additionalColumns(), cancellationHandle_, diff --git a/src/engine/IndexScan.h b/src/engine/IndexScan.h index c10680f59e..d778260efe 100644 --- a/src/engine/IndexScan.h +++ b/src/engine/IndexScan.h @@ -223,11 +223,10 @@ class IndexScan final : public Operation { std::optional> getBlockMetadataOptionallyPrefiltered() const; - // If `isUnconstrained()` yields true, return the blocks as given or the - // prefiltered blocks (if `prefilter_` has value). If `isUnconstrained()` is - // false, return `std::nullopt`. - void applyPrefilterIfPossible( - std::vector& blocks) const; + // Apply the `prefilter_` to the `blocks`. May only be called if the limit is + // unconstrained, and a `prefilter_` exists. + std::vector applyPrefilter( + std::span blocks) const; // Helper functions for the public `getLazyScanFor...` methods and // `chunkedIndexScan` (see above). From 70964d6e5aaaf0d5f507c384c78933327865342a Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Thu, 12 Dec 2024 18:54:24 +0100 Subject: [PATCH 02/14] Allow operations to not store their result in the cache (#1665) Each operation now has a `bool` that determines whether the results can be stored in the cache or not (whether it is actually stored depends on other circumstances, like the available cache size). That `bool` does not have to be fixed when the operation is created, but can be changed. For example, this is useful for index scans that only return a subset of their full result (because of another constraining operation, like a join or a filter). --- src/engine/Operation.cpp | 39 ++++++++++++++++++++++++++++++------ src/engine/Operation.h | 38 +++++++++++++++++------------------ src/util/ConcurrentCache.h | 22 ++++++++++++++++++++ test/ConcurrentCacheTest.cpp | 36 +++++++++++++++++++++++++++++++++ test/OperationTest.cpp | 37 ++++++++++++++++++++++++++++++++++ 5 files changed, 147 insertions(+), 25 deletions(-) diff --git a/src/engine/Operation.cpp b/src/engine/Operation.cpp index e4d7136527..1a9f53fa76 100644 --- a/src/engine/Operation.cpp +++ b/src/engine/Operation.cpp @@ -232,7 +232,8 @@ CacheValue Operation::runComputationAndPrepareForCache( auto maxSize = std::min(RuntimeParameters().get<"lazy-result-max-cache-size">(), cache.getMaxSizeSingleEntry()); - if (!result.isFullyMaterialized() && !unlikelyToFitInCache(maxSize)) { + if (canResultBeCached() && !result.isFullyMaterialized() && + !unlikelyToFitInCache(maxSize)) { AD_CONTRACT_CHECK(!pinned); result.cacheDuringConsumption( [maxSize]( @@ -316,11 +317,16 @@ std::shared_ptr Operation::getResult( bool onlyReadFromCache = computationMode == ComputationMode::ONLY_IF_CACHED; - auto result = - pinResult ? cache.computeOncePinned(cacheKey, cacheSetup, - onlyReadFromCache, suitedForCache) - : cache.computeOnce(cacheKey, cacheSetup, onlyReadFromCache, - suitedForCache); + auto result = [&]() { + auto compute = [&](auto&&... args) { + if (!canResultBeCached()) { + return cache.computeButDontStore(AD_FWD(args)...); + } + return pinResult ? cache.computeOncePinned(AD_FWD(args)...) + : cache.computeOnce(AD_FWD(args)...); + }; + return compute(cacheKey, cacheSetup, onlyReadFromCache, suitedForCache); + }(); if (result._resultPointer == nullptr) { AD_CORRECTNESS_CHECK(onlyReadFromCache); @@ -596,3 +602,24 @@ void Operation::signalQueryUpdate() const { _executionContext->signalQueryUpdate(*_rootRuntimeInfo); } } + +// _____________________________________________________________________________ +std::string Operation::getCacheKey() const { + auto result = getCacheKeyImpl(); + if (_limit._limit.has_value()) { + absl::StrAppend(&result, " LIMIT ", _limit._limit.value()); + } + if (_limit._offset != 0) { + absl::StrAppend(&result, " OFFSET ", _limit._offset); + } + return result; +} + +// _____________________________________________________________________________ +uint64_t Operation::getSizeEstimate() { + if (_limit._limit.has_value()) { + return std::min(_limit._limit.value(), getSizeEstimateBeforeLimit()); + } else { + return getSizeEstimateBeforeLimit(); + } +} diff --git a/src/engine/Operation.h b/src/engine/Operation.h index 3e06a9498e..9e649cb0a4 100644 --- a/src/engine/Operation.h +++ b/src/engine/Operation.h @@ -90,6 +90,9 @@ class Operation { // limit/offset is applied post computation. bool externalLimitApplied_ = false; + // See the documentation of the getter function below. + bool canResultBeCached_ = true; + public: // Holds a `PrefilterExpression` with its corresponding `Variable`. using PrefilterVariablePair = sparqlExpression::PrefilterExprVariablePair; @@ -162,20 +165,23 @@ class Operation { // Get a unique, not ambiguous string representation for a subtree. // This should act like an ID for each subtree. // Calls `getCacheKeyImpl` and adds the information about the `LIMIT` clause. - virtual string getCacheKey() const final { - auto result = getCacheKeyImpl(); - if (_limit._limit.has_value()) { - absl::StrAppend(&result, " LIMIT ", _limit._limit.value()); - } - if (_limit._offset != 0) { - absl::StrAppend(&result, " OFFSET ", _limit._offset); - } - return result; - } + virtual std::string getCacheKey() const final; + + // If this function returns `false`, then the result of this `Operation` will + // never be stored in the cache. It might however be read from the cache. + // This can be used, if the operation actually only returns a subset of the + // actual result because it has been constrained by a parent operation (e.g. + // an IndexScan that has been prefiltered by another operation which it is + // joined with). + virtual bool canResultBeCached() const { return canResultBeCached_; } + + // After calling this function, `canResultBeCached()` will return `false` (see + // above for details). + virtual void disableStoringInCache() final { canResultBeCached_ = false; } private: - // The individual implementation of `getCacheKey` (see above) that has to be - // customized by every child class. + // The individual implementation of `getCacheKey` (see above) that has to + // be customized by every child class. virtual string getCacheKeyImpl() const = 0; public: @@ -186,13 +192,7 @@ class Operation { virtual size_t getCostEstimate() = 0; - virtual uint64_t getSizeEstimate() final { - if (_limit._limit.has_value()) { - return std::min(_limit._limit.value(), getSizeEstimateBeforeLimit()); - } else { - return getSizeEstimateBeforeLimit(); - } - } + virtual uint64_t getSizeEstimate() final; private: virtual uint64_t getSizeEstimateBeforeLimit() = 0; diff --git a/src/util/ConcurrentCache.h b/src/util/ConcurrentCache.h index 2f22efde8c..21262da12b 100644 --- a/src/util/ConcurrentCache.h +++ b/src/util/ConcurrentCache.h @@ -208,6 +208,28 @@ class ConcurrentCache { suitedForCache); } + // If the result is contained in the cache, read and return it. Otherwise, + // compute it, but do not store it in the cache. The interface is the same as + // for the above two functions, therefore some of the arguments are unused. + ResultAndCacheStatus computeButDontStore( + const Key& key, + const InvocableWithConvertibleReturnType auto& computeFunction, + bool onlyReadFromCache, + [[maybe_unused]] const InvocableWithConvertibleReturnType< + bool, const Value&> auto& suitedForCache) { + { + auto resultPtr = _cacheAndInProgressMap.wlock()->_cache[key]; + if (resultPtr != nullptr) { + return {std::move(resultPtr), CacheStatus::cachedNotPinned}; + } + } + if (onlyReadFromCache) { + return {nullptr, CacheStatus::notInCacheAndNotComputed}; + } + auto value = std::make_shared(computeFunction()); + return {std::move(value), CacheStatus::computed}; + } + // Insert `value` into the cache, if the `key` is not already present. In case // `pinned` is true and the key is already present, the existing value is // pinned in case it is not pinned yet. diff --git a/test/ConcurrentCacheTest.cpp b/test/ConcurrentCacheTest.cpp index f52eca0561..9dbfbde509 100644 --- a/test/ConcurrentCacheTest.cpp +++ b/test/ConcurrentCacheTest.cpp @@ -530,3 +530,39 @@ TEST(ConcurrentCache, testTryInsertIfNotPresentDoesWorkCorrectly) { expectContainsSingleElementAtKey0(true, "jkl"); } + +TEST(ConcurrentCache, computeButDontStore) { + SimpleConcurrentLruCache cache{}; + + // The last argument of `computeOnce...`: For the sake of this test, all + // results are suitable for the cache. Note: In the `computeButDontStore` + // function this argument is ignored, because the results are never stored in + // the cache. + auto alwaysSuitable = [](auto&&) { return true; }; + // Store the element in the cache. + cache.computeOnce( + 42, []() { return "42"; }, false, alwaysSuitable); + + // The result is read from the cache, so we get "42", not "blubb". + auto res = cache.computeButDontStore( + 42, []() { return "blubb"; }, false, alwaysSuitable); + EXPECT_EQ(*res._resultPointer, "42"); + + // The same with `onlyReadFromCache` == true; + res = cache.computeButDontStore( + 42, []() { return "blubb"; }, true, alwaysSuitable); + EXPECT_EQ(*res._resultPointer, "42"); + + cache.clearAll(); + + // Compute, but don't store. + res = cache.computeButDontStore( + 42, []() { return "blubb"; }, false, alwaysSuitable); + EXPECT_EQ(*res._resultPointer, "blubb"); + + // Nothing is stored in the cache, so we cannot read it. + EXPECT_FALSE(cache.getIfContained(42).has_value()); + res = cache.computeButDontStore( + 42, []() { return "blubb"; }, true, alwaysSuitable); + EXPECT_EQ(res._resultPointer, nullptr); +} diff --git a/test/OperationTest.cpp b/test/OperationTest.cpp index c1ad709a4f..4ad1f1313c 100644 --- a/test/OperationTest.cpp +++ b/test/OperationTest.cpp @@ -653,3 +653,40 @@ TEST(Operation, checkLazyOperationIsNotCachedIfUnlikelyToFitInCache) { EXPECT_FALSE( qec->getQueryTreeCache().cacheContains(makeQueryCacheKey("test"))); } + +TEST(OperationTest, disableCaching) { + auto qec = getQec(); + qec->getQueryTreeCache().clearAll(); + std::vector idTablesVector{}; + idTablesVector.push_back(makeIdTableFromVector({{3, 4}})); + idTablesVector.push_back(makeIdTableFromVector({{7, 8}, {9, 123}})); + ValuesForTesting valuesForTesting{ + qec, std::move(idTablesVector), {Variable{"?x"}, Variable{"?y"}}, true}; + + QueryCacheKey cacheKey{valuesForTesting.getCacheKey(), + qec->locatedTriplesSnapshot().index_}; + + // By default, the result of `valuesForTesting` is cached because it is + // sufficiently small, no matter if it was computed lazily or fully + // materialized. + EXPECT_FALSE(qec->getQueryTreeCache().cacheContains(cacheKey)); + valuesForTesting.getResult(true); + EXPECT_TRUE(qec->getQueryTreeCache().cacheContains(cacheKey)); + qec->getQueryTreeCache().clearAll(); + EXPECT_FALSE(qec->getQueryTreeCache().cacheContains(cacheKey)); + valuesForTesting.getResult(false); + EXPECT_TRUE(qec->getQueryTreeCache().cacheContains(cacheKey)); + + // We now disable caching for the `valuesForTesting`. Then the result is never + // cached, no matter if it is computed lazily or fully materialized. + valuesForTesting.disableStoringInCache(); + qec->getQueryTreeCache().clearAll(); + + EXPECT_FALSE(qec->getQueryTreeCache().cacheContains(cacheKey)); + valuesForTesting.getResult(true); + EXPECT_FALSE(qec->getQueryTreeCache().cacheContains(cacheKey)); + qec->getQueryTreeCache().clearAll(); + EXPECT_FALSE(qec->getQueryTreeCache().cacheContains(cacheKey)); + valuesForTesting.getResult(false); + EXPECT_FALSE(qec->getQueryTreeCache().cacheContains(cacheKey)); +} From 4237e0d4af70e6e400f4357f61756eb5873fe98a Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Thu, 12 Dec 2024 21:49:38 +0100 Subject: [PATCH 03/14] For C++17, use `range-v3` instead of `std::ranges` (#1667) This is a first step towards making QLever compile with C++17. If the compile-time flag `QLEVER_CPP_17` is set, use Eric Niebler's `range-v3` library as a drop-in replacement for `std::ranges`. In the code, we simply write `ql::ranges` instead of `std::ranges` in most places. Some places need special treatment. For example, where `std::ranges` was used as a C++20 concept, we now use the macros `CPP_template` and `CPP_and` (also from the `range-v3` library), which does the right thing for both C++20 and C++17. --- .github/workflows/native-build.yml | 4 + CMakeLists.txt | 22 ++- src/backports/algorithm.h | 59 +++++++ src/backports/concepts.h | 17 ++ src/engine/AddCombinedRowToTable.h | 4 +- src/engine/Bind.cpp | 4 +- src/engine/CallFixedSize.h | 4 +- src/engine/CartesianProductJoin.cpp | 41 +++-- src/engine/CartesianProductJoin.h | 9 +- src/engine/CheckUsePatternTrick.cpp | 22 +-- src/engine/Distinct.cpp | 44 +++--- src/engine/Engine.cpp | 2 +- src/engine/Engine.h | 2 +- src/engine/ExportQueryExecutionTrees.cpp | 12 +- src/engine/ExportQueryExecutionTrees.h | 2 +- src/engine/Filter.cpp | 2 +- src/engine/GroupBy.cpp | 34 ++-- src/engine/HasPredicateScan.cpp | 2 +- src/engine/IndexScan.cpp | 36 +++-- src/engine/Join.cpp | 12 +- src/engine/LazyGroupBy.h | 4 +- src/engine/LocalVocab.cpp | 6 +- src/engine/LocalVocab.h | 12 +- src/engine/MultiColumnJoin.cpp | 4 +- src/engine/Operation.cpp | 4 +- src/engine/OptionalJoin.cpp | 4 +- src/engine/OrderBy.cpp | 2 +- src/engine/PathSearch.cpp | 6 +- src/engine/QueryExecutionTree.cpp | 4 +- src/engine/QueryPlanner.cpp | 100 ++++++------ src/engine/Result.cpp | 10 +- src/engine/RuntimeInformation.cpp | 8 +- src/engine/Service.cpp | 8 +- src/engine/SpatialJoinAlgorithms.cpp | 6 +- src/engine/TextIndexScanForWord.cpp | 2 +- src/engine/TextLimit.cpp | 16 +- src/engine/TransitivePathBase.cpp | 2 +- src/engine/TransitivePathBinSearch.h | 2 +- src/engine/Union.cpp | 21 ++- src/engine/Values.cpp | 2 +- src/engine/VariableToColumnMap.cpp | 6 +- src/engine/VariableToColumnMap.h | 4 + .../idTable/CompressedExternalIdTable.h | 37 ++--- src/engine/idTable/IdTable.h | 50 +++--- src/engine/idTable/IdTableRow.h | 13 +- .../sparqlExpressions/CountStarExpression.cpp | 14 +- .../sparqlExpressions/LiteralExpression.h | 10 +- .../sparqlExpressions/NaryExpressionImpl.h | 4 +- .../NumericBinaryExpressions.cpp | 4 +- .../NumericUnaryExpressions.cpp | 4 +- .../PrefilterExpressionIndex.cpp | 14 +- .../sparqlExpressions/RegexExpression.cpp | 4 +- .../RelationalExpressions.cpp | 20 +-- .../sparqlExpressions/RelationalExpressions.h | 2 +- .../sparqlExpressions/SetOfIntervals.cpp | 2 +- src/engine/sparqlExpressions/SetOfIntervals.h | 3 +- .../sparqlExpressions/SparqlExpression.cpp | 9 +- .../SparqlExpressionGenerators.h | 2 +- .../SparqlExpressionPimpl.cpp | 2 +- .../sparqlExpressions/SparqlExpressionPimpl.h | 2 +- .../sparqlExpressions/StringExpressions.cpp | 12 +- .../sparqlExpressions/VariadicExpression.h | 2 +- src/global/IdTriple.h | 4 +- src/global/SpecialIds.h | 5 +- src/global/ValueId.h | 4 +- src/global/ValueIdComparators.h | 1 + src/index/CompressedRelation.cpp | 99 ++++++------ src/index/CompressedRelation.h | 4 +- src/index/DeltaTriples.cpp | 27 ++-- src/index/DocsDB.cpp | 3 +- src/index/IndexBuilderTypes.h | 2 +- src/index/IndexImpl.Text.cpp | 18 +-- src/index/IndexImpl.cpp | 10 +- src/index/IndexMetaData.h | 2 +- src/index/LocatedTriples.cpp | 25 ++- src/index/LocatedTriples.h | 6 +- src/index/PatternCreator.cpp | 8 +- src/index/PrefixHeuristic.cpp | 2 +- src/index/StringSortComparator.h | 2 +- src/index/StxxlSortFunctors.h | 2 +- src/index/Vocabulary.cpp | 4 +- src/index/Vocabulary.h | 2 +- src/index/VocabularyMergerImpl.h | 16 +- src/index/vocabulary/CompressionWrappers.h | 2 +- .../vocabulary/VocabularyBinarySearchMixin.h | 6 +- .../VocabularyInMemoryBinSearch.cpp | 2 +- src/parser/LiteralOrIri.cpp | 2 +- src/parser/ParsedQuery.cpp | 10 +- src/parser/RdfEscaping.cpp | 4 +- src/parser/RdfParser.cpp | 4 +- .../sparqlParser/SparqlQleverVisitor.cpp | 30 ++-- src/util/Algorithm.h | 10 +- src/util/BatchedPipeline.h | 2 +- src/util/BlankNodeManager.cpp | 6 +- src/util/BlankNodeManager.h | 2 +- src/util/ChunkedForLoop.h | 27 ++-- src/util/ConfigManager/ConfigManager.cpp | 80 +++++----- src/util/ConfigManager/ConfigOption.cpp | 4 +- src/util/ConstexprMap.h | 3 +- src/util/ConstexprUtils.h | 3 +- src/util/FsstCompressor.h | 2 +- src/util/Generator.h | 18 ++- src/util/JoinAlgorithms/FindUndefRanges.h | 22 +-- src/util/JoinAlgorithms/JoinAlgorithms.h | 70 +++++---- src/util/MemorySize/MemorySize.h | 2 +- src/util/ParallelMultiwayMerge.h | 13 +- src/util/PriorityQueue.h | 2 +- src/util/Random.h | 2 +- src/util/Serializer/ByteBufferSerializer.h | 2 +- src/util/Simple8bCode.h | 2 +- src/util/StringUtils.cpp | 15 +- src/util/StringUtils.h | 112 +++----------- src/util/StringUtilsImpl.h | 97 ++++++++++++ src/util/TaskQueue.h | 2 +- src/util/ThreadSafeQueue.h | 2 +- src/util/Views.h | 146 +++++++++++------- src/util/http/MediaTypes.cpp | 6 +- test/AddCombinedRowToTableTest.cpp | 2 +- test/AlgorithmTest.cpp | 30 ++-- test/AsyncStreamTest.cpp | 4 +- test/BenchmarkMeasurementContainerTest.cpp | 10 +- test/CMakeLists.txt | 1 + test/CallFixedSizeTest.cpp | 2 +- test/CompactStringVectorTest.cpp | 2 +- test/ComparisonWithNanTest.cpp | 4 +- test/CompressedRelationsTest.cpp | 27 ++-- test/ConfigManagerTest.cpp | 18 +-- test/DeltaTriplesTest.cpp | 2 +- test/FindUndefRangesTest.cpp | 4 +- test/GeoPointTest.cpp | 1 + test/GroupByTest.cpp | 8 +- test/HttpTest.cpp | 2 +- test/IdTableHelpersTest.cpp | 63 ++++---- test/IdTableTest.cpp | 13 +- test/JoinAlgorithmsTest.cpp | 4 +- test/JoinTest.cpp | 16 +- test/LocalVocabTest.cpp | 8 +- test/LocatedTriplesTest.cpp | 2 +- test/MemorySizeTest.cpp | 36 ++--- test/OrderByTest.cpp | 8 +- test/ParallelMultiwayMergeTest.cpp | 22 +-- test/PrefilterExpressionIndexTest.cpp | 2 +- test/QueryPlannerTestHelpers.h | 4 +- test/RandomTest.cpp | 33 ++-- test/RdfParserTest.cpp | 2 +- test/RelationalExpressionTest.cpp | 6 +- test/ResultTableColumnOperationsTest.cpp | 2 +- test/SortTest.cpp | 8 +- test/StringUtilsTest.cpp | 11 +- test/ThreadSafeQueueTest.cpp | 10 +- test/ViewsTest.cpp | 18 +-- test/backports/CMakeLists.txt | 6 + test/backports/DebugJoinView.cpp | 38 +++++ test/backports/algorithmTest.cpp | 9 ++ test/engine/BindTest.cpp | 4 +- test/engine/CartesianProductJoinTest.cpp | 8 +- test/engine/DistinctTest.cpp | 10 +- test/engine/IndexScanTest.cpp | 2 +- test/engine/ValuesForTesting.h | 17 +- .../idTable/CompressedExternalIdTableTest.cpp | 12 +- test/index/PatternCreatorTest.cpp | 4 +- test/util/IdTableHelpers.cpp | 50 +++--- test/util/RandomTestHelpers.h | 4 +- 163 files changed, 1258 insertions(+), 1026 deletions(-) create mode 100644 src/backports/algorithm.h create mode 100644 src/backports/concepts.h create mode 100644 src/util/StringUtilsImpl.h create mode 100644 test/backports/CMakeLists.txt create mode 100644 test/backports/DebugJoinView.cpp create mode 100644 test/backports/algorithmTest.cpp diff --git a/.github/workflows/native-build.yml b/.github/workflows/native-build.yml index 41e7561e23..da8bc0f727 100644 --- a/.github/workflows/native-build.yml +++ b/.github/workflows/native-build.yml @@ -40,6 +40,10 @@ jobs: - compiler: clang compiler-version: 13 include: + - compiler: gcc + compiler-version: 11 + additional-cmake-options: "-DUSE_CPP_17_BACKPORTS=ON" + build-type: Release - compiler: clang compiler-version: 16 asan-flags: "-fsanitize=address -fno-omit-frame-pointer" diff --git a/CMakeLists.txt b/CMakeLists.txt index 0503cd210f..3679de4c51 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -96,6 +96,17 @@ FetchContent_Declare( SOURCE_SUBDIR runtime/Cpp ) +################################# +# Range v3 (for C++-17 backwards compatibility) +################################ +FetchContent_Declare( + range-v3 + GIT_REPOSITORY https://github.com/joka921/range-v3 + GIT_TAG 1dc0b09abab1bdc7d085a78754abd5c6e37a5d0c # 0.12.0 +) + + + ################################ # Threading ################################ @@ -184,6 +195,14 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra") # Enable the specification of additional compiler flags manually from the commandline set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ADDITIONAL_COMPILER_FLAGS}") +# Enable the manual usage of the C++ 17 backports (currently `range-v3` instead +# of `std::ranges` and the `std::enable_if_t` based expansion of the concept +# macros from `range-v3`. +set(USE_CPP_17_BACKPORTS OFF CACHE BOOL "Use the C++17 backports (range-v3 and enable_if_t instead of std::ranges and concepts)") +if (${USE_CPP_17_BACKPORTS}) + add_definitions("-DQLEVER_CPP_17 -DCPP_CXX_CONCEPTS=0") +endif() + # Enable the specification of additional linker flags manually from the commandline set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${ADDITIONAL_LINKER_FLAGS}") set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${ADDITIONAL_LINKER_FLAGS}") @@ -321,7 +340,7 @@ FetchContent_Declare( ################################ # Apply FetchContent ################################ -FetchContent_MakeAvailable(googletest ctre abseil re2 stxxl fsst s2 nlohmann-json antlr) +FetchContent_MakeAvailable(googletest ctre abseil re2 stxxl fsst s2 nlohmann-json antlr range-v3) # Disable some warnings in RE2, STXXL, and GTEST target_compile_options(s2 PRIVATE -Wno-sign-compare -Wno-unused-parameter -Wno-class-memaccess -Wno-comment -Wno-redundant-move -Wno-unknown-warning-option -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-unused-but-set-variable -Wno-unused-function) target_compile_options(re2 PRIVATE -Wno-unused-parameter) @@ -333,6 +352,7 @@ include_directories(${ctre_SOURCE_DIR}/single-header) target_compile_options(fsst PRIVATE -Wno-extra -Wno-all -Wno-error) target_compile_options(fsst12 PRIVATE -Wno-extra -Wno-all -Wno-error) include_directories(${fsst_SOURCE_DIR}) +include_directories(${range-v3_SOURCE_DIR}/include) target_compile_options(antlr4_static PRIVATE -Wno-all -Wno-extra -Wno-error -Wno-deprecated-declarations) # Only required because a lot of classes that do not explicitly link against antlr4_static use the headers. include_directories(SYSTEM "${antlr_SOURCE_DIR}/runtime/Cpp/runtime/src") diff --git a/src/backports/algorithm.h b/src/backports/algorithm.h new file mode 100644 index 0000000000..90b4e2884c --- /dev/null +++ b/src/backports/algorithm.h @@ -0,0 +1,59 @@ +// Copyright 2024, University of Freiburg, +// Chair of Algorithms and Data Structures. +// Author: Johannes Kalmbach + +#pragma once + +#include +#include +#include +#include + +// The following defines namespaces `ql::ranges` and `ql::views` that are almost +// drop-in replacements for `std::ranges` and `std::views`. In C++20 mode (when +// the `QLEVER_CPP_17` macro is not used), these namespaces are simply aliases +// for `std::ranges` and `std::views`. In C++17 mode they contain the ranges and +// views from Erice Niebler's `range-v3` library. NOTE: `ql::ranges::unique` +// currently doesn't work, because the interface to this function is different +// in both implementations. NOTE: There might be other caveats which we are +// currently not aware of, because they only affect functions that we currently +// don't use. For those, the following header can be expanded in the future. +#ifndef QLEVER_CPP_17 +#include +#include +#endif + +namespace ql { + +namespace ranges { +#ifdef QLEVER_CPP_17 +using namespace ::ranges; + +// The `view` concept (which is rather important when implementing custom views) +// is in a different namespace in range-v3, so we make it manually accessible. +template +CPP_concept view = ::ranges::cpp20::view; +#else +using namespace std::ranges; +#endif +} // namespace ranges + +namespace views { +#ifdef QLEVER_CPP_17 +using namespace ::ranges::views; +#else +using namespace std::views; +#endif +} // namespace views + +// The namespace `ql::concepts` includes concepts that are contained in the +// C++20 standard as well as in `range-v3`. +namespace concepts { +#ifdef QLEVER_CPP_17 +using namespace ::concepts; +#else +using namespace std; +#endif +} // namespace concepts + +} // namespace ql diff --git a/src/backports/concepts.h b/src/backports/concepts.h new file mode 100644 index 0000000000..ad0159da32 --- /dev/null +++ b/src/backports/concepts.h @@ -0,0 +1,17 @@ +// Copyright 2024, University of Freiburg, +// Chair of Algorithms and Data Structures. +// Author: Johannes Kalmbach + +#pragma once + +// Define the following macros: +// `QL_OPT_CONCEPT(arg)` which expands to `arg` in C++20 mode, and to nothing in +// C++17 mode. It can be used to easily opt out of concepts that are only used +// for documentation and increased safety and not for overload resolution. +// Example usage: +// `(QL_OPT_CONCEPT(std::view) auto x = someFunction();` +#ifdef QLEVER_CPP_17 +#define QL_OPT_CONCEPT(arg) +#else +#define QL_OPT_CONCEPT(arg) arg +#endif diff --git a/src/engine/AddCombinedRowToTable.h b/src/engine/AddCombinedRowToTable.h index 8c8939f64c..c43e298a88 100644 --- a/src/engine/AddCombinedRowToTable.h +++ b/src/engine/AddCombinedRowToTable.h @@ -349,8 +349,8 @@ class AddCombinedRowToIdTable { // Make sure to reset `mergedVocab_` so it is in a valid state again. mergedVocab_ = LocalVocab{}; // Only merge non-null vocabs. - auto range = currentVocabs_ | std::views::filter(toBool) | - std::views::transform(dereference); + auto range = currentVocabs_ | ql::views::filter(toBool) | + ql::views::transform(dereference); mergedVocab_.mergeWith(std::ranges::ref_view{range}); } } diff --git a/src/engine/Bind.cpp b/src/engine/Bind.cpp index 2419531888..95de8a4dfe 100644 --- a/src/engine/Bind.cpp +++ b/src/engine/Bind.cpp @@ -86,8 +86,8 @@ IdTable Bind::cloneSubView(const IdTable& idTable, const std::pair& subrange) { IdTable result(idTable.numColumns(), idTable.getAllocator()); result.resize(subrange.second - subrange.first); - std::ranges::copy(idTable.begin() + subrange.first, - idTable.begin() + subrange.second, result.begin()); + ql::ranges::copy(idTable.begin() + subrange.first, + idTable.begin() + subrange.second, result.begin()); return result; } diff --git a/src/engine/CallFixedSize.h b/src/engine/CallFixedSize.h index 065a7f51a3..9a62457597 100644 --- a/src/engine/CallFixedSize.h +++ b/src/engine/CallFixedSize.h @@ -56,7 +56,7 @@ template auto callLambdaForIntArray(std::array array, auto&& lambda, auto&&... args) { AD_CONTRACT_CHECK( - std::ranges::all_of(array, [](auto el) { return el <= maxValue; })); + ql::ranges::all_of(array, [](auto el) { return el <= maxValue; })); using ArrayType = std::array; // Call the `lambda` when the correct compile-time `Int`s are given as a @@ -131,7 +131,7 @@ decltype(auto) callFixedSize(std::array ints, auto&& functor, static_assert(NumIntegers > 0); // TODO Use `std::bind_back` auto p = [](int i) { return detail::mapToZeroIfTooLarge(i, MaxValue); }; - std::ranges::transform(ints, ints.begin(), p); + ql::ranges::transform(ints, ints.begin(), p); // The only step that remains is to lift our single runtime `value` which // is in the range `[0, (MaxValue +1)^ NumIntegers]` to a compile-time diff --git a/src/engine/CartesianProductJoin.cpp b/src/engine/CartesianProductJoin.cpp index 401ba6df58..cedb832648 100644 --- a/src/engine/CartesianProductJoin.cpp +++ b/src/engine/CartesianProductJoin.cpp @@ -15,7 +15,7 @@ CartesianProductJoin::CartesianProductJoin( children_{std::move(children)}, chunkSize_{chunkSize} { AD_CONTRACT_CHECK(!children_.empty()); - AD_CONTRACT_CHECK(std::ranges::all_of( + AD_CONTRACT_CHECK(ql::ranges::all_of( children_, [](auto& child) { return child != nullptr; })); // Check that the variables of the passed in operations are in fact @@ -25,13 +25,13 @@ CartesianProductJoin::CartesianProductJoin( // false as soon as a duplicate is encountered. ad_utility::HashSet vars; auto checkVarsForOp = [&vars](const Operation& op) { - return std::ranges::all_of( - op.getExternallyVisibleVariableColumns() | std::views::keys, + return ql::ranges::all_of( + op.getExternallyVisibleVariableColumns() | ql::views::keys, [&vars](const Variable& variable) { return vars.insert(variable).second; }); }; - return std::ranges::all_of(childView(), checkVarsForOp); + return ql::ranges::all_of(childView(), checkVarsForOp); }(); AD_CONTRACT_CHECK(variablesAreDisjoint); } @@ -39,8 +39,8 @@ CartesianProductJoin::CartesianProductJoin( // ____________________________________________________________________________ std::vector CartesianProductJoin::getChildren() { std::vector result; - std::ranges::copy( - children_ | std::views::transform([](auto& ptr) { return ptr.get(); }), + ql::ranges::copy( + children_ | ql::views::transform([](auto& ptr) { return ptr.get(); }), std::back_inserter(result)); return result; } @@ -49,28 +49,28 @@ std::vector CartesianProductJoin::getChildren() { string CartesianProductJoin::getCacheKeyImpl() const { return "CARTESIAN PRODUCT JOIN " + ad_utility::lazyStrJoin( - std::views::transform( + ql::views::transform( childView(), [](auto& child) { return child.getCacheKey(); }), " "); } // ____________________________________________________________________________ size_t CartesianProductJoin::getResultWidth() const { - auto view = childView() | std::views::transform(&Operation::getResultWidth); + auto view = childView() | ql::views::transform(&Operation::getResultWidth); return std::reduce(view.begin(), view.end(), 0UL, std::plus{}); } // ____________________________________________________________________________ size_t CartesianProductJoin::getCostEstimate() { auto childSizes = - childView() | std::views::transform(&Operation::getCostEstimate); + childView() | ql::views::transform(&Operation::getCostEstimate); return getSizeEstimate() + std::reduce(childSizes.begin(), childSizes.end(), 0UL, std::plus{}); } // ____________________________________________________________________________ uint64_t CartesianProductJoin::getSizeEstimateBeforeLimit() { - auto view = childView() | std::views::transform(&Operation::getSizeEstimate); + auto view = childView() | ql::views::transform(&Operation::getSizeEstimate); return std::reduce(view.begin(), view.end(), 1UL, std::multiplies{}); } @@ -86,7 +86,7 @@ float CartesianProductJoin::getMultiplicity([[maybe_unused]] size_t col) { bool CartesianProductJoin::knownEmptyResult() { // If children were empty, returning false would be the wrong behavior. AD_CORRECTNESS_CHECK(!children_.empty()); - return std::ranges::any_of(childView(), &Operation::knownEmptyResult); + return ql::ranges::any_of(childView(), &Operation::knownEmptyResult); } // ____________________________________________________________________________ @@ -138,16 +138,15 @@ ProtoResult CartesianProductJoin::computeResult(bool requestLaziness) { LocalVocab staticMergedVocab{}; staticMergedVocab.mergeWith( subResults | - std::views::transform([](const auto& result) -> const LocalVocab& { + ql::views::transform([](const auto& result) -> const LocalVocab& { return result->localVocab(); })); if (!requestLaziness) { AD_CORRECTNESS_CHECK(!lazyResult); - return { - writeAllColumns(subResults | std::views::transform(&Result::idTable), - getLimit()._offset, getLimit().limitOrDefault()), - resultSortedOn(), std::move(staticMergedVocab)}; + return {writeAllColumns(subResults | ql::views::transform(&Result::idTable), + getLimit()._offset, getLimit().limitOrDefault()), + resultSortedOn(), std::move(staticMergedVocab)}; } if (lazyResult) { @@ -159,7 +158,7 @@ ProtoResult CartesianProductJoin::computeResult(bool requestLaziness) { // Owning view wrapper to please gcc 11. return {produceTablesLazily(std::move(staticMergedVocab), ad_utility::OwningView{std::move(subResults)} | - std::views::transform(&Result::idTable), + ql::views::transform(&Result::idTable), getLimit()._offset, getLimit().limitOrDefault()), resultSortedOn()}; } @@ -192,11 +191,11 @@ IdTable CartesianProductJoin::writeAllColumns( // single result is left. This can probably be done by using the // `ProtoResult`. - auto sizesView = std::views::transform(idTables, &IdTable::size); + auto sizesView = ql::views::transform(idTables, &IdTable::size); auto totalResultSize = std::reduce(sizesView.begin(), sizesView.end(), 1UL, std::multiplies{}); - if (!std::ranges::empty(idTables) && sizesView.back() != 0) { + if (!ql::ranges::empty(idTables) && sizesView.back() != 0) { totalResultSize += (totalResultSize / sizesView.back()) * lastTableOffset; } else { AD_CORRECTNESS_CHECK(lastTableOffset == 0); @@ -254,7 +253,7 @@ CartesianProductJoin::calculateSubResults(bool requestLaziness) { std::shared_ptr lazyResult = nullptr; auto children = childView(); - AD_CORRECTNESS_CHECK(!std::ranges::empty(children)); + AD_CORRECTNESS_CHECK(!ql::ranges::empty(children)); // Get all child results (possibly with limit, see above). for (Operation& child : children) { if (limitIfPresent.has_value() && child.supportsLimit()) { @@ -346,7 +345,7 @@ Result::Generator CartesianProductJoin::createLazyConsumer( size_t producedTableSize = 0; for (auto& idTableAndVocab : produceTablesLazily( std::move(localVocab), - std::views::transform( + ql::views::transform( idTables, [](const auto& wrapper) -> const IdTable& { return wrapper; }), offset, limit, lastTableOffset)) { diff --git a/src/engine/CartesianProductJoin.h b/src/engine/CartesianProductJoin.h index 72f5ff9a12..8c0a071c98 100644 --- a/src/engine/CartesianProductJoin.h +++ b/src/engine/CartesianProductJoin.h @@ -21,15 +21,14 @@ class CartesianProductJoin : public Operation { // TODO We can move this whole children management into a base class // and clean up the implementation of several other children. auto childView() { - return std::views::transform(children_, [](auto& child) -> Operation& { + return ql::views::transform(children_, [](auto& child) -> Operation& { return *child->getRootOperation(); }); } auto childView() const { - return std::views::transform(children_, - [](auto& child) -> const Operation& { - return *child->getRootOperation(); - }); + return ql::views::transform(children_, [](auto& child) -> const Operation& { + return *child->getRootOperation(); + }); } public: diff --git a/src/engine/CheckUsePatternTrick.cpp b/src/engine/CheckUsePatternTrick.cpp index 79ed50969b..866caa2f9b 100644 --- a/src/engine/CheckUsePatternTrick.cpp +++ b/src/engine/CheckUsePatternTrick.cpp @@ -4,10 +4,10 @@ #include "./CheckUsePatternTrick.h" -#include #include #include +#include "backports/algorithm.h" #include "parser/GraphPatternOperation.h" namespace checkUsePatternTrick { @@ -15,7 +15,7 @@ namespace checkUsePatternTrick { bool isVariableContainedInGraphPattern( const Variable& variable, const ParsedQuery::GraphPattern& graphPattern, const SparqlTriple* tripleToIgnore) { - if (std::ranges::any_of( + if (ql::ranges::any_of( graphPattern._filters, [&variable](const SparqlFilter& filter) { return filter.expression_.isVariableContained(variable); })) { @@ -25,7 +25,7 @@ bool isVariableContainedInGraphPattern( return isVariableContainedInGraphPatternOperation(variable, op, tripleToIgnore); }; - return std::ranges::any_of(graphPattern._graphPatterns, check); + return ql::ranges::any_of(graphPattern._graphPatterns, check); } namespace p = parsedQuery; @@ -101,7 +101,7 @@ static void rewriteTriplesForPatternTrick(const PatternTrickTuple& subAndPred, auto findAndRewriteMatchingTriple = [&subAndPred, &triples]( auto triplePosition, size_t additionalScanColumn) { - auto matchingTriple = std::ranges::find_if( + auto matchingTriple = ql::ranges::find_if( triples, [&subAndPred, triplePosition](const SparqlTriple& t) { return std::invoke(triplePosition, t) == subAndPred.subject_ && t.p_.isIri() && !isVariable(t.p_); @@ -231,7 +231,7 @@ std::optional isTripleSuitableForPatternTrick( std::vector variables{triple.s_.getVariable().name(), triple.o_.getVariable().name(), triple.p_.asString()}; - std::ranges::sort(variables); + ql::ranges::sort(variables); if (std::unique(variables.begin(), variables.end()) != variables.end()) { return std::nullopt; } @@ -270,12 +270,12 @@ std::optional isTripleSuitableForPatternTrick( // Check that the pattern trick triple is the only place in the query // where the predicate variable (and the object variable in the three // variables case) occurs. - if (std::ranges::any_of(patternTrickData.variablesNotAllowedInRestOfQuery_, - [&](const Variable& variable) { - return isVariableContainedInGraphPattern( - variable, parsedQuery->_rootGraphPattern, - &triple); - })) { + if (ql::ranges::any_of(patternTrickData.variablesNotAllowedInRestOfQuery_, + [&](const Variable& variable) { + return isVariableContainedInGraphPattern( + variable, parsedQuery->_rootGraphPattern, + &triple); + })) { return std::nullopt; } diff --git a/src/engine/Distinct.cpp b/src/engine/Distinct.cpp index 06b9718540..a3047569a1 100644 --- a/src/engine/Distinct.cpp +++ b/src/engine/Distinct.cpp @@ -84,8 +84,8 @@ ProtoResult Distinct::computeResult(bool requestLaziness) { // _____________________________________________________________________________ bool Distinct::matchesRow(const auto& a, const auto& b) const { - return std::ranges::all_of(keepIndices_, - [&a, &b](ColumnIndex i) { return a[i] == b[i]; }); + return ql::ranges::all_of(keepIndices_, + [&a, &b](ColumnIndex i) { return a[i] == b[i]; }); } // _____________________________________________________________________________ @@ -100,7 +100,7 @@ IdTable Distinct::distinct( // Variant of `std::ranges::unique` that allows to skip the begin rows of // elements found in the previous table. auto begin = - std::ranges::find_if(result, [this, &previousRow](const auto& row) { + ql::ranges::find_if(result, [this, &previousRow](const auto& row) { // Without explicit this clang seems to // think the this capture is redundant. return !previousRow.has_value() || @@ -111,12 +111,12 @@ IdTable Distinct::distinct( auto dest = result.begin(); if (begin == dest) { // Optimization to avoid redundant move operations. - begin = std::ranges::adjacent_find(begin, end, - [this](const auto& a, const auto& b) { - // Without explicit this clang seems to - // think the this capture is redundant. - return this->matchesRow(a, b); - }); + begin = ql::ranges::adjacent_find(begin, end, + [this](const auto& a, const auto& b) { + // Without explicit this clang seems to + // think the this capture is redundant. + return this->matchesRow(a, b); + }); dest = begin; if (begin != end) { ++begin; @@ -154,13 +154,13 @@ IdTable Distinct::outOfPlaceDistinct(const IdTable& dynInput) const { auto end = inputView.end(); while (begin < end) { int64_t allowedOffset = std::min(end - begin, CHUNK_SIZE); - begin = std::ranges::unique_copy(begin, begin + allowedOffset, - std::back_inserter(output), - [this](const auto& a, const auto& b) { - // Without explicit this clang seems to - // think the this capture is redundant. - return this->matchesRow(a, b); - }) + begin = ql::ranges::unique_copy(begin, begin + allowedOffset, + std::back_inserter(output), + [this](const auto& a, const auto& b) { + // Without explicit this clang seems to + // think the this capture is redundant. + return this->matchesRow(a, b); + }) .in; checkCancellation(); // Skip to next unique value @@ -169,12 +169,12 @@ IdTable Distinct::outOfPlaceDistinct(const IdTable& dynInput) const { // This can only be called when dynInput is not empty, so `begin[-1]` is // always valid. auto lastRow = begin[-1]; - begin = std::ranges::find_if(begin, begin + allowedOffset, - [this, &lastRow](const auto& row) { - // Without explicit this clang seems to - // think the this capture is redundant. - return !this->matchesRow(row, lastRow); - }); + begin = ql::ranges::find_if(begin, begin + allowedOffset, + [this, &lastRow](const auto& row) { + // Without explicit this clang seems to + // think the this capture is redundant. + return !this->matchesRow(row, lastRow); + }); checkCancellation(); } while (begin != end && matchesRow(*begin, begin[-1])); } diff --git a/src/engine/Engine.cpp b/src/engine/Engine.cpp index 7bc031f694..a9d8c80529 100644 --- a/src/engine/Engine.cpp +++ b/src/engine/Engine.cpp @@ -55,7 +55,7 @@ void Engine::sort(IdTable& idTable, const std::vector& sortCols) { size_t Engine::countDistinct(IdTableView<0> input, const std::function& checkCancellation) { AD_EXPENSIVE_CHECK( - std::ranges::is_sorted(input, std::ranges::lexicographical_compare), + ql::ranges::is_sorted(input, ql::ranges::lexicographical_compare), "Input to Engine::countDistinct must be sorted"); if (input.empty()) { return 0; diff --git a/src/engine/Engine.h b/src/engine/Engine.h index 47a415674a..577e982f81 100644 --- a/src/engine/Engine.h +++ b/src/engine/Engine.h @@ -3,11 +3,11 @@ // Author: Björn Buchhold (buchhold@informatik.uni-freiburg.de) #pragma once -#include #include #include #include +#include "backports/algorithm.h" #include "engine/IndexSequence.h" #include "engine/idTable/IdTable.h" #include "global/Constants.h" diff --git a/src/engine/ExportQueryExecutionTrees.cpp b/src/engine/ExportQueryExecutionTrees.cpp index 6cea915b55..3375e82924 100644 --- a/src/engine/ExportQueryExecutionTrees.cpp +++ b/src/engine/ExportQueryExecutionTrees.cpp @@ -19,7 +19,7 @@ bool getResultForAsk(const std::shared_ptr& result) { if (result->isFullyMaterialized()) { return !result->idTable().empty(); } else { - return std::ranges::any_of(result->idTables(), [](const auto& pair) { + return ql::ranges::any_of(result->idTables(), [](const auto& pair) { return !pair.idTable_.empty(); }); } @@ -139,7 +139,7 @@ ExportQueryExecutionTrees::getRowIndices(LimitOffsetClause limitOffset, // If there is something to be exported, yield it. if (numRowsToBeExported > 0) { co_yield {std::move(tableWithVocab), - std::views::iota(rangeBegin, rangeBegin + numRowsToBeExported)}; + ql::views::iota(rangeBegin, rangeBegin + numRowsToBeExported)}; } // Add to `resultSize` and update the effective offset (which becomes zero @@ -565,8 +565,8 @@ ExportQueryExecutionTrees::selectQueryResultToStream( selectClause.getSelectedVariablesAsStrings(); // In the CSV format, the variables don't include the question mark. if (format == MediaType::csv) { - std::ranges::for_each(variables, - [](std::string& var) { var = var.substr(1); }); + ql::ranges::for_each(variables, + [](std::string& var) { var = var.substr(1); }); } co_yield absl::StrJoin(variables, std::string_view{&separator, 1}); co_yield '\n'; @@ -688,7 +688,7 @@ ad_utility::streams::stream_generator ExportQueryExecutionTrees:: std::shared_ptr result = qet.getResult(true); // In the XML format, the variables don't include the question mark. - auto varsWithoutQuestionMark = std::views::transform( + auto varsWithoutQuestionMark = ql::views::transform( variables, [](std::string_view var) { return var.substr(1); }); for (std::string_view var : varsWithoutQuestionMark) { co_yield absl::StrCat("\n "sv); @@ -740,7 +740,7 @@ ad_utility::streams::stream_generator ExportQueryExecutionTrees:: qet.selectedVariablesToColumnIndices(selectClause, false); auto vars = selectClause.getSelectedVariablesAsStrings(); - std::ranges::for_each(vars, [](std::string& var) { var = var.substr(1); }); + ql::ranges::for_each(vars, [](std::string& var) { var = var.substr(1); }); nlohmann::json jsonVars = vars; co_yield absl::StrCat(R"({"head":{"vars":)", jsonVars.dump(), R"(},"results":{"bindings":[)"); diff --git a/src/engine/ExportQueryExecutionTrees.h b/src/engine/ExportQueryExecutionTrees.h index a1443e802d..93eb05a5b4 100644 --- a/src/engine/ExportQueryExecutionTrees.h +++ b/src/engine/ExportQueryExecutionTrees.h @@ -164,7 +164,7 @@ class ExportQueryExecutionTrees { // access the `IdTable` with. struct TableWithRange { TableConstRefWithVocab tableWithVocab_; - std::ranges::iota_view view_; + ql::ranges::iota_view view_; }; private: diff --git a/src/engine/Filter.cpp b/src/engine/Filter.cpp index 7e0c66b551..9ecdd85f7a 100644 --- a/src/engine/Filter.cpp +++ b/src/engine/Filter.cpp @@ -6,9 +6,9 @@ #include "./Filter.h" -#include #include +#include "backports/algorithm.h" #include "engine/CallFixedSize.h" #include "engine/QueryExecutionTree.h" #include "engine/sparqlExpressions/SparqlExpression.h" diff --git a/src/engine/GroupBy.cpp b/src/engine/GroupBy.cpp index a2f52e9e60..1ae50b1b79 100644 --- a/src/engine/GroupBy.cpp +++ b/src/engine/GroupBy.cpp @@ -49,7 +49,7 @@ GroupBy::GroupBy(QueryExecutionContext* qec, vector groupByVariables, // NOTE: It is tempting to do the same also for the aliases, but that would // break the case when an alias reuses a variable that was bound by a previous // alias. - std::ranges::sort(_groupByVariables, std::less<>{}, &Variable::name); + ql::ranges::sort(_groupByVariables, std::less<>{}, &Variable::name); auto sortColumns = computeSortColumns(subtree.get()); _subtree = @@ -179,8 +179,8 @@ uint64_t GroupBy::getSizeEstimateBeforeLimit() { // TODO Once we can use `std::views` this can be solved // more elegantly. - float minMultiplicity = std::ranges::min( - _groupByVariables | std::views::transform(varToMultiplicity)); + float minMultiplicity = ql::ranges::min( + _groupByVariables | ql::views::transform(varToMultiplicity)); return _subtree->getSizeEstimate() / minMultiplicity; } @@ -420,7 +420,7 @@ size_t GroupBy::searchBlockBoundaries( for (size_t pos = 0; pos < idTable.size(); pos++) { checkCancellation(); bool rowMatchesCurrentBlock = - std::ranges::all_of(currentGroupBlock, [&](const auto& colIdxAndValue) { + ql::ranges::all_of(currentGroupBlock, [&](const auto& colIdxAndValue) { return idTable(pos, colIdxAndValue.first) == colIdxAndValue.second; }); if (!rowMatchesCurrentBlock) { @@ -735,7 +735,7 @@ std::optional GroupBy::computeGroupByForFullIndexScan() const { } else if (!variableIsBoundInSubtree) { // The variable inside the COUNT() is not part of the input, so it is always // unbound and has a count of 0 in each group. - std::ranges::fill(table.getColumn(1), Id::makeFromInt(0)); + ql::ranges::fill(table.getColumn(1), Id::makeFromInt(0)); } // TODO This optimization should probably also apply if @@ -848,7 +848,7 @@ std::optional GroupBy::computeGroupByForJoinWithFullScan() const { const auto& index = getExecutionContext()->getIndex(); // TODO Simplify the following pattern by using - // `std::views::chunk_by` and implement a lazy version of this view for + // `ql::views::chunkd_by` and implement a lazy version of this view for // input iterators. // Take care of duplicate values in the input. @@ -1021,7 +1021,7 @@ GroupBy::isSupportedAggregate(sparqlExpression::SparqlExpression* expr) { return std::nullopt; // `expr` is not a nested aggregated - if (std::ranges::any_of(expr->children(), [](const auto& ptr) { + if (ql::ranges::any_of(expr->children(), [](const auto& ptr) { return ptr->containsAggregate(); })) { return std::nullopt; @@ -1164,7 +1164,7 @@ void GroupBy::substituteGroupVariable( for (const auto& occurrence : occurrences) { sparqlExpression::VectorWithMemoryLimit values(allocator); values.resize(groupValues.size()); - std::ranges::copy(groupValues, values.begin()); + ql::ranges::copy(groupValues, values.begin()); auto newExpression = std::make_unique( std::move(values)); @@ -1276,7 +1276,7 @@ GroupBy::HashMapAggregationData::getSortedGroupColumns() } // Sort data. - std::ranges::sort(sortedKeys.begin(), sortedKeys.end()); + ql::ranges::sort(sortedKeys.begin(), sortedKeys.end()); // Get data in a column-wise manner. ArrayOrVector> result; @@ -1307,7 +1307,7 @@ void GroupBy::evaluateAlias( // have to be substituted away before evaluation auto substitutions = alias.groupedVariables_; - auto topLevelGroupedVariable = std::ranges::find_if( + auto topLevelGroupedVariable = ql::ranges::find_if( substitutions, [](HashMapGroupedVariableInformation& val) { return std::get_if(&val.occurrences_); }); @@ -1320,13 +1320,13 @@ void GroupBy::evaluateAlias( result->getColumn(topLevelGroupedVariable->resultColumnIndex_) .subspan(evaluationContext._beginIndex, evaluationContext.size()); decltype(auto) outValues = result->getColumn(alias.outCol_); - std::ranges::copy(groupValues, - outValues.begin() + evaluationContext._beginIndex); + ql::ranges::copy(groupValues, + outValues.begin() + evaluationContext._beginIndex); // We also need to store it for possible future use sparqlExpression::VectorWithMemoryLimit values(allocator); values.resize(groupValues.size()); - std::ranges::copy(groupValues, values.begin()); + ql::ranges::copy(groupValues, values.begin()); evaluationContext._previousResultsFromSameGroup.at(alias.outCol_) = sparqlExpression::copyExpressionResult( @@ -1345,8 +1345,8 @@ void GroupBy::evaluateAlias( // Copy to result table decltype(auto) outValues = result->getColumn(alias.outCol_); - std::ranges::copy(aggregateResults, - outValues.begin() + evaluationContext._beginIndex); + ql::ranges::copy(aggregateResults, + outValues.begin() + evaluationContext._beginIndex); // Copy the result so that future aliases may reuse it evaluationContext._previousResultsFromSameGroup.at(alias.outCol_) = @@ -1375,7 +1375,7 @@ void GroupBy::evaluateAlias( // Restore original children. Only necessary when the expression will be // used in the future (not the case for the hash map optimization). - // TODO Use `std::views::zip(info, originalChildren)`. + // TODO Use `ql::views::zip(info, originalChildren)`. for (size_t i = 0; i < info.size(); ++i) { auto& aggregate = info.at(i); auto parentAndIndex = aggregate.parentAndIndex_.value(); @@ -1434,7 +1434,7 @@ IdTable GroupBy::createResultFromHashMap( // Copy grouped by values for (size_t idx = 0; idx < aggregationData.numOfGroupedColumns_; ++idx) { - std::ranges::copy(sortedKeys.at(idx), result.getColumn(idx).begin()); + ql::ranges::copy(sortedKeys.at(idx), result.getColumn(idx).begin()); } // Initialize evaluation context diff --git a/src/engine/HasPredicateScan.cpp b/src/engine/HasPredicateScan.cpp index b01ede635b..9804c226c4 100644 --- a/src/engine/HasPredicateScan.cpp +++ b/src/engine/HasPredicateScan.cpp @@ -347,7 +347,7 @@ void HasPredicateScan::computeFreeO( for (Id patternId : hasPattern.getColumn(0)) { const auto& pattern = patterns[patternId.getInt()]; resultTable->resize(pattern.size()); - std::ranges::copy(pattern, resultTable->getColumn(0).begin()); + ql::ranges::copy(pattern, resultTable->getColumn(0).begin()); } } diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index 5bf47dd4c8..0cd735863d 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -117,10 +117,10 @@ string IndexScan::getCacheKeyImpl() const { if (graphsToFilter_.has_value()) { // The graphs are stored as a hash set, but we need a deterministic order. std::vector graphIdVec; - std::ranges::transform(graphsToFilter_.value(), - std::back_inserter(graphIdVec), - &TripleComponent::toRdfLiteral); - std::ranges::sort(graphIdVec); + ql::ranges::transform(graphsToFilter_.value(), + std::back_inserter(graphIdVec), + &TripleComponent::toRdfLiteral); + ql::ranges::sort(graphIdVec); os << "\nFiltered by Graphs:"; os << absl::StrJoin(graphIdVec, " "); } @@ -146,8 +146,10 @@ size_t IndexScan::getResultWidth() const { // _____________________________________________________________________________ vector IndexScan::resultSortedOn() const { - auto resAsView = ad_utility::integerRange(ColumnIndex{numVariables_}); - std::vector result{resAsView.begin(), resAsView.end()}; + std::vector result; + for (auto i : ad_utility::integerRange(ColumnIndex{numVariables_})) { + result.push_back(i); + } for (size_t i = 0; i < additionalColumns_.size(); ++i) { if (additionalColumns_.at(i) == ADDITIONAL_COLUMN_GRAPH_ID) { result.push_back(numVariables_ + i); @@ -167,7 +169,7 @@ IndexScan::setPrefilterGetUpdatedQueryExecutionTree( } const auto& [sortedVar, colIdx] = optSortedVarColIdxPair.value(); auto it = - std::ranges::find(prefilterVariablePairs, sortedVar, ad_utility::second); + ql::ranges::find(prefilterVariablePairs, sortedVar, ad_utility::second); if (it != prefilterVariablePairs.end()) { return makeCopyWithAddedPrefilters( std::make_pair(it->first->clone(), colIdx)); @@ -190,7 +192,7 @@ VariableToColumnMap IndexScan::computeVariableToColumnMap() const { addCol(ptr->getVariable()); } } - std::ranges::for_each(additionalVariables_, addCol); + ql::ranges::for_each(additionalVariables_, addCol); return variableToColumnMap; } @@ -285,7 +287,7 @@ void IndexScan::determineMultiplicities() { } }(); for ([[maybe_unused]] size_t i : - std::views::iota(multiplicity_.size(), getResultWidth())) { + ql::views::iota(multiplicity_.size(), getResultWidth())) { multiplicity_.emplace_back(1); } AD_CONTRACT_CHECK(multiplicity_.size() == getResultWidth()); @@ -442,7 +444,7 @@ IndexScan::lazyScanForJoinOfTwoScans(const IndexScan& s1, const IndexScan& s2) { // _____________________________________________________________________________ Permutation::IdTableGenerator IndexScan::lazyScanForJoinOfColumnWithScan( std::span joinColumn) const { - AD_EXPENSIVE_CHECK(std::ranges::is_sorted(joinColumn)); + AD_EXPENSIVE_CHECK(ql::ranges::is_sorted(joinColumn)); AD_CORRECTNESS_CHECK(numVariables_ <= 3 && numVariables_ > 0); AD_CONTRACT_CHECK(joinColumn.empty() || !joinColumn[0].isUndefined()); @@ -545,7 +547,7 @@ struct IndexScan::SharedGeneratorState { } auto& idTable = iterator_.value()->idTable_; auto joinColumn = idTable.getColumn(joinColumn_); - AD_EXPENSIVE_CHECK(std::ranges::is_sorted(joinColumn)); + AD_EXPENSIVE_CHECK(ql::ranges::is_sorted(joinColumn)); AD_CORRECTNESS_CHECK(!joinColumn.empty()); // Skip processing for undef case, it will be handled differently if (hasUndef_) { @@ -564,12 +566,12 @@ struct IndexScan::SharedGeneratorState { // matching blocks. auto startIterator = lastBlockIndex_.has_value() - ? std::ranges::upper_bound(newBlocks, lastBlockIndex_.value(), {}, - &CompressedBlockMetadata::blockIndex_) + ? ql::ranges::upper_bound(newBlocks, lastBlockIndex_.value(), {}, + &CompressedBlockMetadata::blockIndex_) : newBlocks.begin(); lastBlockIndex_ = newBlocks.back().blockIndex_; - std::ranges::move(startIterator, newBlocks.end(), - std::back_inserter(pendingBlocks_)); + ql::ranges::move(startIterator, newBlocks.end(), + std::back_inserter(pendingBlocks_)); } } @@ -588,8 +590,8 @@ Result::Generator IndexScan::createPrefilteredJoinSide( std::shared_ptr innerState) { if (innerState->hasUndef()) { AD_CORRECTNESS_CHECK(innerState->prefetchedValues_.empty()); - for (auto& value : std::ranges::subrange{innerState->iterator_.value(), - innerState->generator_.end()}) { + for (auto& value : ql::ranges::subrange{innerState->iterator_.value(), + innerState->generator_.end()}) { co_yield value; } co_return; diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp index 93557d06e5..b7b25a8e74 100644 --- a/src/engine/Join.cpp +++ b/src/engine/Join.cpp @@ -371,10 +371,10 @@ void Join::join(const IdTable& a, const IdTable& b, IdTable* result) const { // The UNDEF values are right at the start, so this calculation works. size_t numUndefA = - std::ranges::upper_bound(joinColumnL, ValueId::makeUndefined()) - + ql::ranges::upper_bound(joinColumnL, ValueId::makeUndefined()) - joinColumnL.begin(); size_t numUndefB = - std::ranges::upper_bound(joinColumnR, ValueId::makeUndefined()) - + ql::ranges::upper_bound(joinColumnR, ValueId::makeUndefined()) - joinColumnR.begin(); std::pair undefRangeA{joinColumnL.begin(), joinColumnL.begin() + numUndefA}; std::pair undefRangeB{joinColumnR.begin(), joinColumnR.begin() + numUndefB}; @@ -389,11 +389,11 @@ void Join::join(const IdTable& a, const IdTable& b, IdTable* result) const { auto inverseAddRow = [&addRow](const auto& rowA, const auto& rowB) { addRow(rowB, rowA); }; - ad_utility::gallopingJoin(joinColumnR, joinColumnL, std::ranges::less{}, + ad_utility::gallopingJoin(joinColumnR, joinColumnL, ql::ranges::less{}, inverseAddRow, {}, cancellationCallback); } else if (b.size() / a.size() > GALLOP_THRESHOLD && numUndefA == 0 && numUndefB == 0) { - ad_utility::gallopingJoin(joinColumnL, joinColumnR, std::ranges::less{}, + ad_utility::gallopingJoin(joinColumnL, joinColumnR, ql::ranges::less{}, addRow, {}, cancellationCallback); } else { auto findSmallerUndefRangeLeft = @@ -414,12 +414,12 @@ void Join::join(const IdTable& a, const IdTable& b, IdTable* result) const { auto numOutOfOrder = [&]() { if (numUndefB == 0 && numUndefA == 0) { return ad_utility::zipperJoinWithUndef( - joinColumnL, joinColumnR, std::ranges::less{}, addRow, + joinColumnL, joinColumnR, ql::ranges::less{}, addRow, ad_utility::noop, ad_utility::noop, {}, cancellationCallback); } else { return ad_utility::zipperJoinWithUndef( - joinColumnL, joinColumnR, std::ranges::less{}, addRow, + joinColumnL, joinColumnR, ql::ranges::less{}, addRow, findSmallerUndefRangeLeft, findSmallerUndefRangeRight, {}, cancellationCallback); } diff --git a/src/engine/LazyGroupBy.h b/src/engine/LazyGroupBy.h index f95915eb09..389cbdab40 100644 --- a/src/engine/LazyGroupBy.h +++ b/src/engine/LazyGroupBy.h @@ -51,9 +51,9 @@ class LazyGroupBy { auto allAggregateInfoView() const { return aggregateAliases_ | - std::views::transform( + ql::views::transform( &GroupBy::HashMapAliasInformation::aggregateInfo_) | - std::views::join; + ql::views::join; } FRIEND_TEST(LazyGroupBy, verifyGroupConcatIsCorrectlyInitialized); diff --git a/src/engine/LocalVocab.cpp b/src/engine/LocalVocab.cpp index 67ee0c36ab..a50e888aa7 100644 --- a/src/engine/LocalVocab.cpp +++ b/src/engine/LocalVocab.cpp @@ -19,7 +19,7 @@ LocalVocab LocalVocab::clone() const { // _____________________________________________________________________________ LocalVocab LocalVocab::merge(std::span vocabs) { LocalVocab result; - result.mergeWith(vocabs | std::views::transform(ad_utility::dereference)); + result.mergeWith(vocabs | ql::views::transform(ad_utility::dereference)); return result; } @@ -67,9 +67,9 @@ const LocalVocabEntry& LocalVocab::getWord( // _____________________________________________________________________________ std::vector LocalVocab::getAllWordsForTesting() const { std::vector result; - std::ranges::copy(primaryWordSet(), std::back_inserter(result)); + ql::ranges::copy(primaryWordSet(), std::back_inserter(result)); for (const auto& previous : otherWordSets_) { - std::ranges::copy(*previous, std::back_inserter(result)); + ql::ranges::copy(*previous, std::back_inserter(result)); } return result; } diff --git a/src/engine/LocalVocab.h b/src/engine/LocalVocab.h index 28d5ab2dac..e8bd3be550 100644 --- a/src/engine/LocalVocab.h +++ b/src/engine/LocalVocab.h @@ -8,7 +8,6 @@ #include #include -#include #include #include #include @@ -16,6 +15,7 @@ #include #include +#include "backports/algorithm.h" #include "index/LocalVocabEntry.h" #include "util/BlankNodeManager.h" #include "util/Exception.h" @@ -115,7 +115,7 @@ class LocalVocab { // primary set of this `LocalVocab` remains unchanged. template void mergeWith(const R& vocabs) { - using std::views::filter; + using ql::views::filter; auto addWordSet = [this](const std::shared_ptr& set) { bool added = otherWordSets_.insert(set).second; size_ += static_cast(added) * set->size(); @@ -125,7 +125,7 @@ class LocalVocab { // typically don't compare equal to each other because of the`shared_ptr` // semantics. for (const auto& vocab : vocabs | filter(std::not_fn(&LocalVocab::empty))) { - std::ranges::for_each(vocab.otherWordSets_, addWordSet); + ql::ranges::for_each(vocab.otherWordSets_, addWordSet); addWordSet(vocab.primaryWordSet_); } @@ -134,12 +134,12 @@ class LocalVocab { ad_utility::BlankNodeManager::LocalBlankNodeManager; auto localManagersView = vocabs | - std::views::transform([](const LocalVocab& vocab) -> const auto& { + ql::views::transform([](const LocalVocab& vocab) -> const auto& { return vocab.localBlankNodeManager_; }); - auto it = std::ranges::find_if(localManagersView, - [](const auto& l) { return l != nullptr; }); + auto it = ql::ranges::find_if(localManagersView, + [](const auto& l) { return l != nullptr; }); if (it == localManagersView.end()) { return; } diff --git a/src/engine/MultiColumnJoin.cpp b/src/engine/MultiColumnJoin.cpp index 0c92e421c9..bb3e4e5995 100644 --- a/src/engine/MultiColumnJoin.cpp +++ b/src/engine/MultiColumnJoin.cpp @@ -260,12 +260,12 @@ void MultiColumnJoin::computeMultiColumnJoin( if (isCheap) { return ad_utility::zipperJoinWithUndef( leftJoinColumns, rightJoinColumns, - std::ranges::lexicographical_compare, addRow, ad_utility::noop, + ql::ranges::lexicographical_compare, addRow, ad_utility::noop, ad_utility::noop, ad_utility::noop, checkCancellationLambda); } else { return ad_utility::zipperJoinWithUndef( leftJoinColumns, rightJoinColumns, - std::ranges::lexicographical_compare, addRow, findUndef, findUndef, + ql::ranges::lexicographical_compare, addRow, findUndef, findUndef, ad_utility::noop, checkCancellationLambda); } }(); diff --git a/src/engine/Operation.cpp b/src/engine/Operation.cpp index 1a9f53fa76..3a25e7521c 100644 --- a/src/engine/Operation.cpp +++ b/src/engine/Operation.cpp @@ -442,7 +442,7 @@ void Operation::updateRuntimeInformationWhenOptimizedOut( // `totalTime_ - #sum of childrens' total time#` in `getOperationTime()`. // To set it to zero we thus have to set the `totalTime_` to that sum. auto timesOfChildren = _runtimeInfo->children_ | - std::views::transform(&RuntimeInformation::totalTime_); + ql::views::transform(&RuntimeInformation::totalTime_); _runtimeInfo->totalTime_ = std::reduce(timesOfChildren.begin(), timesOfChildren.end(), 0us); @@ -575,7 +575,7 @@ std::optional Operation::getPrimarySortKeyVariable() const { return std::nullopt; } - auto it = std::ranges::find( + auto it = ql::ranges::find( varToColMap, sortedIndices.front(), [](const auto& keyValue) { return keyValue.second.columnIndex_; }); if (it == varToColMap.end()) { diff --git a/src/engine/OptionalJoin.cpp b/src/engine/OptionalJoin.cpp index dea5fcc259..8f009d963e 100644 --- a/src/engine/OptionalJoin.cpp +++ b/src/engine/OptionalJoin.cpp @@ -258,7 +258,7 @@ auto OptionalJoin::computeImplementationFromIdTables( -> Implementation { auto implementation = Implementation::NoUndef; auto anyIsUndefined = [](auto column) { - return std::ranges::any_of(column, &Id::isUndefined); + return ql::ranges::any_of(column, &Id::isUndefined); }; for (size_t i = 0; i < joinColumns.size(); ++i) { auto [leftCol, rightCol] = joinColumns.at(i); @@ -308,7 +308,7 @@ void OptionalJoin::optionalJoin( auto rightPermuted = right.asColumnSubsetView(joinColumnData.permutationRight()); - auto lessThanBoth = std::ranges::lexicographical_compare; + auto lessThanBoth = ql::ranges::lexicographical_compare; auto rowAdder = ad_utility::AddCombinedRowToIdTable( joinColumns.size(), leftPermuted, rightPermuted, std::move(*result), diff --git a/src/engine/OrderBy.cpp b/src/engine/OrderBy.cpp index 8ee3a9c687..5d999e62bc 100644 --- a/src/engine/OrderBy.cpp +++ b/src/engine/OrderBy.cpp @@ -25,7 +25,7 @@ OrderBy::OrderBy(QueryExecutionContext* qec, subtree_{std::move(subtree)}, sortIndices_{std::move(sortIndices)} { AD_CONTRACT_CHECK(!sortIndices_.empty()); - AD_CONTRACT_CHECK(std::ranges::all_of( + AD_CONTRACT_CHECK(ql::ranges::all_of( sortIndices_, [this](ColumnIndex index) { return index < getResultWidth(); }, ad_utility::first)); diff --git a/src/engine/PathSearch.cpp b/src/engine/PathSearch.cpp index 9291fba19a..0a60197341 100644 --- a/src/engine/PathSearch.cpp +++ b/src/engine/PathSearch.cpp @@ -4,7 +4,6 @@ #include "PathSearch.h" -#include #include #include #include @@ -13,6 +12,7 @@ #include #include +#include "backports/algorithm.h" #include "engine/CallFixedSize.h" #include "engine/QueryExecutionTree.h" #include "engine/VariableToColumnMap.h" @@ -31,7 +31,7 @@ BinSearchWrapper::BinSearchWrapper(const IdTable& table, size_t startCol, // _____________________________________________________________________________ std::vector BinSearchWrapper::outgoingEdes(const Id node) const { auto startIds = table_.getColumn(startCol_); - auto range = std::ranges::equal_range(startIds, node); + auto range = ql::ranges::equal_range(startIds, node); auto startIndex = std::distance(startIds.begin(), range.begin()); std::vector edges; @@ -47,7 +47,7 @@ std::vector BinSearchWrapper::outgoingEdes(const Id node) const { std::vector BinSearchWrapper::getSources() const { auto startIds = table_.getColumn(startCol_); std::vector sources; - std::ranges::unique_copy(startIds, std::back_inserter(sources)); + ql::ranges::unique_copy(startIds, std::back_inserter(sources)); return sources; } diff --git a/src/engine/QueryExecutionTree.cpp b/src/engine/QueryExecutionTree.cpp index c9496fe958..7f22de2020 100644 --- a/src/engine/QueryExecutionTree.cpp +++ b/src/engine/QueryExecutionTree.cpp @@ -182,7 +182,7 @@ std::vector> QueryExecutionTree::getJoinColumns( } } - std::ranges::sort(jcs, std::ranges::lexicographical_compare); + ql::ranges::sort(jcs, ql::ranges::lexicographical_compare); return jcs; } @@ -219,7 +219,7 @@ auto QueryExecutionTree::getSortedSubtreesAndJoinColumns( const VariableToColumnMap::value_type& QueryExecutionTree::getVariableAndInfoByColumnIndex(ColumnIndex colIdx) const { const auto& varColMap = getVariableColumns(); - auto it = std::ranges::find_if(varColMap, [leftCol = colIdx](const auto& el) { + auto it = ql::ranges::find_if(varColMap, [leftCol = colIdx](const auto& el) { return el.second.columnIndex_ == leftCol; }); AD_CONTRACT_CHECK(it != varColMap.end()); diff --git a/src/engine/QueryPlanner.cpp b/src/engine/QueryPlanner.cpp index a01aaaece5..9e0cd56083 100644 --- a/src/engine/QueryPlanner.cpp +++ b/src/engine/QueryPlanner.cpp @@ -8,12 +8,12 @@ #include -#include #include #include #include #include +#include "backports/algorithm.h" #include "engine/Bind.h" #include "engine/CartesianProductJoin.h" #include "engine/CheckUsePatternTrick.h" @@ -131,7 +131,7 @@ std::vector QueryPlanner::createExecutionTrees( // this is handled correctly in all cases. bool doGroupBy = !pq._groupByVariables.empty() || patternTrickTuple.has_value() || - std::ranges::any_of(pq.getAliases(), [](const Alias& alias) { + ql::ranges::any_of(pq.getAliases(), [](const Alias& alias) { return alias._expression.containsAggregate(); }); @@ -786,32 +786,31 @@ auto QueryPlanner::seedWithScansAndText( continue; } - auto addIndexScan = [this, pushPlan, node, - &relevantGraphs = - activeDatasetClauses_.defaultGraphs_]( - Permutation::Enum permutation, - std::optional triple = - std::nullopt) { - if (!triple.has_value()) { - triple = node.triple_.getSimple(); - } + auto addIndexScan = + [this, pushPlan, node, + &relevantGraphs = activeDatasetClauses_.defaultGraphs_]( + Permutation::Enum permutation, + std::optional triple = std::nullopt) { + if (!triple.has_value()) { + triple = node.triple_.getSimple(); + } - // We are inside a `GRAPH ?var {...}` clause, so all index scans have - // to add the graph variable as an additional column. - auto& additionalColumns = triple.value().additionalScanColumns_; - AD_CORRECTNESS_CHECK(!ad_utility::contains( - additionalColumns | std::views::keys, ADDITIONAL_COLUMN_GRAPH_ID)); - if (activeGraphVariable_.has_value()) { - additionalColumns.emplace_back(ADDITIONAL_COLUMN_GRAPH_ID, - activeGraphVariable_.value()); - } + // We are inside a `GRAPH ?var {...}` clause, so all index scans have + // to add the graph variable as an additional column. + auto& additionalColumns = triple.value().additionalScanColumns_; + AD_CORRECTNESS_CHECK(!ad_utility::contains( + additionalColumns | ql::views::keys, ADDITIONAL_COLUMN_GRAPH_ID)); + if (activeGraphVariable_.has_value()) { + additionalColumns.emplace_back(ADDITIONAL_COLUMN_GRAPH_ID, + activeGraphVariable_.value()); + } - // TODO Handle the case, that the Graph variable is also used - // inside the `GRAPH` clause, e.g. by being used inside a triple. + // TODO Handle the case, that the Graph variable is also used + // inside the `GRAPH` clause, e.g. by being used inside a triple. - pushPlan(makeSubtreePlan( - _qec, permutation, std::move(triple.value()), relevantGraphs)); - }; + pushPlan(makeSubtreePlan( + _qec, permutation, std::move(triple.value()), relevantGraphs)); + }; auto addFilter = [&filters = result.filters_](SparqlFilter filter) { filters.push_back(std::move(filter)); @@ -1208,10 +1207,10 @@ void QueryPlanner::applyFiltersIfPossible( continue; } - if (std::ranges::all_of(filters[i].expression_.containedVariables(), - [&plan](const auto& variable) { - return plan._qet->isVariableCovered(*variable); - })) { + if (ql::ranges::all_of(filters[i].expression_.containedVariables(), + [&plan](const auto& variable) { + return plan._qet->isVariableCovered(*variable); + })) { // Apply this filter. SubtreePlan newPlan = makeSubtreePlan(_qec, plan._qet, filters[i].expression_); @@ -1294,12 +1293,12 @@ size_t QueryPlanner::findUniqueNodeIds( const std::vector& connectedComponent) { ad_utility::HashSet uniqueNodeIds; auto nodeIds = connectedComponent | - std::views::transform(&SubtreePlan::_idsOfIncludedNodes); + ql::views::transform(&SubtreePlan::_idsOfIncludedNodes); // Check that all the `_idsOfIncludedNodes` are one-hot encodings of a single // value, i.e. they have exactly one bit set. - AD_CORRECTNESS_CHECK(std::ranges::all_of( + AD_CORRECTNESS_CHECK(ql::ranges::all_of( nodeIds, [](auto nodeId) { return std::popcount(nodeId) == 1; })); - std::ranges::copy(nodeIds, std::inserter(uniqueNodeIds, uniqueNodeIds.end())); + ql::ranges::copy(nodeIds, std::inserter(uniqueNodeIds, uniqueNodeIds.end())); return uniqueNodeIds.size(); } @@ -1341,10 +1340,9 @@ size_t QueryPlanner::countSubgraphs( std::vector graph, size_t budget) { // Remove duplicate plans from `graph`. auto getId = [](const SubtreePlan* v) { return v->_idsOfIncludedNodes; }; - std::ranges::sort(graph, std::ranges::less{}, getId); - graph.erase( - std::ranges::unique(graph, std::ranges::equal_to{}, getId).begin(), - graph.end()); + ql::ranges::sort(graph, ql::ranges::less{}, getId); + graph.erase(std::ranges::unique(graph, ql::ranges::equal_to{}, getId).begin(), + graph.end()); // Qlever currently limits the number of triples etc. per group to be <= 64 // anyway, so we can simply assert here. @@ -1409,7 +1407,7 @@ vector> QueryPlanner::fillDpTab( const vector>& children) { auto [initialPlans, additionalFilters] = seedWithScansAndText(tg, children, textLimits); - std::ranges::move(additionalFilters, std::back_inserter(filters)); + ql::ranges::move(additionalFilters, std::back_inserter(filters)); if (filters.size() > 64) { AD_THROW("At most 64 filters allowed at the moment."); } @@ -1419,7 +1417,7 @@ vector> QueryPlanner::fillDpTab( components[componentIndices.at(i)].push_back(std::move(initialPlans.at(i))); } vector> lastDpRowFromComponents; - for (auto& component : components | std::views::values) { + for (auto& component : components | ql::views::values) { std::vector g; for (const auto& plan : component) { g.push_back(&plan); @@ -1461,9 +1459,9 @@ vector> QueryPlanner::fillDpTab( uint64_t nodes = 0; uint64_t filterIds = 0; uint64_t textLimitIds = 0; - std::ranges::for_each( + ql::ranges::for_each( lastDpRowFromComponents | - std::views::transform([this](auto& vec) -> decltype(auto) { + ql::views::transform([this](auto& vec) -> decltype(auto) { return vec.at(findCheapestExecutionTree(vec)); }), [&](SubtreePlan& plan) { @@ -1603,7 +1601,7 @@ vector QueryPlanner::TripleGraph::pickFilters( coveredVariables.insert(node._variables.begin(), node._variables.end()); } for (auto& f : origFilters) { - if (std::ranges::any_of( + if (ql::ranges::any_of( f.expression_.containedVariables(), [&](const auto* var) { return coveredVariables.contains(*var); })) { ret.push_back(f); @@ -1775,7 +1773,7 @@ size_t QueryPlanner::findCheapestExecutionTree( return aCost < bCost; } }; - return std::ranges::min_element(lastRow, compare) - lastRow.begin(); + return ql::ranges::min_element(lastRow, compare) - lastRow.begin(); }; // _________________________________________________________________________________ @@ -1788,7 +1786,7 @@ size_t QueryPlanner::findSmallestExecutionTree( }; return tie(a) < tie(b); }; - return std::ranges::min_element(lastRow, compare) - lastRow.begin(); + return ql::ranges::min_element(lastRow, compare) - lastRow.begin(); }; // _____________________________________________________________________________ @@ -2138,7 +2136,7 @@ void QueryPlanner::QueryGraph::setupGraph( ad_utility::HashMap> result; for (const auto& node : nodes_) { for (const auto& var : - node->plan_->_qet->getVariableColumns() | std::views::keys) { + node->plan_->_qet->getVariableColumns() | ql::views::keys) { result[var].push_back(node.get()); } } @@ -2150,8 +2148,8 @@ void QueryPlanner::QueryGraph::setupGraph( ad_utility::HashMap> adjacentNodes = [&varToNode]() { ad_utility::HashMap> result; - for (auto& nodesThatContainSameVar : varToNode | std::views::values) { - // TODO Use std::views::cartesian_product + for (auto& nodesThatContainSameVar : varToNode | ql::views::values) { + // TODO Use ql::views::cartesian_product for (auto* n1 : nodesThatContainSameVar) { for (auto* n2 : nodesThatContainSameVar) { if (n1 != n2) { @@ -2216,12 +2214,12 @@ void QueryPlanner::GraphPatternPlanner::visitGroupOptionalOrMinus( // Optionals that occur before any of their variables have been bound, // actually behave like ordinary (Group)GraphPatterns. - auto variables = candidates[0]._qet->getVariableColumns() | std::views::keys; + auto variables = candidates[0]._qet->getVariableColumns() | ql::views::keys; using enum SubtreePlan::Type; if (auto type = candidates[0].type; (type == OPTIONAL || type == MINUS) && - std::ranges::all_of(variables, [this](const Variable& var) { + ql::ranges::all_of(variables, [this](const Variable& var) { return !boundVariables_.contains(var); })) { // A MINUS clause that doesn't share any variable with the preceding @@ -2240,7 +2238,7 @@ void QueryPlanner::GraphPatternPlanner::visitGroupOptionalOrMinus( // All variables seen so far are considered bound and cannot appear as the // RHS of a BIND operation. This is also true for variables from OPTIONALs // and MINUS clauses (this used to be a bug in an old version of the code). - std::ranges::for_each( + ql::ranges::for_each( variables, [this](const Variable& var) { boundVariables_.insert(var); }); // If our input is not OPTIONAL and not a MINUS, this means that we can still @@ -2568,9 +2566,9 @@ void QueryPlanner::GraphPatternPlanner::visitSubquery( plan._qet->getRootOperation()->setSelectedVariablesForSubquery( selectedVariables); }; - std::ranges::for_each(candidatesForSubquery, setSelectedVariables); + ql::ranges::for_each(candidatesForSubquery, setSelectedVariables); // A subquery must also respect LIMIT and OFFSET clauses - std::ranges::for_each(candidatesForSubquery, [&](SubtreePlan& plan) { + ql::ranges::for_each(candidatesForSubquery, [&](SubtreePlan& plan) { plan._qet->getRootOperation()->setLimit(arg.get()._limitOffset); }); visitGroupOptionalOrMinus(std::move(candidatesForSubquery)); diff --git a/src/engine/Result.cpp b/src/engine/Result.cpp index bc731e2433..a1cfa10583 100644 --- a/src/engine/Result.cpp +++ b/src/engine/Result.cpp @@ -92,7 +92,7 @@ Result::Result(Generator idTables, std::vector sortedBy) // _____________________________________________________________________________ // Apply `LimitOffsetClause` to given `IdTable`. void resizeIdTable(IdTable& idTable, const LimitOffsetClause& limitOffset) { - std::ranges::for_each( + ql::ranges::for_each( idTable.getColumns(), [offset = limitOffset.actualOffset(idTable.numRows()), upperBound = @@ -178,10 +178,10 @@ void Result::assertThatLimitWasRespected(const LimitOffsetClause& limitOffset) { // _____________________________________________________________________________ void Result::checkDefinedness(const VariableToColumnMap& varColMap) { auto performCheck = [](const auto& map, IdTable& idTable) { - return std::ranges::all_of(map, [&](const auto& varAndCol) { + return ql::ranges::all_of(map, [&](const auto& varAndCol) { const auto& [columnIndex, mightContainUndef] = varAndCol.second; if (mightContainUndef == ColumnIndexAndTypeInfo::AlwaysDefined) { - return std::ranges::all_of(idTable.getColumn(columnIndex), [](Id id) { + return ql::ranges::all_of(idTable.getColumn(columnIndex), [](Id id) { return id.getDatatype() != Datatype::Undefined; }); } @@ -239,12 +239,12 @@ void Result::runOnNewChunkComputed( void Result::assertSortOrderIsRespected( const IdTable& idTable, const std::vector& sortedBy) { AD_CONTRACT_CHECK( - std::ranges::all_of(sortedBy, [&idTable](ColumnIndex colIndex) { + ql::ranges::all_of(sortedBy, [&idTable](ColumnIndex colIndex) { return colIndex < idTable.numColumns(); })); AD_EXPENSIVE_CHECK( - std::ranges::is_sorted(idTable, compareRowsBySortColumns(sortedBy))); + ql::ranges::is_sorted(idTable, compareRowsBySortColumns(sortedBy))); } // _____________________________________________________________________________ diff --git a/src/engine/RuntimeInformation.cpp b/src/engine/RuntimeInformation.cpp index 2e9abd05c1..f9f2d851c8 100644 --- a/src/engine/RuntimeInformation.cpp +++ b/src/engine/RuntimeInformation.cpp @@ -109,9 +109,9 @@ void RuntimeInformation::setColumnNames(const VariableToColumnMap& columnMap) { // Resize the `columnNames_` vector such that we can use the keys from // columnMap (which are not necessarily consecutive) as indexes. - ColumnIndex maxColumnIndex = std::ranges::max( - columnMap | std::views::values | - std::views::transform(&ColumnIndexAndTypeInfo::columnIndex_)); + ColumnIndex maxColumnIndex = ql::ranges::max( + columnMap | ql::views::values | + ql::views::transform(&ColumnIndexAndTypeInfo::columnIndex_)); columnNames_.resize(maxColumnIndex + 1); // Now copy the `variable, index` pairs from the map to the vector. If the @@ -145,7 +145,7 @@ std::chrono::microseconds RuntimeInformation::getOperationTime() const { // computing that child is *not* included in this operation's // `totalTime_`. That's why we skip such children in the following loop. auto timesOfChildren = - children_ | std::views::transform(&RuntimeInformation::totalTime_); + children_ | ql::views::transform(&RuntimeInformation::totalTime_); // Prevent "negative" computation times in case totalTime_ was not // computed for this yet. return std::max(0us, totalTime_ - std::reduce(timesOfChildren.begin(), diff --git a/src/engine/Service.cpp b/src/engine/Service.cpp index 8c946a2fb3..81df6be64c 100644 --- a/src/engine/Service.cpp +++ b/src/engine/Service.cpp @@ -175,9 +175,9 @@ ProtoResult Service::computeResultImpl([[maybe_unused]] bool requestLaziness) { // for the variables sent in the response as they're maybe not read before // the bindings. std::vector expVariableKeys; - std::ranges::transform(parsedServiceClause_.visibleVariables_, - std::back_inserter(expVariableKeys), - [](const Variable& v) { return v.name().substr(1); }); + ql::ranges::transform(parsedServiceClause_.visibleVariables_, + std::back_inserter(expVariableKeys), + [](const Variable& v) { return v.name().substr(1); }); auto body = ad_utility::LazyJsonParser::parse(std::move(response.body_), {"results", "bindings"}); @@ -569,7 +569,7 @@ void Service::precomputeSiblingResult(std::shared_ptr left, for (auto& pair : pairs) { co_yield pair; } - for (auto& pair : std::ranges::subrange{it, prevGenerator.end()}) { + for (auto& pair : ql::ranges::subrange{it, prevGenerator.end()}) { co_yield pair; } }; diff --git a/src/engine/SpatialJoinAlgorithms.cpp b/src/engine/SpatialJoinAlgorithms.cpp index f95efa7154..82d5102df6 100644 --- a/src/engine/SpatialJoinAlgorithms.cpp +++ b/src/engine/SpatialJoinAlgorithms.cpp @@ -407,7 +407,7 @@ bool SpatialJoinAlgorithms::isContainedInBoundingBoxes( const std::vector& boundingBox, Point point) const { convertToNormalCoordinates(point); - return std::ranges::any_of(boundingBox, [point](const Box& aBox) { + return ql::ranges::any_of(boundingBox, [point](const Box& aBox) { return boost::geometry::covered_by(point, aBox); }); } @@ -512,11 +512,11 @@ Result SpatialJoinAlgorithms::BoundingBoxAlgorithm() { std::vector bbox = computeBoundingBox(p); results.clear(); - std::ranges::for_each(bbox, [&](const Box& bbox) { + ql::ranges::for_each(bbox, [&](const Box& bbox) { rtree.query(bgi::intersects(bbox), std::back_inserter(results)); }); - std::ranges::for_each(results, [&](const Value& res) { + ql::ranges::for_each(results, [&](const Value& res) { size_t rowLeft = res.second; size_t rowRight = i; if (!leftResSmaller) { diff --git a/src/engine/TextIndexScanForWord.cpp b/src/engine/TextIndexScanForWord.cpp index 6e141ad518..612780e28b 100644 --- a/src/engine/TextIndexScanForWord.cpp +++ b/src/engine/TextIndexScanForWord.cpp @@ -22,7 +22,7 @@ ProtoResult TextIndexScanForWord::computeResult( IdTable smallIdTable{getExecutionContext()->getAllocator()}; smallIdTable.setNumColumns(1); smallIdTable.resize(idTable.numRows()); - std::ranges::copy(idTable.getColumn(0), smallIdTable.getColumn(0).begin()); + ql::ranges::copy(idTable.getColumn(0), smallIdTable.getColumn(0).begin()); return {std::move(smallIdTable), resultSortedOn(), LocalVocab{}}; } diff --git a/src/engine/TextLimit.cpp b/src/engine/TextLimit.cpp index 4125f676cd..6ec7b7868f 100644 --- a/src/engine/TextLimit.cpp +++ b/src/engine/TextLimit.cpp @@ -34,11 +34,11 @@ ProtoResult TextLimit::computeResult([[maybe_unused]] bool requestLaziness) { auto compareScores = [this](const auto& lhs, const auto& rhs) { size_t lhsScore = 0; size_t rhsScore = 0; - std::ranges::for_each(scoreColumns_, - [&lhs, &rhs, &lhsScore, &rhsScore](const auto& col) { - lhsScore += lhs[col].getInt(); - rhsScore += rhs[col].getInt(); - }); + ql::ranges::for_each(scoreColumns_, + [&lhs, &rhs, &lhsScore, &rhsScore](const auto& col) { + lhsScore += lhs[col].getInt(); + rhsScore += rhs[col].getInt(); + }); if (lhsScore > rhsScore) { return 1; } else if (lhsScore < rhsScore) { @@ -49,7 +49,7 @@ ProtoResult TextLimit::computeResult([[maybe_unused]] bool requestLaziness) { auto compareEntities = [this](const auto& lhs, const auto& rhs) { auto it = - std::ranges::find_if(entityColumns_, [&lhs, &rhs](const auto& col) { + ql::ranges::find_if(entityColumns_, [&lhs, &rhs](const auto& col) { return lhs[col] < rhs[col] || lhs[col] > rhs[col]; }); @@ -64,8 +64,8 @@ ProtoResult TextLimit::computeResult([[maybe_unused]] bool requestLaziness) { return 0; }; - std::ranges::sort(idTable, [this, compareScores, compareEntities]( - const auto& lhs, const auto& rhs) { + ql::ranges::sort(idTable, [this, compareScores, compareEntities]( + const auto& lhs, const auto& rhs) { return compareEntities(lhs, rhs) == 1 || (compareEntities(lhs, rhs) == 0 && (compareScores(lhs, rhs) == 1 || diff --git a/src/engine/TransitivePathBase.cpp b/src/engine/TransitivePathBase.cpp index 1db8a7eb0d..3d6ec38262 100644 --- a/src/engine/TransitivePathBase.cpp +++ b/src/engine/TransitivePathBase.cpp @@ -363,7 +363,7 @@ std::shared_ptr TransitivePathBase::bindLeftOrRightSide( maxDist_)); } - auto& p = *std::ranges::min_element( + auto& p = *ql::ranges::min_element( candidates, {}, [](const auto& tree) { return tree->getCostEstimate(); }); // Note: The `variable` in the following structured binding is `const`, even diff --git a/src/engine/TransitivePathBinSearch.h b/src/engine/TransitivePathBinSearch.h index 7928cba9c1..4973d9da5e 100644 --- a/src/engine/TransitivePathBinSearch.h +++ b/src/engine/TransitivePathBinSearch.h @@ -47,7 +47,7 @@ struct BinSearchMap { * startIds_ == node. */ auto successors(const Id node) const { - auto range = std::ranges::equal_range(startIds_, node); + auto range = ql::ranges::equal_range(startIds_, node); auto startIndex = std::distance(startIds_.begin(), range.begin()); diff --git a/src/engine/Union.cpp b/src/engine/Union.cpp index 0fac7b31f3..46e5d06f4a 100644 --- a/src/engine/Union.cpp +++ b/src/engine/Union.cpp @@ -42,7 +42,7 @@ Union::Union(QueryExecutionContext* qec, _columnOrigins[it.second.columnIndex_][1] = NO_COLUMN; } } - AD_CORRECTNESS_CHECK(std::ranges::all_of(_columnOrigins, [](const auto& el) { + AD_CORRECTNESS_CHECK(ql::ranges::all_of(_columnOrigins, [](const auto& el) { return el[0] != NO_COLUMN || el[1] != NO_COLUMN; })); } @@ -75,7 +75,7 @@ VariableToColumnMap Union::computeVariableToColumnMap() const { // A variable is only guaranteed to always be bound if it exists in all the // subtrees and if it is guaranteed to be bound in all the subtrees. auto mightContainUndef = [this](const Variable& var) { - return std::ranges::any_of( + return ql::ranges::any_of( _subtrees, [&](const std::shared_ptr& subtree) { const auto& varCols = subtree->getVariableColumns(); return !varCols.contains(var) || @@ -86,7 +86,7 @@ VariableToColumnMap Union::computeVariableToColumnMap() const { // Note: it is tempting to declare `nextColumnIndex` inside the lambda // `addVariableColumnIfNotExists`, but that doesn't work because - // `std::ranges::for_each` takes the lambda by value and creates a new + // `ql::ranges::for_each` takes the lambda by value and creates a new // variable at every invocation. size_t nextColumnIndex = 0; auto addVariableColumnIfNotExists = @@ -102,14 +102,13 @@ VariableToColumnMap Union::computeVariableToColumnMap() const { } }; - auto addVariablesForSubtree = - [&addVariableColumnIfNotExists](const auto& subtree) { - std::ranges::for_each( - copySortedByColumnIndex(subtree->getVariableColumns()), - addVariableColumnIfNotExists); - }; + auto addVariablesForSubtree = [&addVariableColumnIfNotExists]( + const auto& subtree) { + ql::ranges::for_each(copySortedByColumnIndex(subtree->getVariableColumns()), + addVariableColumnIfNotExists); + }; - std::ranges::for_each(_subtrees, addVariablesForSubtree); + ql::ranges::for_each(_subtrees, addVariablesForSubtree); return variableColumns; } @@ -203,7 +202,7 @@ IdTable Union::computeUnion( [this]() { checkCancellation(); }); } else { ad_utility::chunkedFill( - std::ranges::subrange{ + ql::ranges::subrange{ targetColumn.begin() + offset, targetColumn.begin() + offset + inputTable.size()}, Id::makeUndefined(), chunkSize, [this]() { checkCancellation(); }); diff --git a/src/engine/Values.cpp b/src/engine/Values.cpp index 181f2f4207..7a4535ceed 100644 --- a/src/engine/Values.cpp +++ b/src/engine/Values.cpp @@ -18,7 +18,7 @@ Values::Values(QueryExecutionContext* qec, SparqlValues parsedValues) : Operation(qec), parsedValues_(std::move(parsedValues)) { AD_CONTRACT_CHECK( - std::ranges::all_of(parsedValues_._values, [&](const auto& row) { + ql::ranges::all_of(parsedValues_._values, [&](const auto& row) { return row.size() == parsedValues_._variables.size(); })); } diff --git a/src/engine/VariableToColumnMap.cpp b/src/engine/VariableToColumnMap.cpp index 00eeb64f47..ff7f221a2b 100644 --- a/src/engine/VariableToColumnMap.cpp +++ b/src/engine/VariableToColumnMap.cpp @@ -11,8 +11,8 @@ std::vector> copySortedByColumnIndex(VariableToColumnMap map) { std::vector> result{ std::make_move_iterator(map.begin()), std::make_move_iterator(map.end())}; - std::ranges::sort(result, std::less<>{}, - [](const auto& pair) { return pair.second.columnIndex_; }); + ql::ranges::sort(result, std::less<>{}, + [](const auto& pair) { return pair.second.columnIndex_; }); return result; } @@ -34,7 +34,7 @@ VariableToColumnMap makeVarToColMapForJoinOperation( const auto& colIdxRight = columnIndexWithType.columnIndex_; // Figure out if the column (from the right operand) is a join column. auto joinColumnIt = - std::ranges::find(joinColumns, colIdxRight, ad_utility::second); + ql::ranges::find(joinColumns, colIdxRight, ad_utility::second); if (joinColumnIt != joinColumns.end()) { // For non-optional joins, a join column is `AlwaysDefined` if it is // always defined in ANY of the inputs. For optional joins a join column diff --git a/src/engine/VariableToColumnMap.h b/src/engine/VariableToColumnMap.h index 7b31b5222f..e0ad443baf 100644 --- a/src/engine/VariableToColumnMap.h +++ b/src/engine/VariableToColumnMap.h @@ -8,6 +8,10 @@ #include "parser/data/Variable.h" #include "util/HashMap.h" +// TODO We have a cyclic dependency between `Id.h` and +// `VariableToColumnMap.h`. +using ColumnIndex = uint64_t; + // Store an index of a column together with additional information about that // column which can be inferred from the `QueryExecutionTree` without actually // computing the result. diff --git a/src/engine/idTable/CompressedExternalIdTable.h b/src/engine/idTable/CompressedExternalIdTable.h index 962d18b6aa..12bc406e16 100644 --- a/src/engine/idTable/CompressedExternalIdTable.h +++ b/src/engine/idTable/CompressedExternalIdTable.h @@ -7,10 +7,10 @@ #include -#include #include #include +#include "backports/algorithm.h" #include "engine/CallFixedSize.h" #include "engine/idTable/IdTable.h" #include "util/AsyncStream.h" @@ -116,12 +116,12 @@ class CompressedExternalIdTableWriter { // fine-grained) but only once we have a reasonable abstraction for // parallelism. std::vector> compressColumFutures; - for (auto i : std::views::iota(0u, numColumns())) { + for (auto i : ql::views::iota(0u, numColumns())) { compressColumFutures.push_back( std::async(std::launch::async, [this, i, blockSize, &table]() { auto& blockMetadata = blocksPerColumn_.at(i); decltype(auto) column = table.getColumn(i); - // TODO Use `std::views::chunk` + // TODO Use `ql::views::chunkd` for (size_t lower = 0; lower < column.size(); lower += blockSize) { size_t upper = std::min(lower + blockSize, column.size()); auto thisBlockSizeUncompressed = (upper - lower) * sizeof(Id); @@ -151,7 +151,7 @@ class CompressedExternalIdTableWriter { file_.wlock()->flush(); std::vector>> result; result.reserve(startOfSingleIdTables_.size()); - for (auto i : std::views::iota(0u, startOfSingleIdTables_.size())) { + for (auto i : ql::views::iota(0u, startOfSingleIdTables_.size())) { result.push_back(makeGeneratorForIdTable(i)); } return result; @@ -164,7 +164,7 @@ class CompressedExternalIdTableWriter { file_.wlock()->flush(); std::vector(0))> result; result.reserve(startOfSingleIdTables_.size()); - for (auto i : std::views::iota(0u, startOfSingleIdTables_.size())) { + for (auto i : ql::views::iota(0u, startOfSingleIdTables_.size())) { result.push_back(makeGeneratorForRows(i)); } return result; @@ -173,8 +173,9 @@ class CompressedExternalIdTableWriter { template auto getGeneratorForAllRows() { // Note: As soon as we drop the support for GCC11 this can be - // `return getAllRowGenerators() | std::views::join; - return std::views::join(ad_utility::OwningView{getAllRowGenerators()}); + // `return getAllRowGenerators() | ql::views::join; + return ql::views::join( + ad_utility::OwningViewNoConst{getAllRowGenerators()}); } // Clear the underlying file and completely reset the data structure s.t. it @@ -189,7 +190,7 @@ class CompressedExternalIdTableWriter { file_.wlock()->close(); ad_utility::deleteFile(filename_); file_.wlock()->open(filename_, "w+"); - std::ranges::for_each(blocksPerColumn_, [](auto& block) { block.clear(); }); + ql::ranges::for_each(blocksPerColumn_, [](auto& block) { block.clear(); }); startOfSingleIdTables_.clear(); } @@ -197,7 +198,7 @@ class CompressedExternalIdTableWriter { // Get the row generator for a single IdTable, specified by the `index`. template auto makeGeneratorForRows(size_t index) { - return std::views::join( + return ql::views::join( ad_utility::OwningView{makeGeneratorForIdTable(index)}); } // Get the block generator for a single IdTable, specified by the `index`. @@ -255,7 +256,7 @@ class CompressedExternalIdTableWriter { blocksPerColumn_.at(0).at(blockIdx).uncompressedSize_ / sizeof(Id); block.resize(blockSize); std::vector> readColumnFutures; - for (auto i : std::views::iota(0u, numColumns())) { + for (auto i : ql::views::iota(0u, numColumns())) { readColumnFutures.push_back( std::async(std::launch::async, [&block, this, i, blockIdx]() { decltype(auto) col = block.getColumn(i); @@ -478,10 +479,10 @@ class CompressedExternalIdTable co_yield block; }(this->currentBlock_); auto rowView = - std::views::join(ad_utility::OwningView{std::move(generator)}); + ql::views::join(ad_utility::OwningView{std::move(generator)}); std::vector vec; vec.push_back(std::move(rowView)); - return std::views::join(ad_utility::OwningView(std::move(vec))); + return ql::views::join(ad_utility::OwningViewNoConst(std::move(vec))); } this->pushBlock(std::move(this->currentBlock_)); this->resetCurrentBlock(false); @@ -534,7 +535,7 @@ struct BlockSorter { #ifdef _PARALLEL_SORT ad_utility::parallel_sort(std::begin(block), std::end(block), comparator_); #else - std::ranges::sort(block, comparator_); + ql::ranges::sort(block, comparator_); #endif } }; @@ -612,7 +613,7 @@ class CompressedExternalIdTableSorter // one. Either this function or the following function must be called exactly // once. auto sortedView() { - return std::views::join(ad_utility::OwningView{getSortedBlocks()}); + return ql::views::join(ad_utility::OwningView{getSortedBlocks()}); } // Similar to `sortedView` (see above), but the elements are yielded in @@ -642,8 +643,8 @@ class CompressedExternalIdTableSorter // once. void pushBlock(const IdTableStatic<0>& block) override { AD_CONTRACT_CHECK(block.numColumns() == this->numColumns_); - std::ranges::for_each(block, - [ptr = this](const auto& row) { ptr->push(row); }); + ql::ranges::for_each(block, + [ptr = this](const auto& row) { ptr->push(row); }); } // The implementation of the type-erased interface. Get the sorted blocks as @@ -680,7 +681,7 @@ class CompressedExternalIdTableSorter co_yield blockAsStatic; } } else { - // TODO Use `std::views::chunk`. + // TODO Use `ql::views::chunkd`. for (size_t i = 0; i < block.numRows(); i += blocksizeOutput) { size_t upper = std::min(i + blocksizeOutput, block.numRows()); auto curBlock = IdTableStatic( @@ -749,7 +750,7 @@ class CompressedExternalIdTableSorter #ifdef _PARALLEL_SORT ad_utility::parallel_sort(block.begin(), block.end(), comparator_); #else - std::ranges::sort(block, comparator_); + ql::ranges::sort(block, comparator_); #endif } diff --git a/src/engine/idTable/IdTable.h b/src/engine/idTable/IdTable.h index fb28b4aa99..4ffe33138a 100644 --- a/src/engine/idTable/IdTable.h +++ b/src/engine/idTable/IdTable.h @@ -214,7 +214,7 @@ class IdTable { if (data().size() > numColumns_) { data().erase(data().begin() + numColumns_, data().end()); } - AD_CONTRACT_CHECK(std::ranges::all_of( + AD_CONTRACT_CHECK(ql::ranges::all_of( data(), [](const auto& column) { return column.empty(); })); } @@ -263,7 +263,7 @@ class IdTable { AD_CORRECTNESS_CHECK(numColumns == NumColumns); } AD_CORRECTNESS_CHECK(this->data().size() == numColumns_); - AD_CORRECTNESS_CHECK(std::ranges::all_of( + AD_CORRECTNESS_CHECK(ql::ranges::all_of( this->data(), [this](const auto& column) { return column.size() == numRows_; })); } @@ -382,8 +382,8 @@ class IdTable { // Note: The semantics of this function is similar to `std::vector::resize`. // To set the capacity, use the `reserve` function. void resize(size_t numRows) requires(!isView) { - std::ranges::for_each(data(), - [numRows](auto& column) { column.resize(numRows); }); + ql::ranges::for_each(data(), + [numRows](auto& column) { column.resize(numRows); }); numRows_ = numRows; } @@ -394,8 +394,8 @@ class IdTable { // of the next `numRows - size()` elements (via `insert` or `push_back`) can // be done in O(1) time without dynamic allocations. void reserve(size_t numRows) requires(!isView) { - std::ranges::for_each(data(), - [numRows](auto& column) { column.reserve(numRows); }); + ql::ranges::for_each(data(), + [numRows](auto& column) { column.reserve(numRows); }); } // Delete all the elements, but keep the allocated memory (`capacityRows_` @@ -403,14 +403,14 @@ class IdTable { // `shrinkToFit()` after calling `clear()` . void clear() requires(!isView) { numRows_ = 0; - std::ranges::for_each(data(), [](auto& column) { column.clear(); }); + ql::ranges::for_each(data(), [](auto& column) { column.clear(); }); } // Adjust the capacity to exactly match the size. This optimizes the memory // consumption of this table. This operation runs in O(size()), allocates // memory, and invalidates all iterators. void shrinkToFit() requires(!isView) { - std::ranges::for_each(data(), [](auto& column) { column.shrink_to_fit(); }); + ql::ranges::for_each(data(), [](auto& column) { column.shrink_to_fit(); }); } // Note: The following functions `emplace_back` and `push_back` all have the @@ -421,7 +421,7 @@ class IdTable { // Insert a new uninitialized row at the end. void emplace_back() requires(!isView) { - std::ranges::for_each(data(), [](auto& column) { column.emplace_back(); }); + ql::ranges::for_each(data(), [](auto& column) { column.emplace_back(); }); ++numRows_; } @@ -434,10 +434,10 @@ class IdTable { void push_back(const RowLike& newRow) requires(!isView) { AD_EXPENSIVE_CHECK(newRow.size() == numColumns()); ++numRows_; - std::ranges::for_each(ad_utility::integerRange(numColumns()), - [this, &newRow](auto i) { - data()[i].push_back(*(std::begin(newRow) + i)); - }); + ql::ranges::for_each(ad_utility::integerRange(numColumns()), + [this, &newRow](auto i) { + data()[i].push_back(*(std::begin(newRow) + i)); + }); } void push_back(const std::initializer_list& newRow) requires(!isView) { @@ -482,7 +482,7 @@ class IdTable { AD_CONTRACT_CHECK(newColumns.size() >= numColumns()); Data newStorage(std::make_move_iterator(newColumns.begin()), std::make_move_iterator(newColumns.begin() + numColumns())); - std::ranges::for_each( + ql::ranges::for_each( ad_utility::integerRange(numColumns()), [this, &newStorage](auto i) { newStorage[i].insert(newStorage[i].end(), data()[i].begin(), data()[i].end()); @@ -549,7 +549,7 @@ class IdTable { // the argument `columnIndices`. IdTable asColumnSubsetView( std::span columnIndices) const requires isDynamic { - AD_CONTRACT_CHECK(std::ranges::all_of( + AD_CONTRACT_CHECK(ql::ranges::all_of( columnIndices, [this](size_t idx) { return idx < numColumns(); })); ViewSpans viewSpans; viewSpans.reserve(columnIndices.size()); @@ -574,7 +574,7 @@ class IdTable { // First check that the `subset` is indeed a subset of the column // indices. std::vector check{subset.begin(), subset.end()}; - std::ranges::sort(check); + ql::ranges::sort(check); AD_CONTRACT_CHECK(std::unique(check.begin(), check.end()) == check.end()); AD_CONTRACT_CHECK(!subset.empty() && subset.back() < numColumns()); @@ -586,7 +586,7 @@ class IdTable { Data newData; newData.reserve(subset.size()); - std::ranges::for_each(subset, [this, &newData](ColumnIndex colIdx) { + ql::ranges::for_each(subset, [this, &newData](ColumnIndex colIdx) { newData.push_back(std::move(data().at(colIdx))); }); data() = std::move(newData); @@ -691,12 +691,12 @@ class IdTable { auto numInserted = end - begin; auto oldSize = size(); resize(numRows() + numInserted); - std::ranges::for_each( - ad_utility::integerRange(numColumns()), - [this, &table, oldSize, begin, numInserted](size_t i) { - std::ranges::copy(table.getColumn(i).subspan(begin, numInserted), - getColumn(i).begin() + oldSize); - }); + ql::ranges::for_each(ad_utility::integerRange(numColumns()), + [this, &table, oldSize, begin, numInserted](size_t i) { + ql::ranges::copy( + table.getColumn(i).subspan(begin, numInserted), + getColumn(i).begin() + oldSize); + }); } // Check whether two `IdTables` have the same content. Mostly used for unit @@ -710,7 +710,7 @@ class IdTable { } // TODO This can be implemented using `zip_view` and - // `std::ranges::all_of`. The iteration over the columns is cache-friendly. + // `ql::ranges::all_of`. The iteration over the columns is cache-friendly. const auto& cols = getColumns(); const auto& otherCols = other.getColumns(); for (size_t i = 0; i < numColumns(); ++i) { @@ -795,7 +795,7 @@ class IdTableStatic friend std::ostream& operator<<(std::ostream& os, const IdTableStatic& idTable) { os << "{ "; - std::ranges::copy( + ql::ranges::copy( idTable, std::ostream_iterator>(os, " ")); os << "}"; return os; diff --git a/src/engine/idTable/IdTableRow.h b/src/engine/idTable/IdTableRow.h index 4ab4c77776..23290d2305 100644 --- a/src/engine/idTable/IdTableRow.h +++ b/src/engine/idTable/IdTableRow.h @@ -91,7 +91,7 @@ class Row { explicit operator std::array() const requires(numStaticColumns != 0) { std::array result; - std::ranges::copy(*this, result.begin()); + ql::ranges::copy(*this, result.begin()); return result; } @@ -282,7 +282,7 @@ class RowReferenceImpl { explicit operator std::array() const requires(numStaticColumns != 0) { std::array result; - std::ranges::copy(*this, result.begin()); + ql::ranges::copy(*this, result.begin()); return result; } @@ -319,18 +319,19 @@ class RowReferenceImpl { // This strange overload needs to be declared to make `Row` a // `std::random_access_range` that can be used e.g. with - // `std::ranges::sort`. There is no need to define it, as it is only + // `ql::ranges::sort`. There is no need to define it, as it is only // needed to fulfill the concept `std::indirectly_writable`. For more // details on this "esoteric" overload see the notes at the end of // `https://en.cppreference.com/w/cpp/iterator/indirectly_writable` This& operator=(const Row& other) const&&; - protected: // No need to copy this internal type, but the implementation of the - // `RowReference` class below requires it, - // so the copy Constructor is protected. + // `RowReference` class and the `input_range` concept from `range-v3` + // require it. RowReferenceWithRestrictedAccess(const RowReferenceWithRestrictedAccess&) = default; + RowReferenceWithRestrictedAccess(RowReferenceWithRestrictedAccess&&) = + default; }; }; diff --git a/src/engine/sparqlExpressions/CountStarExpression.cpp b/src/engine/sparqlExpressions/CountStarExpression.cpp index e4abc2f20a..45cd94314f 100644 --- a/src/engine/sparqlExpressions/CountStarExpression.cpp +++ b/src/engine/sparqlExpressions/CountStarExpression.cpp @@ -37,22 +37,22 @@ ExpressionResult CountStarExpression::evaluate( // part of the DISTINCT computation. auto varToColNoInternalVariables = - ctx->_variableToColumnMap | std::views::filter([](const auto& varAndIdx) { + ctx->_variableToColumnMap | ql::views::filter([](const auto& varAndIdx) { return !varAndIdx.first.name().starts_with( QLEVER_INTERNAL_VARIABLE_PREFIX); }); - table.setNumColumns(std::ranges::distance(varToColNoInternalVariables)); + table.setNumColumns(ql::ranges::distance(varToColNoInternalVariables)); table.resize(ctx->size()); auto checkCancellation = [ctx]() { ctx->cancellationHandle_->throwIfCancelled(); }; size_t targetColIdx = 0; for (const auto& [sourceColIdx, _] : - varToColNoInternalVariables | std::views::values) { + varToColNoInternalVariables | ql::views::values) { const auto& sourceColumn = ctx->_inputTable.getColumn(sourceColIdx); - std::ranges::copy(sourceColumn.begin() + ctx->_beginIndex, - sourceColumn.begin() + ctx->_endIndex, - table.getColumn(targetColIdx).begin()); + ql::ranges::copy(sourceColumn.begin() + ctx->_beginIndex, + sourceColumn.begin() + ctx->_endIndex, + table.getColumn(targetColIdx).begin()); ++targetColIdx; checkCancellation(); } @@ -60,7 +60,7 @@ ExpressionResult CountStarExpression::evaluate( table.numRows(), table.numColumns(), ctx->deadline_, "Sort for COUNT(DISTINCT *)"); ad_utility::callFixedSize(table.numColumns(), [&table]() { - Engine::sort(&table, std::ranges::lexicographical_compare); + Engine::sort(&table, ql::ranges::lexicographical_compare); }); return Id::makeFromInt( static_cast(Engine::countDistinct(table, checkCancellation))); diff --git a/src/engine/sparqlExpressions/LiteralExpression.h b/src/engine/sparqlExpressions/LiteralExpression.h index 50e0de5fd3..4d9be1db5b 100644 --- a/src/engine/sparqlExpressions/LiteralExpression.h +++ b/src/engine/sparqlExpressions/LiteralExpression.h @@ -171,11 +171,11 @@ class LiteralExpression : public SparqlExpression { if (context->_groupedVariables.contains(variable) && !isInsideAggregate()) { const auto& table = context->_inputTable; auto constantValue = table.at(context->_beginIndex, column.value()); - AD_EXPENSIVE_CHECK((std::ranges::all_of( - table.begin() + context->_beginIndex, - table.begin() + context->_endIndex, [&](const auto& row) { - return row[column.value()] == constantValue; - }))); + AD_EXPENSIVE_CHECK(( + std::all_of(table.begin() + context->_beginIndex, + table.begin() + context->_endIndex, [&](const auto& row) { + return row[column.value()] == constantValue; + }))); return constantValue; } else { return variable; diff --git a/src/engine/sparqlExpressions/NaryExpressionImpl.h b/src/engine/sparqlExpressions/NaryExpressionImpl.h index 3f852b5f09..1061e64d0d 100644 --- a/src/engine/sparqlExpressions/NaryExpressionImpl.h +++ b/src/engine/sparqlExpressions/NaryExpressionImpl.h @@ -82,7 +82,7 @@ class NaryExpression : public SparqlExpression { using ResultType = typename decltype(resultGenerator)::value_type; VectorWithMemoryLimit result{context->_allocator}; result.reserve(targetSize); - std::ranges::move(resultGenerator, std::back_inserter(result)); + ql::ranges::move(resultGenerator, std::back_inserter(result)); if constexpr (resultIsConstant) { AD_CORRECTNESS_CHECK(result.size() == 1); @@ -186,7 +186,7 @@ requires(isOperation) [[nodiscard]] string NaryExpression::getCacheKey( const VariableToColumnMap& varColMap) const { string key = typeid(*this).name(); key += ad_utility::lazyStrJoin( - children_ | std::views::transform([&varColMap](const auto& child) { + children_ | ql::views::transform([&varColMap](const auto& child) { return child->getCacheKey(varColMap); }), ""); diff --git a/src/engine/sparqlExpressions/NumericBinaryExpressions.cpp b/src/engine/sparqlExpressions/NumericBinaryExpressions.cpp index e8bf9a450f..173ed08271 100644 --- a/src/engine/sparqlExpressions/NumericBinaryExpressions.cpp +++ b/src/engine/sparqlExpressions/NumericBinaryExpressions.cpp @@ -223,8 +223,8 @@ std::vector mergeChildrenForBinaryOpExpressionImpl( } } if constexpr (binOp == AND) { - std::ranges::move(itLeft, leftChild.end(), std::back_inserter(resPairs)); - std::ranges::move(itRight, rightChild.end(), std::back_inserter(resPairs)); + ql::ranges::move(itLeft, leftChild.end(), std::back_inserter(resPairs)); + ql::ranges::move(itRight, rightChild.end(), std::back_inserter(resPairs)); } pd::checkPropertiesForPrefilterConstruction(resPairs); return resPairs; diff --git a/src/engine/sparqlExpressions/NumericUnaryExpressions.cpp b/src/engine/sparqlExpressions/NumericUnaryExpressions.cpp index a9d46d645e..c97e7ce57e 100644 --- a/src/engine/sparqlExpressions/NumericUnaryExpressions.cpp +++ b/src/engine/sparqlExpressions/NumericUnaryExpressions.cpp @@ -68,8 +68,8 @@ class UnaryNegateExpressionImpl : public NaryExpression { auto child = this->getNthChild(0).value()->getPrefilterExpressionForMetadata( !isNegated); - std::ranges::for_each( - child | std::views::keys, + ql::ranges::for_each( + child | ql::views::keys, [](std::unique_ptr& expression) { expression = std::make_unique(std::move(expression)); diff --git a/src/engine/sparqlExpressions/PrefilterExpressionIndex.cpp b/src/engine/sparqlExpressions/PrefilterExpressionIndex.cpp index 93af26df7e..616d432429 100644 --- a/src/engine/sparqlExpressions/PrefilterExpressionIndex.cpp +++ b/src/engine/sparqlExpressions/PrefilterExpressionIndex.cpp @@ -69,7 +69,7 @@ static void checkEvalRequirements(std::span input, throw std::runtime_error(errorMessage); }; // Check for duplicates. - if (auto it = std::ranges::adjacent_find(input); it != input.end()) { + if (auto it = ql::ranges::adjacent_find(input); it != input.end()) { throwRuntimeError("The provided data blocks must be unique."); } // Helper to check for fully sorted blocks. Return `true` if `b1 < b2` is @@ -91,7 +91,7 @@ static void checkEvalRequirements(std::span input, } return false; }; - if (!std::ranges::is_sorted(input, checkOrder)) { + if (!ql::ranges::is_sorted(input, checkOrder)) { throwRuntimeError("The blocks must be provided in sorted order."); } // Helper to check for column consistency. Returns `true` if the columns for @@ -103,7 +103,7 @@ static void checkEvalRequirements(std::span input, getMaskedTriple(b2.firstTriple_, evaluationColumn) || checkBlockIsInconsistent(b2, evaluationColumn); }; - if (auto it = std::ranges::adjacent_find(input, checkColumnConsistency); + if (auto it = ql::ranges::adjacent_find(input, checkColumnConsistency); it != input.end()) { throwRuntimeError( "The values in the columns up to the evaluation column must be " @@ -498,14 +498,14 @@ static std::unique_ptr makeMirroredExpression( //______________________________________________________________________________ void checkPropertiesForPrefilterConstruction( const std::vector& vec) { - auto viewVariable = vec | std::views::values; - if (!std::ranges::is_sorted(viewVariable, std::less<>{})) { + auto viewVariable = vec | ql::views::values; + if (!ql::ranges::is_sorted(viewVariable, std::less<>{})) { throw std::runtime_error( "The vector must contain the pairs in " "sorted order w.r.t. Variable value."); } - if (auto it = std::ranges::adjacent_find(viewVariable); - it != std::ranges::end(viewVariable)) { + if (auto it = ql::ranges::adjacent_find(viewVariable); + it != ql::ranges::end(viewVariable)) { throw std::runtime_error( "For each relevant Variable must exist exactly one " " pair."); diff --git a/src/engine/sparqlExpressions/RegexExpression.cpp b/src/engine/sparqlExpressions/RegexExpression.cpp index 695f12b947..02047297ad 100644 --- a/src/engine/sparqlExpressions/RegexExpression.cpp +++ b/src/engine/sparqlExpressions/RegexExpression.cpp @@ -247,7 +247,7 @@ ExpressionResult RegexExpression::evaluatePrefixRegex( result.reserve(resultSize); for (auto id : detail::makeGenerator(variable, resultSize, context)) { result.push_back(Id::makeFromBool( - std::ranges::any_of(lowerAndUpperIds, [&](const auto& lowerUpper) { + ql::ranges::any_of(lowerAndUpperIds, [&](const auto& lowerUpper) { return !valueIdComparators::compareByBits(id, lowerUpper.first) && valueIdComparators::compareByBits(id, lowerUpper.second); }))); @@ -270,7 +270,7 @@ ExpressionResult RegexExpression::evaluateGeneralCase( // `std::nullopt` for a row, the result is `UNDEF`. Otherwise, we have a // string and evaluate the regex on it. auto computeResult = [&](const ValueGetter& getter) { - std::ranges::for_each( + ql::ranges::for_each( detail::makeGenerator(AD_FWD(input), resultSize, context), [&getter, &context, &result, this](const auto& id) { auto str = getter(id, context); diff --git a/src/engine/sparqlExpressions/RelationalExpressions.cpp b/src/engine/sparqlExpressions/RelationalExpressions.cpp index 5bc36b528e..603e334c69 100644 --- a/src/engine/sparqlExpressions/RelationalExpressions.cpp +++ b/src/engine/sparqlExpressions/RelationalExpressions.cpp @@ -342,15 +342,15 @@ SparqlExpression::Estimates getEstimatesForFilterExpressionImpl( // filtering on the `LocalVocab`. // Check iff all the pairs `(children[0], someOtherChild)` can be evaluated // using binary search. - if (std::ranges::all_of(children | std::views::drop(1), - [&lhs = children.at(0), - &canBeEvaluatedWithBinarySearch](const auto& child) { - // The implementation automatically chooses the - // cheaper direction, so we can do the same when - // estimating the cost. - return canBeEvaluatedWithBinarySearch(lhs, child) || - canBeEvaluatedWithBinarySearch(child, lhs); - })) { + if (ql::ranges::all_of(children | ql::views::drop(1), + [&lhs = children.at(0), + &canBeEvaluatedWithBinarySearch](const auto& child) { + // The implementation automatically chooses the + // cheaper direction, so we can do the same when + // estimating the cost. + return canBeEvaluatedWithBinarySearch(lhs, child) || + canBeEvaluatedWithBinarySearch(child, lhs); + })) { // When evaluating via binary search, the only significant cost that occurs // is that of writing the output. costEstimate = sizeEstimate; @@ -385,7 +385,7 @@ ExpressionResult InExpression::evaluate( auto lhs = children_.at(0)->evaluate(context); ExpressionResult result{ad_utility::SetOfIntervals{}}; bool firstChild = true; - for (const auto& child : children_ | std::views::drop(1)) { + for (const auto& child : children_ | ql::views::drop(1)) { auto rhs = child->evaluate(context); auto evaluateEqualsExpression = [context](const auto& a, auto b) -> ExpressionResult { diff --git a/src/engine/sparqlExpressions/RelationalExpressions.h b/src/engine/sparqlExpressions/RelationalExpressions.h index d406afc936..45fc44f471 100644 --- a/src/engine/sparqlExpressions/RelationalExpressions.h +++ b/src/engine/sparqlExpressions/RelationalExpressions.h @@ -69,7 +69,7 @@ class InExpression : public SparqlExpression { explicit InExpression(SparqlExpression::Ptr lhs, Children children) { children_.reserve(children.size() + 1); children_.push_back(std::move(lhs)); - std::ranges::move(children, std::back_inserter(children_)); + ql::ranges::move(children, std::back_inserter(children_)); } ExpressionResult evaluate(EvaluationContext* context) const override; diff --git a/src/engine/sparqlExpressions/SetOfIntervals.cpp b/src/engine/sparqlExpressions/SetOfIntervals.cpp index 9db82c4fd9..6022a2f3a2 100644 --- a/src/engine/sparqlExpressions/SetOfIntervals.cpp +++ b/src/engine/sparqlExpressions/SetOfIntervals.cpp @@ -4,7 +4,7 @@ #include "SetOfIntervals.h" -#include +#include "backports/algorithm.h" namespace ad_utility { // ___________________________________________________________________________ diff --git a/src/engine/sparqlExpressions/SetOfIntervals.h b/src/engine/sparqlExpressions/SetOfIntervals.h index a899facbe4..4ca8bea107 100644 --- a/src/engine/sparqlExpressions/SetOfIntervals.h +++ b/src/engine/sparqlExpressions/SetOfIntervals.h @@ -8,6 +8,7 @@ #include #include +#include "backports/algorithm.h" #include "util/Exception.h" namespace ad_utility { @@ -77,7 +78,7 @@ struct SetOfIntervals { inline static std::vector toBitVector(const SetOfIntervals& a, size_t targetSize) { std::vector result(targetSize, false); - toBitVector(a, targetSize, std::ranges::begin(result)); + toBitVector(a, targetSize, ql::ranges::begin(result)); return result; } }; diff --git a/src/engine/sparqlExpressions/SparqlExpression.cpp b/src/engine/sparqlExpressions/SparqlExpression.cpp index 7e8fa1648c..b5ec3aa0f7 100644 --- a/src/engine/sparqlExpressions/SparqlExpression.cpp +++ b/src/engine/sparqlExpressions/SparqlExpression.cpp @@ -49,7 +49,7 @@ bool SparqlExpression::containsAggregate() const { return true; } - return std::ranges::any_of( + return ql::ranges::any_of( children(), [](const Ptr& child) { return child->containsAggregate(); }); } @@ -84,10 +84,9 @@ std::optional<::Variable> SparqlExpression::getVariableOrNullopt() const { // _____________________________________________________________________________ bool SparqlExpression::containsLangExpression() const { - return std::ranges::any_of(children(), - [](const SparqlExpression::Ptr& child) { - return child->containsLangExpression(); - }); + return ql::ranges::any_of(children(), [](const SparqlExpression::Ptr& child) { + return child->containsLangExpression(); + }); } // _____________________________________________________________________________ diff --git a/src/engine/sparqlExpressions/SparqlExpressionGenerators.h b/src/engine/sparqlExpressions/SparqlExpressionGenerators.h index 7f5378d4ea..33b46914d0 100644 --- a/src/engine/sparqlExpressions/SparqlExpressionGenerators.h +++ b/src/engine/sparqlExpressions/SparqlExpressionGenerators.h @@ -54,7 +54,7 @@ template requires(std::ranges::input_range) auto resultGenerator(T&& vector, size_t numItems, Transformation transformation = {}) { AD_CONTRACT_CHECK(numItems == vector.size()); - return ad_utility::allView(AD_FWD(vector)) | std::views::transform(std::move(transformation)); + return ad_utility::allView(AD_FWD(vector)) | ql::views::transform(std::move(transformation)); } template diff --git a/src/engine/sparqlExpressions/SparqlExpressionPimpl.cpp b/src/engine/sparqlExpressions/SparqlExpressionPimpl.cpp index 99d3f19e3f..0f6cf3ebb9 100644 --- a/src/engine/sparqlExpressions/SparqlExpressionPimpl.cpp +++ b/src/engine/sparqlExpressions/SparqlExpressionPimpl.cpp @@ -73,7 +73,7 @@ void SparqlExpressionPimpl::setDescriptor(std::string descriptor) { // _____________________________________________________________________________ bool SparqlExpressionPimpl::isVariableContained( const Variable& variable) const { - return std::ranges::any_of( + return ql::ranges::any_of( containedVariables(), [&variable](const auto* varPtr) { return *varPtr == variable; }); } diff --git a/src/engine/sparqlExpressions/SparqlExpressionPimpl.h b/src/engine/sparqlExpressions/SparqlExpressionPimpl.h index 0b4b7b8f48..cd01eda9e5 100644 --- a/src/engine/sparqlExpressions/SparqlExpressionPimpl.h +++ b/src/engine/sparqlExpressions/SparqlExpressionPimpl.h @@ -44,7 +44,7 @@ class SparqlExpressionPimpl { // COUNT(?x) + ?m returns true if and only if ?m is in `groupedVariables`. [[nodiscard]] bool isAggregate( const ad_utility::HashSet& groupedVariables) const { - // TODO This can be std::ranges::all_of as soon as libc++ supports + // TODO This can be ql::ranges::all_of as soon as libc++ supports // it, or the combination of clang + libstdc++ + coroutines works. auto unaggregatedVariables = getUnaggregatedVariables(); for (const auto& var : unaggregatedVariables) { diff --git a/src/engine/sparqlExpressions/StringExpressions.cpp b/src/engine/sparqlExpressions/StringExpressions.cpp index f7e7d9bca7..3378ae7fdb 100644 --- a/src/engine/sparqlExpressions/StringExpressions.cpp +++ b/src/engine/sparqlExpressions/StringExpressions.cpp @@ -128,7 +128,7 @@ using IriOrUriExpression = NARY<1, FV>; [[maybe_unused]] auto strlen = [](std::string_view s) { // Count UTF-8 characters by skipping continuation bytes (those starting with // "10"). - auto utf8Len = std::ranges::count_if( + auto utf8Len = ql::ranges::count_if( s, [](char c) { return (static_cast(c) & 0xC0) != 0x80; }); return Id::makeFromInt(utf8Len); }; @@ -393,7 +393,7 @@ class ConcatExpression : public detail::VariadicExpression { // One of the previous children was not a constant, so we already // store a vector. auto& resultAsVector = std::get(result); - std::ranges::for_each(resultAsVector, [&](std::string& target) { + ql::ranges::for_each(resultAsVector, [&](std::string& target) { target.append(strFromConstant); }); } @@ -417,7 +417,7 @@ class ConcatExpression : public detail::VariadicExpression { // The `result` already is a vector, and the current child also returns // multiple results, so we do the `natural` way. auto& resultAsVec = std::get(result); - // TODO Use `std::views::zip` or `enumerate`. + // TODO Use `ql::views::zip` or `enumerate`. size_t i = 0; for (auto& el : gen) { if (auto str = StringValueGetter{}(std::move(el), ctx); @@ -430,7 +430,7 @@ class ConcatExpression : public detail::VariadicExpression { } ctx->cancellationHandle_->throwIfCancelled(); }; - std::ranges::for_each( + ql::ranges::for_each( childrenVec(), [&ctx, &visitSingleExpressionResult](const auto& child) { std::visit(visitSingleExpressionResult, child->evaluate(ctx)); }); @@ -443,8 +443,8 @@ class ConcatExpression : public detail::VariadicExpression { auto& stringVec = std::get(result); VectorWithMemoryLimit resultAsVec(ctx->_allocator); resultAsVec.reserve(stringVec.size()); - std::ranges::copy(stringVec | std::views::transform(toLiteral), - std::back_inserter(resultAsVec)); + ql::ranges::copy(stringVec | ql::views::transform(toLiteral), + std::back_inserter(resultAsVec)); return resultAsVec; } } diff --git a/src/engine/sparqlExpressions/VariadicExpression.h b/src/engine/sparqlExpressions/VariadicExpression.h index 8a7b81f479..3352c20d1f 100644 --- a/src/engine/sparqlExpressions/VariadicExpression.h +++ b/src/engine/sparqlExpressions/VariadicExpression.h @@ -34,7 +34,7 @@ class VariadicExpression : public SparqlExpression { std::string getCacheKey(const VariableToColumnMap& varColMap) const override { string key = typeid(*this).name(); auto childKeys = ad_utility::lazyStrJoin( - children_ | std::views::transform([&varColMap](const auto& childPtr) { + children_ | ql::views::transform([&varColMap](const auto& childPtr) { return childPtr->getCacheKey(varColMap); }), ", "); diff --git a/src/global/IdTriple.h b/src/global/IdTriple.h index b349743f9a..142dacab5b 100644 --- a/src/global/IdTriple.h +++ b/src/global/IdTriple.h @@ -30,8 +30,8 @@ struct IdTriple { friend std::ostream& operator<<(std::ostream& os, const IdTriple& triple) { os << "IdTriple("; - std::ranges::copy(triple.ids_, std::ostream_iterator(os, ", ")); - std::ranges::copy(triple.payload_, std::ostream_iterator(os, ", ")); + ql::ranges::copy(triple.ids_, std::ostream_iterator(os, ", ")); + ql::ranges::copy(triple.payload_, std::ostream_iterator(os, ", ")); os << ")"; return os; } diff --git a/src/global/SpecialIds.h b/src/global/SpecialIds.h index b844e11724..10ad76b144 100644 --- a/src/global/SpecialIds.h +++ b/src/global/SpecialIds.h @@ -32,13 +32,12 @@ inline const ad_utility::HashMap& specialIds() { // Perform the following checks: All the special IDs are unique, all of them // have the `Undefined` datatype, but none of them is equal to the "actual" // UNDEF value. - auto values = std::views::values(result); + auto values = ql::views::values(result); auto undefTypeButNotUndefValue = [](Id id) { return id != Id::makeUndefined() && id.getDatatype() == Datatype::Undefined; }; - AD_CORRECTNESS_CHECK( - std::ranges::all_of(values, undefTypeButNotUndefValue)); + AD_CORRECTNESS_CHECK(ql::ranges::all_of(values, undefTypeButNotUndefValue)); ad_utility::HashSet uniqueIds(values.begin(), values.end()); AD_CORRECTNESS_CHECK(uniqueIds.size() == result.size()); return result; diff --git a/src/global/ValueId.h b/src/global/ValueId.h index fe429b98fa..91930ff1f3 100644 --- a/src/global/ValueId.h +++ b/src/global/ValueId.h @@ -101,8 +101,8 @@ class ValueId { // Assert that the types in `stringTypes_` are directly adjacent. This is // required to make the comparison of IDs in `ValueIdComparators.h` work. - static constexpr Datatype maxStringType_ = std::ranges::max(stringTypes_); - static constexpr Datatype minStringType_ = std::ranges::min(stringTypes_); + static constexpr Datatype maxStringType_ = ql::ranges::max(stringTypes_); + static constexpr Datatype minStringType_ = ql::ranges::min(stringTypes_); static_assert(static_cast(maxStringType_) - static_cast(minStringType_) + 1 == stringTypes_.size()); diff --git a/src/global/ValueIdComparators.h b/src/global/ValueIdComparators.h index 0b23621c77..bfbf7bb551 100644 --- a/src/global/ValueIdComparators.h +++ b/src/global/ValueIdComparators.h @@ -8,6 +8,7 @@ #include #include "global/ValueId.h" +#include "util/Algorithm.h" #include "util/ComparisonWithNan.h" #include "util/OverloadCallOperator.h" diff --git a/src/index/CompressedRelation.cpp b/src/index/CompressedRelation.cpp index 05ce8a8a82..fc306e892f 100644 --- a/src/index/CompressedRelation.cpp +++ b/src/index/CompressedRelation.cpp @@ -26,7 +26,7 @@ using namespace std::chrono_literals; // A small helper function to obtain the begin and end iterator of a range static auto getBeginAndEnd(auto& range) { - return std::pair{std::ranges::begin(range), std::ranges::end(range)}; + return std::pair{ql::ranges::begin(range), ql::ranges::end(range)}; } // modify the `block` according to the `limitOffset`. Also modify the @@ -156,9 +156,9 @@ bool CompressedRelationReader::FilterDuplicatesAndGraphs:: if (!metadata.graphInfo_.has_value()) { return true; } - return !std::ranges::all_of( - metadata.graphInfo_.value(), - [&wantedGraphs = desiredGraphs_.value()](Id containedGraph) { + const auto& graphInfo = metadata.graphInfo_.value(); + return !ql::ranges::all_of( + graphInfo, [&wantedGraphs = desiredGraphs_.value()](Id containedGraph) { return wantedGraphs.contains(containedGraph); }); } @@ -184,7 +184,7 @@ bool CompressedRelationReader::FilterDuplicatesAndGraphs:: } else { AD_EXPENSIVE_CHECK( !desiredGraphs_.has_value() || - std::ranges::all_of(block, isDesiredGraphId(), graphIdFromRow)); + ql::ranges::all_of(block, isDesiredGraphId(), graphIdFromRow)); } return needsFilteringByGraph; } @@ -224,7 +224,7 @@ bool CompressedRelationReader::FilterDuplicatesAndGraphs::canBlockBeSkipped( return false; } const auto& containedGraphs = block.graphInfo_.value(); - return std::ranges::none_of( + return ql::ranges::none_of( desiredGraphs_.value(), [&containedGraphs](const auto& desiredGraph) { return ad_utility::contains(containedGraphs, desiredGraph); }); @@ -246,7 +246,7 @@ CompressedRelationReader::IdTableGenerator CompressedRelationReader::lazyScan( // Compute the sequence of relevant blocks. If the sequence is empty, there // is nothing to yield. auto relevantBlocks = getRelevantBlocks(scanSpec, blockMetadata); - if (std::ranges::empty(relevantBlocks)) { + if (ql::ranges::empty(relevantBlocks)) { co_return; } @@ -417,10 +417,10 @@ std::vector CompressedRelationReader::getBlocksForJoin( // `!lessThan(a,b) && !lessThan(b, a)` is not transitive. std::vector result; auto blockIsNeeded = [&joinColumn, &lessThan](const auto& block) { - return !std::ranges::equal_range(joinColumn, block, lessThan).empty(); + return !ql::ranges::equal_range(joinColumn, block, lessThan).empty(); }; - std::ranges::copy(relevantBlocks | std::views::filter(blockIsNeeded), - std::back_inserter(result)); + ql::ranges::copy(relevantBlocks | ql::views::filter(blockIsNeeded), + std::back_inserter(result)); // The following check is cheap as there are only few blocks. AD_CORRECTNESS_CHECK(std::ranges::unique(result).empty()); return result; @@ -446,21 +446,22 @@ CompressedRelationReader::getBlocksForJoin( // Transform all the relevant blocks from a `ScanSpecAndBlocksAndBounds` a // `BlockWithFirstAndLastId` struct (see above). - auto getBlocksWithFirstAndLastId = [&blockLessThanBlock]( - const ScanSpecAndBlocksAndBounds& - metadataAndBlocks) { - auto getSingleBlock = - [&metadataAndBlocks]( - const CompressedBlockMetadata& block) -> BlockWithFirstAndLastId { - return {block, + auto getBlocksWithFirstAndLastId = + [&blockLessThanBlock]( + const ScanSpecAndBlocksAndBounds& metadataAndBlocks) { + auto getSingleBlock = + [&metadataAndBlocks](const CompressedBlockMetadata& block) + -> BlockWithFirstAndLastId { + return { + block, getRelevantIdFromTriple(block.firstTriple_, metadataAndBlocks), getRelevantIdFromTriple(block.lastTriple_, metadataAndBlocks)}; - }; - auto result = std::views::transform( - getBlocksFromMetadata(metadataAndBlocks), getSingleBlock); - AD_CORRECTNESS_CHECK(std::ranges::is_sorted(result, blockLessThanBlock)); - return result; - }; + }; + auto result = ql::views::transform( + getBlocksFromMetadata(metadataAndBlocks), getSingleBlock); + AD_CORRECTNESS_CHECK(ql::ranges::is_sorted(result, blockLessThanBlock)); + return result; + }; auto blocksWithFirstAndLastId1 = getBlocksWithFirstAndLastId(metadataAndBlocks1); @@ -475,7 +476,7 @@ CompressedRelationReader::getBlocksForJoin( const auto& otherBlocks) { std::vector result; for (const auto& block : blocks) { - if (!std::ranges::equal_range(otherBlocks, block, blockLessThanBlock) + if (!ql::ranges::equal_range(otherBlocks, block, blockLessThanBlock) .empty()) { result.push_back(block.block_); } @@ -504,8 +505,8 @@ IdTable CompressedRelationReader::scan( // Compute an upper bound for the size and reserve enough space in the // result. auto relevantBlocks = getRelevantBlocks(scanSpec, blocks); - auto sizes = relevantBlocks | - std::views::transform(&CompressedBlockMetadata::numRows_); + auto sizes = + relevantBlocks | ql::views::transform(&CompressedBlockMetadata::numRows_); auto upperBoundSize = std::accumulate(sizes.begin(), sizes.end(), size_t{0}); if (limitOffset._limit.has_value()) { upperBoundSize = std::min(upperBoundSize, @@ -534,9 +535,9 @@ DecompressedBlock CompressedRelationReader::readPossiblyIncompleteBlock( // We first scan the complete block including ALL columns. std::vector allAdditionalColumns; - std::ranges::copy( - std::views::iota(ADDITIONAL_COLUMN_GRAPH_ID, - blockMetadata.offsetsAndCompressedSize_.size()), + ql::ranges::copy( + ql::views::iota(ADDITIONAL_COLUMN_GRAPH_ID, + blockMetadata.offsetsAndCompressedSize_.size()), std::back_inserter(allAdditionalColumns)); ScanSpecification specForAllColumns{std::nullopt, std::nullopt, @@ -578,7 +579,7 @@ DecompressedBlock CompressedRelationReader::readPossiblyIncompleteBlock( return; } const auto& column = block.getColumn(columnIdx); - auto matchingRange = std::ranges::equal_range( + auto matchingRange = ql::ranges::equal_range( column.begin() + beginIdx, column.begin() + endIdx, relevantId.value()); beginIdx = matchingRange.begin() - column.begin(); endIdx = matchingRange.end() - column.begin(); @@ -599,12 +600,12 @@ DecompressedBlock CompressedRelationReader::readPossiblyIncompleteBlock( size_t i = 0; const auto& columnIndices = scanConfig.scanColumns_; for (const auto& inputColIdx : - columnIndices | std::views::filter([&](const auto& idx) { + columnIndices | ql::views::filter([&](const auto& idx) { return !manuallyDeleteGraphColumn || idx != ADDITIONAL_COLUMN_GRAPH_ID; })) { const auto& inputCol = block.getColumn(inputColIdx); - std::ranges::copy(inputCol.begin() + beginIdx, inputCol.begin() + endIdx, - result.getColumn(i).begin()); + ql::ranges::copy(inputCol.begin() + beginIdx, inputCol.begin() + endIdx, + result.getColumn(i).begin()); ++i; } @@ -656,8 +657,8 @@ std::pair CompressedRelationReader::getResultSizeImpl( // First accumulate the complete blocks in the "middle" std::size_t inserted = 0; std::size_t deleted = 0; - std::ranges::for_each( - std::ranges::subrange{beginBlock, endBlock}, [&](const auto& block) { + ql::ranges::for_each( + ql::ranges::subrange{beginBlock, endBlock}, [&](const auto& block) { const auto [ins, del] = locatedTriplesPerBlock.numTriples(block.blockIndex_); if (!exactSize || (ins == 0 && del == 0)) { @@ -766,7 +767,7 @@ IdTable CompressedRelationReader::getDistinctColIdsAndCountsImpl( continue; } const auto& block = optionalBlock.value(); - // TODO: use `std::views::chunk_by`. + // TODO: use `ql::views::chunkd_by`. for (size_t j = 0; j < block.numRows(); ++j) { Id colId = block(j, 0); processColId(colId, 1); @@ -841,7 +842,7 @@ CompressedBlock CompressedRelationReader::readCompressedBlockFromFile( ColumnIndicesRef columnIndices) const { CompressedBlock compressedBuffer; compressedBuffer.resize(columnIndices.size()); - // TODO Use `std::views::zip` + // TODO Use `ql::views::zip` for (size_t i = 0; i < compressedBuffer.size(); ++i) { const auto& offset = blockMetaData.offsetsAndCompressedSize_.at(columnIndices[i]); @@ -944,9 +945,9 @@ static std::pair>> getGraphInfo( // Return the contained graphs, or `nullopt` if there are too many of them. auto graphInfo = [&block]() -> std::optional> { std::vector graphColumn; - std::ranges::copy(block->getColumn(ADDITIONAL_COLUMN_GRAPH_ID), - std::back_inserter(graphColumn)); - std::ranges::sort(graphColumn); + ql::ranges::copy(block->getColumn(ADDITIONAL_COLUMN_GRAPH_ID), + std::back_inserter(graphColumn)); + ql::ranges::sort(graphColumn); auto [endOfUnique, _] = std::ranges::unique(graphColumn); size_t numGraphs = endOfUnique - graphColumn.begin(); if (numGraphs > MAX_NUM_GRAPHS_STORED_IN_BLOCK_METADATA) { @@ -1030,7 +1031,7 @@ CompressedRelationReader::getRelevantBlocks( return blockA.lastTriple_ < blockB.firstTriple_; }; - return std::ranges::equal_range(blockMetadata, key, comp); + return ql::ranges::equal_range(blockMetadata, key, comp); } // _____________________________________________________________________________ @@ -1087,8 +1088,8 @@ std::vector CompressedRelationReader::prepareColumnIndices( ColumnIndicesRef additionalColumns) { std::vector result; result.reserve(baseColumns.size() + additionalColumns.size()); - std::ranges::copy(baseColumns, std::back_inserter(result)); - std::ranges::copy(additionalColumns, std::back_inserter(result)); + ql::ranges::copy(baseColumns, std::back_inserter(result)); + ql::ranges::copy(additionalColumns, std::back_inserter(result)); return result; } @@ -1109,7 +1110,7 @@ std::vector CompressedRelationReader::prepareColumnIndices( // ___________________________________________________________________________ std::pair CompressedRelationReader::prepareLocatedTriples( ColumnIndicesRef columns) { - AD_CORRECTNESS_CHECK(std::ranges::is_sorted(columns)); + AD_CORRECTNESS_CHECK(ql::ranges::is_sorted(columns)); // Compute number of columns that should be read (except the graph column // and any payload columns). size_t numScanColumns = [&]() -> size_t { @@ -1120,7 +1121,7 @@ std::pair CompressedRelationReader::prepareLocatedTriples( } }(); // Check if one of the columns is the graph column. - auto it = std::ranges::find(columns, ADDITIONAL_COLUMN_GRAPH_ID); + auto it = ql::ranges::find(columns, ADDITIONAL_COLUMN_GRAPH_ID); bool containsGraphId = it != columns.end(); if (containsGraphId) { AD_CORRECTNESS_CHECK(it - columns.begin() == @@ -1153,7 +1154,7 @@ CompressedRelationMetadata CompressedRelationWriter::addSmallRelation( smallRelationsBuffer_.resize(offsetInBlock + numRows); for (size_t i = 0; i < relation.numColumns(); ++i) { - std::ranges::copy( + ql::ranges::copy( relation.getColumn(i), smallRelationsBuffer_.getColumn(i).begin() + offsetInBlock); } @@ -1277,7 +1278,7 @@ CompressedRelationMetadata CompressedRelationWriter::addCompleteLargeRelation( Id col0Id, auto&& sortedBlocks) { DistinctIdCounter distinctCol1Counter; for (auto& block : sortedBlocks) { - std::ranges::for_each(block.getColumn(1), std::ref(distinctCol1Counter)); + ql::ranges::for_each(block.getColumn(1), std::ref(distinctCol1Counter)); addBlockForLargeRelation( col0Id, std::make_shared(std::move(block).toDynamic())); } @@ -1373,7 +1374,7 @@ auto CompressedRelationWriter::createPermutationPair( return std::tie(a[0], a[1], a[2], a[3]) < std::tie(b[0], b[1], b[2], b[3]); }; - std::ranges::sort(relation, compare); + ql::ranges::sort(relation, compare); AD_CORRECTNESS_CHECK(!relation.empty()); writer2.compressAndWriteBlock(relation.at(0, 0), relation.at(relation.size() - 1, 0), @@ -1558,7 +1559,7 @@ auto CompressedRelationReader::getScanConfig( return {0, false}; } auto idx = static_cast( - std::ranges::find(columnIndices, ADDITIONAL_COLUMN_GRAPH_ID) - + ql::ranges::find(columnIndices, ADDITIONAL_COLUMN_GRAPH_ID) - columnIndices.begin()); bool deleteColumn = false; if (idx == columnIndices.size()) { diff --git a/src/index/CompressedRelation.h b/src/index/CompressedRelation.h index 36cbf14af0..96fefef2be 100644 --- a/src/index/CompressedRelation.h +++ b/src/index/CompressedRelation.h @@ -4,10 +4,10 @@ #pragma once -#include #include #include +#include "backports/algorithm.h" #include "engine/idTable/IdTable.h" #include "global/Id.h" #include "index/ScanSpecification.h" @@ -299,7 +299,7 @@ class CompressedRelationWriter { std::vector getFinishedBlocks() && { finish(); auto blocks = std::move(*(blockBuffer_.wlock())); - std::ranges::sort( + ql::ranges::sort( blocks, {}, [](const CompressedBlockMetadataNoBlockIndex& bl) { return std::tie(bl.firstTriple_.col0Id_, bl.firstTriple_.col1Id_, bl.firstTriple_.col2Id_); diff --git a/src/index/DeltaTriples.cpp b/src/index/DeltaTriples.cpp index e6b3bcd555..8b69c8d363 100644 --- a/src/index/DeltaTriples.cpp +++ b/src/index/DeltaTriples.cpp @@ -21,7 +21,7 @@ LocatedTriples::iterator& DeltaTriples::LocatedTripleHandles::forPermutation( void DeltaTriples::clear() { triplesInserted_.clear(); triplesDeleted_.clear(); - std::ranges::for_each(locatedTriples(), &LocatedTriplesPerBlock::clear); + ql::ranges::for_each(locatedTriples(), &LocatedTriplesPerBlock::clear); } // ____________________________________________________________________________ @@ -133,9 +133,9 @@ void DeltaTriples::rewriteLocalVocabEntriesAndBlankNodes(Triples& triples) { }; // Convert all local vocab and blank node `Id`s in all `triples`. - std::ranges::for_each(triples, [&convertId](IdTriple<0>& triple) { - std::ranges::for_each(triple.ids_, convertId); - std::ranges::for_each(triple.payload_, convertId); + ql::ranges::for_each(triples, [&convertId](IdTriple<0>& triple) { + ql::ranges::for_each(triple.ids_, convertId); + ql::ranges::for_each(triple.payload_, convertId); }); } @@ -145,26 +145,25 @@ void DeltaTriples::modifyTriplesImpl(CancellationHandle cancellationHandle, TriplesToHandlesMap& targetMap, TriplesToHandlesMap& inverseMap) { rewriteLocalVocabEntriesAndBlankNodes(triples); - std::ranges::sort(triples); + ql::ranges::sort(triples); auto [first, last] = std::ranges::unique(triples); triples.erase(first, last); std::erase_if(triples, [&targetMap](const IdTriple<0>& triple) { return targetMap.contains(triple); }); - std::ranges::for_each(triples, - [this, &inverseMap](const IdTriple<0>& triple) { - auto handle = inverseMap.find(triple); - if (handle != inverseMap.end()) { - eraseTripleInAllPermutations(handle->second); - inverseMap.erase(triple); - } - }); + ql::ranges::for_each(triples, [this, &inverseMap](const IdTriple<0>& triple) { + auto handle = inverseMap.find(triple); + if (handle != inverseMap.end()) { + eraseTripleInAllPermutations(handle->second); + inverseMap.erase(triple); + } + }); std::vector handles = locateAndAddTriples(std::move(cancellationHandle), triples, shouldExist); AD_CORRECTNESS_CHECK(triples.size() == handles.size()); - // TODO: replace with std::views::zip in C++23 + // TODO: replace with ql::views::zip in C++23 for (size_t i = 0; i < triples.size(); i++) { targetMap.insert({triples[i], handles[i]}); } diff --git a/src/index/DocsDB.cpp b/src/index/DocsDB.cpp index 9a2b182735..cf353b264b 100644 --- a/src/index/DocsDB.cpp +++ b/src/index/DocsDB.cpp @@ -4,9 +4,8 @@ #include "DocsDB.h" -#include - #include "../global/Constants.h" +#include "backports/algorithm.h" // _____________________________________________________________________________ void DocsDB::init(const string& fileName) { diff --git a/src/index/IndexBuilderTypes.h b/src/index/IndexBuilderTypes.h index 173e323095..85f3f5d195 100644 --- a/src/index/IndexBuilderTypes.h +++ b/src/index/IndexBuilderTypes.h @@ -102,7 +102,7 @@ class MonotonicBuffer { // the buffer. std::string_view addString(std::string_view input) { auto ptr = charAllocator_->allocate(input.size()); - std::ranges::copy(input, ptr); + ql::ranges::copy(input, ptr); return {ptr, ptr + input.size()}; } }; diff --git a/src/index/IndexImpl.Text.cpp b/src/index/IndexImpl.Text.cpp index 58fe7e577c..bd46c81e53 100644 --- a/src/index/IndexImpl.Text.cpp +++ b/src/index/IndexImpl.Text.cpp @@ -8,13 +8,13 @@ #include -#include #include #include #include #include #include +#include "backports/algorithm.h" #include "engine/CallFixedSize.h" #include "index/FTSAlgorithms.h" #include "parser/ContextFileParser.h" @@ -724,9 +724,9 @@ IdTable IndexImpl::readWordCl( static_cast(tbmd._cl._startWordlist - tbmd._cl._startContextlist), &TextRecordIndex::make); idTable.resize(cids.size()); - std::ranges::transform(cids, idTable.getColumn(0).begin(), - &Id::makeFromTextRecordIndex); - std::ranges::transform( + ql::ranges::transform(cids, idTable.getColumn(0).begin(), + &Id::makeFromTextRecordIndex); + ql::ranges::transform( readFreqComprList( tbmd._cl._nofElements, tbmd._cl._startWordlist, static_cast(tbmd._cl._startScorelist - @@ -748,16 +748,16 @@ IdTable IndexImpl::readWordEntityCl( tbmd._entityCl._startContextlist), &TextRecordIndex::make); idTable.resize(cids.size()); - std::ranges::transform(cids, idTable.getColumn(0).begin(), - &Id::makeFromTextRecordIndex); - std::ranges::copy( + ql::ranges::transform(cids, idTable.getColumn(0).begin(), + &Id::makeFromTextRecordIndex); + ql::ranges::copy( readFreqComprList(tbmd._entityCl._nofElements, tbmd._entityCl._startWordlist, static_cast(tbmd._entityCl._startScorelist - tbmd._entityCl._startWordlist), &Id::fromBits), idTable.getColumn(1).begin()); - std::ranges::transform( + ql::ranges::transform( readFreqComprList( tbmd._entityCl._nofElements, tbmd._entityCl._startScorelist, static_cast(tbmd._entityCl._lastByte + 1 - @@ -961,7 +961,7 @@ size_t IndexImpl::getSizeEstimate(const string& words) const { } return 1 + optTbmd.value().tbmd_._entityCl._nofElements / 100; }; - return std::ranges::min(terms | std::views::transform(termToEstimate)); + return ql::ranges::min(terms | ql::views::transform(termToEstimate)); } // _____________________________________________________________________________ diff --git a/src/index/IndexImpl.cpp b/src/index/IndexImpl.cpp index c878e4365c..ed8a6dd526 100644 --- a/src/index/IndexImpl.cpp +++ b/src/index/IndexImpl.cpp @@ -6,7 +6,6 @@ #include "./IndexImpl.h" -#include #include #include #include @@ -16,6 +15,7 @@ #include "CompilationInfo.h" #include "Index.h" #include "absl/strings/str_join.h" +#include "backports/algorithm.h" #include "engine/AddCombinedRowToTable.h" #include "engine/CallFixedSize.h" #include "index/IndexFormatVersion.h" @@ -151,7 +151,7 @@ auto fixBlockAfterPatternJoin(auto block) { static constexpr auto permutation = makePermutationFirstThirdSwitched(); block.value().setColumnSubset(permutation); - std::ranges::for_each( + ql::ranges::for_each( block.value().getColumn(ADDITIONAL_COLUMN_INDEX_OBJECT_PATTERN), [](Id& id) { id = id.isUndefined() ? Id::makeFromInt(NO_PATTERN) : id; }); return std::move(block.value()).template toStatic<0>(); @@ -612,6 +612,8 @@ auto IndexImpl::convertPartialToGlobalIds( auto& result = *resultPtr; auto& internalResult = *internalTriplesPtr; auto triplesGenerator = data.getRows(); + // static_assert(!std::is_const_v); + // static_assert(std::is_const_v); auto it = triplesGenerator.begin(); using Buffer = IdTableStatic; struct Buffers { @@ -782,7 +784,7 @@ IndexImpl::createPermutationPairImpl(size_t numColumns, const string& fileName1, // blocks. auto liftCallback = [](auto callback) { return [callback](const auto& block) mutable { - std::ranges::for_each(block, callback); + ql::ranges::for_each(block, callback); }; }; auto callback1 = @@ -1323,7 +1325,7 @@ void IndexImpl::readIndexBuilderSettingsFromFile() { turtleParserIntegerOverflowBehavior_ = TurtleParserIntegerOverflowBehavior::OverflowingToDouble; } else { - AD_CONTRACT_CHECK(std::ranges::find(allModes, value) == allModes.end()); + AD_CONTRACT_CHECK(ql::ranges::find(allModes, value) == allModes.end()); AD_LOG_ERROR << "Invalid value for " << key << std::endl; AD_LOG_INFO << "The currently supported values are " << absl::StrJoin(allModes, ",") << std::endl; diff --git a/src/index/IndexMetaData.h b/src/index/IndexMetaData.h index a3b4cdccd6..7b865fafa7 100644 --- a/src/index/IndexMetaData.h +++ b/src/index/IndexMetaData.h @@ -6,7 +6,6 @@ #include -#include #include #include #include @@ -14,6 +13,7 @@ #include #include +#include "backports/algorithm.h" #include "global/Id.h" #include "index/CompressedRelation.h" #include "index/MetaDataHandler.h" diff --git a/src/index/LocatedTriples.cpp b/src/index/LocatedTriples.cpp index 05353324a5..c8d977f473 100644 --- a/src/index/LocatedTriples.cpp +++ b/src/index/LocatedTriples.cpp @@ -6,9 +6,8 @@ #include "index/LocatedTriples.h" -#include - #include "absl/strings/str_join.h" +#include "backports/algorithm.h" #include "index/CompressedRelation.h" #include "index/ConstantsIndexBuilding.h" #include "util/ChunkedForLoop.h" @@ -29,9 +28,9 @@ std::vector LocatedTriple::locateTriplesInPermutation( // that larger than or equal to the triple. See `LocatedTriples.h` for a // discussion of the corner cases. size_t blockIndex = - std::ranges::lower_bound(blockMetadata, triple.toPermutedTriple(), - std::less<>{}, - &CompressedBlockMetadata::lastTriple_) - + ql::ranges::lower_bound(blockMetadata, triple.toPermutedTriple(), + std::less<>{}, + &CompressedBlockMetadata::lastTriple_) - blockMetadata.begin(); out.emplace_back(blockIndex, triple, shouldExist); }, @@ -53,7 +52,7 @@ NumAddedAndDeleted LocatedTriplesPerBlock::numTriples(size_t blockIndex) const { } auto blockUpdateTriples = map_.at(blockIndex); - size_t countInserts = std::ranges::count_if( + size_t countInserts = ql::ranges::count_if( blockUpdateTriples, &LocatedTriple::shouldTripleExist_); return {countInserts, blockUpdateTriples.size() - countInserts}; } @@ -169,9 +168,9 @@ IdTable LocatedTriplesPerBlock::mergeTriplesImpl(size_t blockIndex, if (locatedTripleIt != locatedTriples.end()) { AD_CORRECTNESS_CHECK(rowIt == block.end()); - std::ranges::for_each( - std::ranges::subrange(locatedTripleIt, locatedTriples.end()) | - std::views::filter(&LocatedTriple::shouldTripleExist_), + ql::ranges::for_each( + ql::ranges::subrange(locatedTripleIt, locatedTriples.end()) | + ql::views::filter(&LocatedTriple::shouldTripleExist_), writeLocatedTripleToResult); } if (rowIt != block.end()) { @@ -290,7 +289,7 @@ static auto updateGraphMetadata(CompressedBlockMetadata& blockMetadata, // Sort the stored graphs. Note: this is currently not expected by the code // that uses the graph info, but makes testing much easier. - std::ranges::sort(graphs.value()); + ql::ranges::sort(graphs.value()); } // ____________________________________________________________________________ @@ -341,14 +340,14 @@ void LocatedTriplesPerBlock::updateAugmentedMetadata() { // ____________________________________________________________________________ std::ostream& operator<<(std::ostream& os, const LocatedTriples& lts) { os << "{ "; - std::ranges::copy(lts, std::ostream_iterator(os, " ")); + ql::ranges::copy(lts, std::ostream_iterator(os, " ")); os << "}"; return os; } // ____________________________________________________________________________ std::ostream& operator<<(std::ostream& os, const std::vector>& v) { - std::ranges::copy(v, std::ostream_iterator>(os, ", ")); + ql::ranges::copy(v, std::ostream_iterator>(os, ", ")); return os; } @@ -362,7 +361,7 @@ bool LocatedTriplesPerBlock::isLocatedTriple(const IdTriple<0>& triple, return ad_utility::contains(lt, locatedTriple); }; - return std::ranges::any_of(map_, [&blockContains](auto& indexAndBlock) { + return ql::ranges::any_of(map_, [&blockContains](auto& indexAndBlock) { const auto& [index, block] = indexAndBlock; return blockContains(block, index); }); diff --git a/src/index/LocatedTriples.h b/src/index/LocatedTriples.h index a424a9de63..0280ce8cc6 100644 --- a/src/index/LocatedTriples.h +++ b/src/index/LocatedTriples.h @@ -185,9 +185,9 @@ class LocatedTriplesPerBlock { const LocatedTriplesPerBlock& ltpb) { // Get the block indices in sorted order. std::vector blockIndices; - std::ranges::copy(ltpb.map_ | std::views::keys, - std::back_inserter(blockIndices)); - std::ranges::sort(blockIndices); + ql::ranges::copy(ltpb.map_ | ql::views::keys, + std::back_inserter(blockIndices)); + ql::ranges::sort(blockIndices); for (auto blockIndex : blockIndices) { os << "LTs in Block #" << blockIndex << ": " << ltpb.map_.at(blockIndex) << std::endl; diff --git a/src/index/PatternCreator.cpp b/src/index/PatternCreator.cpp index fecb4eb2be..03bf20544c 100644 --- a/src/index/PatternCreator.cpp +++ b/src/index/PatternCreator.cpp @@ -72,7 +72,7 @@ void PatternCreator::finishSubject(Id subject, const Pattern& pattern) { // Note: This has to be done for all triples, including those where the // subject has no pattern. auto curSubject = currentSubject_.value(); - std::ranges::for_each( + ql::ranges::for_each( tripleBuffer_, [this, patternId, &curSubject](const auto& t) { static_assert(NumColumnsIndexBuilding == 4, "The following lines have to be changed when additional " @@ -109,11 +109,11 @@ void PatternCreator::finish() { // TODO Use `ranges::to`. std::vector> orderedPatterns{ patternToIdAndCount_.begin(), patternToIdAndCount_.end()}; - std::ranges::sort(orderedPatterns, std::less<>{}, - [](const auto& a) { return a.second.patternId_; }); + ql::ranges::sort(orderedPatterns, std::less<>{}, + [](const auto& a) { return a.second.patternId_; }); CompactVectorOfStrings::Writer patternWriter{ std::move(patternSerializer_).file()}; - for (const auto& pattern : orderedPatterns | std::views::keys) { + for (const auto& pattern : orderedPatterns | ql::views::keys) { patternWriter.push(pattern.data(), pattern.size()); } patternWriter.finish(); diff --git a/src/index/PrefixHeuristic.cpp b/src/index/PrefixHeuristic.cpp index 2353362f9c..e42b75ccf7 100644 --- a/src/index/PrefixHeuristic.cpp +++ b/src/index/PrefixHeuristic.cpp @@ -4,7 +4,6 @@ #include "./PrefixHeuristic.h" -#include #include #include "../parser/RdfEscaping.h" @@ -13,6 +12,7 @@ #include "../util/File.h" #include "../util/Log.h" #include "../util/StringUtils.h" +#include "backports/algorithm.h" using std::string; diff --git a/src/index/StringSortComparator.h b/src/index/StringSortComparator.h index 9f583480a5..da0324ff5e 100644 --- a/src/index/StringSortComparator.h +++ b/src/index/StringSortComparator.h @@ -797,7 +797,7 @@ class TripleComponentComparator { auto alloc = std::pmr::polymorphic_allocator(allocator->resource()); auto ptr = alloc.allocate(s.size()); - std::ranges::copy(s, ptr); + ql::ranges::copy(s, ptr); return {ptr, ptr + s.size()}; }; LocaleManager::SortKeyView sortKey; diff --git a/src/index/StxxlSortFunctors.h b/src/index/StxxlSortFunctors.h index c9d939e2c1..5c7b839d6e 100644 --- a/src/index/StxxlSortFunctors.h +++ b/src/index/StxxlSortFunctors.h @@ -21,7 +21,7 @@ struct SortTriple { } constexpr auto compare = &Id::compareWithoutLocalVocab; // TODO The manual invoking is ugly, probably we could use - // `std::ranges::lexicographical_compare`, but we have to carefully measure + // `ql::ranges::lexicographical_compare`, but we have to carefully measure // that this change doesn't slow down the index build. auto c1 = std::invoke(compare, a[i0], b[i0]); if (c1 != 0) { diff --git a/src/index/Vocabulary.cpp b/src/index/Vocabulary.cpp index 9afc172f19..ab2cb52505 100644 --- a/src/index/Vocabulary.cpp +++ b/src/index/Vocabulary.cpp @@ -30,7 +30,7 @@ Vocabulary::PrefixRanges::PrefixRanges( template bool Vocabulary::PrefixRanges::contain( IndexT index) const { - return std::ranges::any_of( + return ql::ranges::any_of( ranges_, [index](const std::pair& range) { return range.first <= index && index < range.second; }); @@ -125,7 +125,7 @@ bool Vocabulary::shouldEntityBeExternalized( } // Otherwise, externalize if and only if there is a prefix match for one of // `externalizedPrefixes_`. - return std::ranges::any_of(externalizedPrefixes_, [&word](const auto& p) { + return ql::ranges::any_of(externalizedPrefixes_, [&word](const auto& p) { return word.starts_with(p); }); } diff --git a/src/index/Vocabulary.h b/src/index/Vocabulary.h index f2533873a4..3ae8a09da7 100644 --- a/src/index/Vocabulary.h +++ b/src/index/Vocabulary.h @@ -8,7 +8,6 @@ #pragma once -#include #include #include #include @@ -17,6 +16,7 @@ #include #include +#include "backports/algorithm.h" #include "global/Constants.h" #include "global/Id.h" #include "global/Pattern.h" diff --git a/src/index/VocabularyMergerImpl.h b/src/index/VocabularyMergerImpl.h index 31925ab2a0..2b8113daf5 100644 --- a/src/index/VocabularyMergerImpl.h +++ b/src/index/VocabularyMergerImpl.h @@ -94,7 +94,7 @@ auto VocabularyMerger::mergeVocabulary(const std::string& basename, 0.8 * memoryToUse, generators, lessThanForQueue); ad_utility::ProgressBar progressBar{metaData_.numWordsTotal(), "Words merged: "}; - for (QueueWord& currentWord : std::views::join(mergedWords)) { + for (QueueWord& currentWord : ql::views::join(mergedWords)) { // Accumulate the globally ordered queue words in a buffer. sortedBuffer.push_back(std::move(currentWord)); @@ -318,10 +318,10 @@ inline ItemVec vocabMapsToVector(ItemMapArray& map) { futures.push_back( std::async(std::launch::async, [&singleMap, &els, &offsets, i] { using T = ItemVec::value_type; - std::ranges::transform(singleMap.map_, els.begin() + offsets[i], - [](auto& el) -> T { - return {el.first, std::move(el.second)}; - }); + ql::ranges::transform(singleMap.map_, els.begin() + offsets[i], + [](auto& el) -> T { + return {el.first, std::move(el.second)}; + }); })); ++i; } @@ -339,13 +339,13 @@ void sortVocabVector(ItemVec* vecPtr, StringSortComparator comp, auto& els = *vecPtr; if constexpr (USE_PARALLEL_SORT) { if (doParallelSort) { - ad_utility::parallel_sort(std::ranges::begin(els), std::ranges::end(els), + ad_utility::parallel_sort(ql::ranges::begin(els), ql::ranges::end(els), comp, ad_utility::parallel_tag(10)); } else { - std::ranges::sort(els, comp); + ql::ranges::sort(els, comp); } } else { - std::ranges::sort(els, comp); + ql::ranges::sort(els, comp); (void)doParallelSort; // avoid compiler warning for unused value. } } diff --git a/src/index/vocabulary/CompressionWrappers.h b/src/index/vocabulary/CompressionWrappers.h index 21a21f908b..6005f731e7 100644 --- a/src/index/vocabulary/CompressionWrappers.h +++ b/src/index/vocabulary/CompressionWrappers.h @@ -110,7 +110,7 @@ struct PrefixCompressionWrapper : detail::DecoderMultiplexer { static BulkResult compressAll(const Strings& strings) { PrefixCompressor compressor; auto stringsCopy = strings; - std::ranges::sort(stringsCopy); + ql::ranges::sort(stringsCopy); auto prefixes = calculatePrefixes(stringsCopy, NUM_COMPRESSION_PREFIXES, 1, true); compressor.buildCodebook(prefixes); diff --git a/src/index/vocabulary/VocabularyBinarySearchMixin.h b/src/index/vocabulary/VocabularyBinarySearchMixin.h index 2d9f5b0c6e..ccc928a70e 100644 --- a/src/index/vocabulary/VocabularyBinarySearchMixin.h +++ b/src/index/vocabulary/VocabularyBinarySearchMixin.h @@ -4,11 +4,11 @@ #pragma once -#include #include #include #include +#include "backports/algorithm.h" #include "index/vocabulary/VocabularyTypes.h" #include "util/Algorithm.h" @@ -40,7 +40,7 @@ class VocabularyBinarySearchMixin { Idx endIdx = std::nullopt) const { auto [begin, end] = getIterators(beginIdx, endIdx); return impl().iteratorToWordAndIndex( - std::ranges::lower_bound(begin, end, word, comparator)); + ql::ranges::lower_bound(begin, end, word, comparator)); } // Return the first entry that is greater than `word`. The interface is the @@ -51,7 +51,7 @@ class VocabularyBinarySearchMixin { Idx endIdx = std::nullopt) const { auto [begin, end] = getIterators(beginIdx, endIdx); return impl().iteratorToWordAndIndex( - std::ranges::upper_bound(begin, end, word, comparator)); + ql::ranges::upper_bound(begin, end, word, comparator)); } // These functions are similar to `lower_bound` and `upper_bound` (see above), diff --git a/src/index/vocabulary/VocabularyInMemoryBinSearch.cpp b/src/index/vocabulary/VocabularyInMemoryBinSearch.cpp index 204bbfaeed..268cc33721 100644 --- a/src/index/vocabulary/VocabularyInMemoryBinSearch.cpp +++ b/src/index/vocabulary/VocabularyInMemoryBinSearch.cpp @@ -24,7 +24,7 @@ void VocabularyInMemoryBinSearch::open(const string& fileName) { // _____________________________________________________________________________ std::optional VocabularyInMemoryBinSearch::operator[]( uint64_t index) const { - auto it = std::ranges::lower_bound(indices_, index); + auto it = ql::ranges::lower_bound(indices_, index); if (it != indices_.end() && *it == index) { return words_[it - indices_.begin()]; } diff --git a/src/parser/LiteralOrIri.cpp b/src/parser/LiteralOrIri.cpp index 077b189c26..10d2ac1bbe 100644 --- a/src/parser/LiteralOrIri.cpp +++ b/src/parser/LiteralOrIri.cpp @@ -4,9 +4,9 @@ #include "parser/LiteralOrIri.h" -#include #include +#include "backports/algorithm.h" #include "index/IndexImpl.h" namespace ad_utility::triple_component { diff --git a/src/parser/ParsedQuery.cpp b/src/parser/ParsedQuery.cpp index a08fc09f30..71351b43d6 100644 --- a/src/parser/ParsedQuery.cpp +++ b/src/parser/ParsedQuery.cpp @@ -96,10 +96,10 @@ void ParsedQuery::addSolutionModifiers(SolutionModifiers modifiers) { const bool isExplicitGroupBy = !_groupByVariables.empty(); const bool isImplicitGroupBy = - std::ranges::any_of(getAliases(), - [](const Alias& alias) { - return alias._expression.containsAggregate(); - }) && + ql::ranges::any_of(getAliases(), + [](const Alias& alias) { + return alias._expression.containsAggregate(); + }) && !isExplicitGroupBy; const bool isGroupBy = isExplicitGroupBy || isImplicitGroupBy; using namespace std::string_literals; @@ -176,7 +176,7 @@ void ParsedQuery::addSolutionModifiers(SolutionModifiers modifiers) { // part of the group by statement. const auto& aliases = selectClause().getAliases(); for (const Variable& var : selectClause().getSelectedVariables()) { - if (auto it = std::ranges::find(aliases, var, &Alias::_target); + if (auto it = ql::ranges::find(aliases, var, &Alias::_target); it != aliases.end()) { const auto& alias = *it; auto relevantVariables = groupVariables; diff --git a/src/parser/RdfEscaping.cpp b/src/parser/RdfEscaping.cpp index fca4610347..58559f3eff 100644 --- a/src/parser/RdfEscaping.cpp +++ b/src/parser/RdfEscaping.cpp @@ -326,8 +326,8 @@ std::string normalizedContentFromLiteralOrIri(std::string&& input) { static NormalizedString toNormalizedString(std::string_view input) { NormalizedString normalizedString; normalizedString.resize(input.size()); - std::ranges::transform(input.begin(), input.end(), normalizedString.begin(), - [](char c) { return NormalizedChar{c}; }); + ql::ranges::transform(input.begin(), input.end(), normalizedString.begin(), + [](char c) { return NormalizedChar{c}; }); return normalizedString; } diff --git a/src/parser/RdfParser.cpp b/src/parser/RdfParser.cpp index 503919f372..8b3bf0681e 100644 --- a/src/parser/RdfParser.cpp +++ b/src/parser/RdfParser.cpp @@ -1040,8 +1040,8 @@ void RdfParallelParser::initialize(const string& filename) { this->prefixMap_ = std::move(declarationParser.getPrefixMap()); auto remainder = declarationParser.getUnparsedRemainder(); remainingBatchFromInitialization.reserve(remainder.size()); - std::ranges::copy(remainder, - std::back_inserter(remainingBatchFromInitialization)); + ql::ranges::copy(remainder, + std::back_inserter(remainingBatchFromInitialization)); } auto feedBatches = [this, firstBatch = std::move( diff --git a/src/parser/sparqlParser/SparqlQleverVisitor.cpp b/src/parser/sparqlParser/SparqlQleverVisitor.cpp index 55fe72eff2..5629f1452c 100644 --- a/src/parser/sparqlParser/SparqlQleverVisitor.cpp +++ b/src/parser/sparqlParser/SparqlQleverVisitor.cpp @@ -504,8 +504,8 @@ ParsedQuery Visitor::visit(Parser::DeleteWhereContext* ctx) { ParsedQuery Visitor::visit(Parser::ModifyContext* ctx) { auto isVisibleIfVariable = [this](const TripleComponent& component) { if (component.isVariable()) { - return std::ranges::find(parsedQuery_.getVisibleVariables(), - component.getVariable()) != + return ql::ranges::find(parsedQuery_.getVisibleVariables(), + component.getVariable()) != parsedQuery_.getVisibleVariables().end(); } else { return true; @@ -643,8 +643,8 @@ vector Visitor::visit(Parser::QuadsContext* ctx) { ctx->triplesTemplate(), [this](Parser::TriplesTemplateContext* ctx) { return transformTriplesTemplate(ctx, std::monostate{}); }); - std::ranges::move(visitVector(ctx->quadsNotTriples()), - std::back_inserter(triplesWithGraph)); + ql::ranges::move(visitVector(ctx->quadsNotTriples()), + std::back_inserter(triplesWithGraph)); return ad_utility::flatten(std::move(triplesWithGraph)); } @@ -994,7 +994,7 @@ OrderClause Visitor::visit(Parser::OrderClauseContext* ctx) { auto isDescending = [](const auto& variant) { return std::visit([](const auto& k) { return k.isDescending_; }, variant); }; - if (std::ranges::any_of(orderKeys, isDescending)) { + if (ql::ranges::any_of(orderKeys, isDescending)) { reportError(ctx, "When using the `INTERNAL SORT BY` modifier, all sorted " "variables have to be ascending"); @@ -1560,7 +1560,7 @@ vector Visitor::visit( for (auto&& [predicate, object] : std::move(predicateObjectPairs)) { triples.emplace_back(subject, std::move(predicate), std::move(object)); } - std::ranges::copy(additionalTriples, std::back_inserter(triples)); + ql::ranges::copy(additionalTriples, std::back_inserter(triples)); for (const auto& triple : triples) { setMatchingWordAndScoreVisibleIfPresent(ctx, triple); } @@ -1599,8 +1599,8 @@ PathObjectPairsAndTriples Visitor::visit( vector pairsAndTriples = visitVector(ctx->tupleWithoutPath()); for (auto& [newPairs, newTriples] : pairsAndTriples) { - std::ranges::move(newPairs, std::back_inserter(pairs)); - std::ranges::move(newTriples, std::back_inserter(triples)); + ql::ranges::move(newPairs, std::back_inserter(pairs)); + ql::ranges::move(newTriples, std::back_inserter(triples)); } return result; } @@ -1656,16 +1656,16 @@ ObjectsAndPathTriples Visitor::visit(Parser::ObjectListPathContext* ctx) { auto objectAndTriplesVec = visitVector(ctx->objectPath()); // First collect all the objects. std::vector objects; - std::ranges::copy( - objectAndTriplesVec | std::views::transform(ad_utility::first), + ql::ranges::copy( + objectAndTriplesVec | ql::views::transform(ad_utility::first), std::back_inserter(objects)); // Collect all the triples. Node: `views::join` flattens the input. std::vector triples; - std::ranges::copy(objectAndTriplesVec | - std::views::transform(ad_utility::second) | - std::views::join, - std::back_inserter(triples)); + ql::ranges::copy(objectAndTriplesVec | + ql::views::transform(ad_utility::second) | + ql::views::join, + std::back_inserter(triples)); return {std::move(objects), std::move(triples)}; } @@ -2380,7 +2380,7 @@ ExpressionPtr Visitor::visit(Parser::AggregateContext* ctx) { std::string functionName = ad_utility::getLowercase(children.at(0)->getText()); - const bool distinct = std::ranges::any_of(children, [](auto* child) { + const bool distinct = ql::ranges::any_of(children, [](auto* child) { return ad_utility::getLowercase(child->getText()) == "distinct"; }); // the only case that there is no child expression is COUNT(*), so we can diff --git a/src/util/Algorithm.h b/src/util/Algorithm.h index 9a5beabb6a..c2229e0e82 100644 --- a/src/util/Algorithm.h +++ b/src/util/Algorithm.h @@ -6,12 +6,12 @@ #ifndef QLEVER_ALGORITHM_H #define QLEVER_ALGORITHM_H -#include #include #include #include #include +#include "backports/algorithm.h" #include "util/Exception.h" #include "util/Forward.h" #include "util/HashSet.h" @@ -34,8 +34,8 @@ constexpr bool contains(Container&& container, const T& element) { ad_utility::isSimilar) { return container.find(element) != container.npos; } else { - return std::ranges::find(std::begin(container), std::end(container), - element) != std::end(container); + return ql::ranges::find(std::begin(container), std::end(container), + element) != std::end(container); } } @@ -75,7 +75,7 @@ auto transform(Range&& input, F unaryOp) { unaryOp, *ad_utility::makeForwardingIterator(input.begin())))>; std::vector out; out.reserve(input.size()); - std::ranges::transform( + ql::ranges::transform( ad_utility::makeForwardingIterator(input.begin()), ad_utility::makeForwardingIterator(input.end()), std::back_inserter(out), unaryOp); @@ -95,7 +95,7 @@ std::vector> zipVectors(const std::vector& vectorA, std::vector> vectorsPairedUp{}; vectorsPairedUp.reserve(vectorA.size()); - std::ranges::transform( + ql::ranges::transform( vectorA, vectorB, std::back_inserter(vectorsPairedUp), [](const auto& a, const auto& b) { return std::make_pair(a, b); }); diff --git a/src/util/BatchedPipeline.h b/src/util/BatchedPipeline.h index dd5879e326..0bf1c09f2d 100644 --- a/src/util/BatchedPipeline.h +++ b/src/util/BatchedPipeline.h @@ -129,7 +129,7 @@ class Batcher { } res.isPipelineGood_ = true; res.content_.reserve(opt->size()); - std::ranges::move(*opt, std::back_inserter(res.content_)); + ql::ranges::move(*opt, std::back_inserter(res.content_)); return res; } else { res.isPipelineGood_ = true; diff --git a/src/util/BlankNodeManager.cpp b/src/util/BlankNodeManager.cpp index 44295b3aeb..9ff9c45352 100644 --- a/src/util/BlankNodeManager.cpp +++ b/src/util/BlankNodeManager.cpp @@ -59,11 +59,11 @@ bool BlankNodeManager::LocalBlankNodeManager::containsBlankNodeIndex( return index >= block.startIdx_ && index < block.nextIdx_; }; - return std::ranges::any_of(*blocks_, containsIndex) || - std::ranges::any_of( + return ql::ranges::any_of(*blocks_, containsIndex) || + ql::ranges::any_of( otherBlocks_, [&](const std::shared_ptr>& blocks) { - return std::ranges::any_of(*blocks, containsIndex); + return ql::ranges::any_of(*blocks, containsIndex); }); } diff --git a/src/util/BlankNodeManager.h b/src/util/BlankNodeManager.h index afdc748281..3ff7613768 100644 --- a/src/util/BlankNodeManager.h +++ b/src/util/BlankNodeManager.h @@ -97,7 +97,7 @@ class BlankNodeManager { if (l == nullptr) { continue; } - std::ranges::copy(l->otherBlocks_, inserter); + ql::ranges::copy(l->otherBlocks_, inserter); *inserter = l->blocks_; } } diff --git a/src/util/ChunkedForLoop.h b/src/util/ChunkedForLoop.h index 9569633e24..3e2dbdc4e8 100644 --- a/src/util/ChunkedForLoop.h +++ b/src/util/ChunkedForLoop.h @@ -5,10 +5,11 @@ #ifndef QLEVER_CHUNKEDFORLOOP_H #define QLEVER_CHUNKEDFORLOOP_H -#include #include #include +#include "backports/algorithm.h" + namespace ad_utility { namespace detail { @@ -69,7 +70,7 @@ template concept SizedInputRange = std::ranges::sized_range && std::ranges::input_range; -// Similar to `std::ranges::copy`, but invokes `chunkOperation` every +// Similar to `ql::ranges::copy`, but invokes `chunkOperation` every // `chunkSize` elements. (Round up to the next chunk size if the range size is // not a multiple of `chunkSize`.) template @@ -77,16 +78,16 @@ inline void chunkedCopy(R&& inputRange, O result, std::ranges::range_difference_t chunkSize, const std::invocable auto& chunkOperation) requires std::indirectly_copyable, O> { - auto begin = std::ranges::begin(inputRange); - auto end = std::ranges::end(inputRange); + auto begin = ql::ranges::begin(inputRange); + auto end = ql::ranges::end(inputRange); auto target = result; - while (std::ranges::distance(begin, end) >= chunkSize) { + while (ql::ranges::distance(begin, end) >= chunkSize) { auto start = begin; std::ranges::advance(begin, chunkSize); - target = std::ranges::copy(start, begin, target).out; + target = ql::ranges::copy(start, begin, target).out; chunkOperation(); } - std::ranges::copy(begin, end, target); + ql::ranges::copy(begin, end, target); chunkOperation(); } @@ -95,22 +96,22 @@ template concept SizedOutputRange = std::ranges::sized_range && std::ranges::output_range; -// Similar to `std::ranges::fill`, but invokes `chunkOperation` every +// Similar to `ql::ranges::fill`, but invokes `chunkOperation` every // `chunkSize` elements. (Round up to the next chunk size if the range size is // not a multiple of `chunkSize`.) template R> inline void chunkedFill(R&& outputRange, const T& value, std::ranges::range_difference_t chunkSize, const std::invocable auto& chunkOperation) { - auto begin = std::ranges::begin(outputRange); - auto end = std::ranges::end(outputRange); - while (std::ranges::distance(begin, end) >= chunkSize) { + auto begin = ql::ranges::begin(outputRange); + auto end = ql::ranges::end(outputRange); + while (ql::ranges::distance(begin, end) >= chunkSize) { auto start = begin; std::ranges::advance(begin, chunkSize); - std::ranges::fill(start, begin, value); + ql::ranges::fill(start, begin, value); chunkOperation(); } - std::ranges::fill(begin, end, value); + ql::ranges::fill(begin, end, value); chunkOperation(); } } // namespace ad_utility diff --git a/src/util/ConfigManager/ConfigManager.cpp b/src/util/ConfigManager/ConfigManager.cpp index 12d7db1313..9ff8c4d3d8 100644 --- a/src/util/ConfigManager/ConfigManager.cpp +++ b/src/util/ConfigManager/ConfigManager.cpp @@ -9,7 +9,6 @@ #include #include -#include #include #include #include @@ -23,6 +22,7 @@ #include #include +#include "backports/algorithm.h" #include "util/Algorithm.h" #include "util/ComparisonWithNan.h" #include "util/ConfigManager/ConfigExceptions.h" @@ -160,26 +160,26 @@ void ConfigManager::visitHashMapEntries(Visitor&& vis, bool sortByCreationOrder, using Pair = decltype(configurationOptions_)::value_type; // Check the hash map entries before using them. - std::ranges::for_each(configurationOptions_, [&pathPrefix](const Pair& pair) { + ql::ranges::for_each(configurationOptions_, [&pathPrefix](const Pair& pair) { const auto& [jsonPath, hashMapEntry] = pair; verifyHashMapEntry(absl::StrCat(pathPrefix, jsonPath), hashMapEntry); }); - // `std::reference_wrapper` works with `std::ranges::sort`. `const + // `std::reference_wrapper` works with `ql::ranges::sort`. `const // Pair&` does not. std::vector> hashMapEntries( configurationOptions_.begin(), configurationOptions_.end()); // Sort the collected `HashMapEntry`s, if wanted. if (sortByCreationOrder) { - std::ranges::sort(hashMapEntries, {}, [](const Pair& pair) { + ql::ranges::sort(hashMapEntries, {}, [](const Pair& pair) { const HashMapEntry& hashMapEntry = pair.second; return hashMapEntry.getInitializationId(); }); } // Call a wrapper for `vis` with the `HashMapEntry::visit` of every entry. - std::ranges::for_each(hashMapEntries, [&vis](const Pair& pair) { + ql::ranges::for_each(hashMapEntries, [&vis](const Pair& pair) { auto& [jsonPath, hashMapEntry] = pair; hashMapEntry.visit( [&jsonPath, &vis](auto& data) { std::invoke(vis, jsonPath, data); }); @@ -242,14 +242,13 @@ requires std::is_object_v auto ConfigManager::allHashMapEntries( hashMapEntry.getSubManager().value()->configurationOptions_, pathToCurrentEntry, predicate); allHashMapEntry.reserve(recursiveResults.size()); - std::ranges::move(std::move(recursiveResults), - std::back_inserter(allHashMapEntry)); + ql::ranges::move(std::move(recursiveResults), + std::back_inserter(allHashMapEntry)); } }; // Collect all the entries in the given `hashMap`. - std::ranges::for_each(hashMap, addHashMapEntryToCollectedOptions, - verifyEntry); + ql::ranges::for_each(hashMap, addHashMapEntryToCollectedOptions, verifyEntry); return allHashMapEntry; } @@ -299,7 +298,7 @@ std::string ConfigManager::createJsonPointerString( // We don't use a `lazyStrJoin` here, so that an empty `keys` produces an // empty string. - std::ranges::for_each( + ql::ranges::for_each( keys, [&escapeSpecialCharacters, &pointerString](std::string_view key) { pointerString << "/" << escapeSpecialCharacters(key); }); @@ -320,7 +319,7 @@ void ConfigManager::verifyPath(const std::vector& path) const { A string must be a valid `NAME` in the short hand. Otherwise, an option can't get accessed with the short hand. */ - if (auto failedKey = std::ranges::find_if_not(path, isNameInShortHand); + if (auto failedKey = ql::ranges::find_if_not(path, isNameInShortHand); failedKey != path.end()) { /* One of the keys failed. `failedKey` is an iterator pointing to the key. @@ -346,8 +345,8 @@ void ConfigManager::verifyPath(const std::vector& path) const { - The path of an already exiting option/manager is a prefix of the new path. The reasons, why it's not allowed, are basically the same. */ - std::ranges::for_each( - std::views::keys(configurationOptions_), + ql::ranges::for_each( + ql::views::keys(configurationOptions_), [&path, this](std::string_view alreadyAddedPath) { const std::string pathAsJsonPointerString = createJsonPointerString(path); @@ -647,7 +646,7 @@ std::string ConfigManager::generateConfigurationDocDetailedList( if (const auto& validators = assignment.getEntriesUnderKey(key); !validators.empty()) { // Validators should be sorted by their creation order. - AD_CORRECTNESS_CHECK(std::ranges::is_sorted( + AD_CORRECTNESS_CHECK(ql::ranges::is_sorted( validators, {}, [](const ConfigOptionValidatorManager& validator) { return validator.getInitializationId(); })); @@ -729,11 +728,11 @@ auto ConfigManager::getValidatorAssignment() const // Assign to the configuration options. const auto& allValidators = validators(true); - std::ranges::for_each( - std::views::filter(allValidators, - [](const ConfigOptionValidatorManager& val) { - return val.configOptionToBeChecked().size() == 1; - }), + ql::ranges::for_each( + ql::views::filter(allValidators, + [](const ConfigOptionValidatorManager& val) { + return val.configOptionToBeChecked().size() == 1; + }), [&assignment](const ConfigOptionValidatorManager& val) { // The validator manager only has one element, so this should be okay. const ConfigOption& opt = **val.configOptionToBeChecked().begin(); @@ -752,18 +751,17 @@ auto ConfigManager::getValidatorAssignment() const *pair.second.getSubManager().value()); })}; allManager.emplace_back(*this); - std::ranges::for_each( - allManager, [&assignment](const ConfigManager& manager) { - std::ranges::for_each( - std::views::filter( - manager.validators_, - [](const auto& validator) { - return validator.configOptionToBeChecked().size() > 1; - }), - [&assignment, &manager](const auto& validator) { - assignment.addEntryUnderKey(manager, validator); - }); - }); + ql::ranges::for_each(allManager, [&assignment](const ConfigManager& manager) { + ql::ranges::for_each( + ql::views::filter(manager.validators_, + [](const auto& validator) { + return validator.configOptionToBeChecked().size() > + 1; + }), + [&assignment, &manager](const auto& validator) { + assignment.addEntryUnderKey(manager, validator); + }); + }); return assignment; } @@ -802,7 +800,7 @@ std::string ConfigManager::printConfigurationDoc(bool detailed) const { std::string ConfigManager::vectorOfKeysForJsonToString( const std::vector& keys) { std::ostringstream keysToString; - std::ranges::for_each(keys, [&keysToString](std::string_view key) { + ql::ranges::for_each(keys, [&keysToString](std::string_view key) { keysToString << "[" << key << "]"; }); return std::move(keysToString).str(); @@ -822,8 +820,8 @@ ConfigManager::validators(const bool sortByInitialization) const { allSubManager{allHashMapEntries( configurationOptions_, "", [](const HashMapEntry& entry) { return entry.holdsSubManager(); })}; - std::ranges::for_each( - std::views::values(allSubManager), + ql::ranges::for_each( + ql::views::values(allSubManager), [&allValidators](const ConfigManager::HashMapEntry& entry) { appendVector(allValidators, entry.getSubManager().value()->validators(false)); @@ -831,17 +829,17 @@ ConfigManager::validators(const bool sortByInitialization) const { // Sort the validators, if wanted. if (sortByInitialization) { - std::ranges::sort(allValidators, {}, - [](const ConfigOptionValidatorManager& validator) { - return validator.getInitializationId(); - }); + ql::ranges::sort(allValidators, {}, + [](const ConfigOptionValidatorManager& validator) { + return validator.getInitializationId(); + }); } return allValidators; } // ____________________________________________________________________________ void ConfigManager::verifyWithValidators() const { - std::ranges::for_each(validators(false), [](auto& validator) { + ql::ranges::for_each(validators(false), [](auto& validator) { validator.get().checkValidator(); }); }; @@ -850,8 +848,8 @@ void ConfigManager::verifyWithValidators() const { bool ConfigManager::containsOption(const ConfigOption& opt) const { const auto allOptions = configurationOptions(); return ad_utility::contains( - std::views::values(allOptions) | - std::views::transform( + ql::views::values(allOptions) | + ql::views::transform( [](const ConfigOption& option) { return &option; }), &opt); } diff --git a/src/util/ConfigManager/ConfigOption.cpp b/src/util/ConfigManager/ConfigOption.cpp index 26249649f7..4f6e1a9cd5 100644 --- a/src/util/ConfigManager/ConfigOption.cpp +++ b/src/util/ConfigManager/ConfigOption.cpp @@ -98,7 +98,7 @@ void ConfigOption::setValueWithJson(const nlohmann::json& json) { */ return j.is_array() && [&j, &isValueTypeSubType]( const std::vector&) { - return std::ranges::all_of(j, [&isValueTypeSubType](const auto& entry) { + return ql::ranges::all_of(j, [&isValueTypeSubType](const auto& entry) { return isValueTypeSubType.template operator()( entry, AD_FWD(isValueTypeSubType)); }); @@ -177,7 +177,7 @@ std::string ConfigOption::contentOfAvailableTypesToString( stream << "["; ad_utility::lazyStrJoin( &stream, - std::views::transform( + ql::views::transform( content, [&variantSubTypeToString](const VectorEntryType& entry) { return variantSubTypeToString(entry, variantSubTypeToString); diff --git a/src/util/ConstexprMap.h b/src/util/ConstexprMap.h index dfcda81d59..273332dd67 100644 --- a/src/util/ConstexprMap.h +++ b/src/util/ConstexprMap.h @@ -5,9 +5,10 @@ #ifndef QLEVER_CONSTEXPRMAP_H #define QLEVER_CONSTEXPRMAP_H -#include #include +#include "backports/algorithm.h" + namespace ad_utility { /// A const and constexpr map from `Key`s to `Value`s. diff --git a/src/util/ConstexprUtils.h b/src/util/ConstexprUtils.h index 12ece671fe..6661748788 100644 --- a/src/util/ConstexprUtils.h +++ b/src/util/ConstexprUtils.h @@ -7,6 +7,7 @@ #include #include +#include "backports/algorithm.h" #include "util/Exception.h" #include "util/Forward.h" #include "util/TypeTraits.h" @@ -169,7 +170,7 @@ template constexpr std::array integerToArray(Int value, Int numValues) { std::array res; - for (auto& el : res | std::views::reverse) { + for (auto& el : res | ql::views::reverse) { el = value % numValues; value /= numValues; } diff --git a/src/util/FsstCompressor.h b/src/util/FsstCompressor.h index 1e673f33ae..b5c54c4d4d 100644 --- a/src/util/FsstCompressor.h +++ b/src/util/FsstCompressor.h @@ -95,7 +95,7 @@ class FsstRepeatedDecoder { nextInput = result; }; - std::ranges::for_each(std::views::reverse(decoders_), decompressSingle); + ql::ranges::for_each(ql::views::reverse(decoders_), decompressSingle); return result; } // Allow this type to be trivially serializable, diff --git a/src/util/Generator.h b/src/util/Generator.h index bd5f0e32de..604ba7e7c3 100644 --- a/src/util/Generator.h +++ b/src/util/Generator.h @@ -12,6 +12,7 @@ #include #include +#include "backports/algorithm.h" #include "util/Exception.h" #include "util/TypeTraits.h" @@ -115,7 +116,7 @@ class generator_promise { struct generator_sentinel {}; -template +template class generator_iterator { using promise_type = generator_promise; using coroutine_handle = std::coroutine_handle; @@ -180,7 +181,9 @@ template class [[nodiscard]] generator { public: using promise_type = detail::generator_promise; - using iterator = detail::generator_iterator; + using iterator = detail::generator_iterator; + // TODO Check if this fixes anything wrt ::ranges + // using const_iterator = detail::generator_iterator; using value_type = typename iterator::value_type; generator() noexcept : m_coroutine(nullptr) {} @@ -213,10 +216,21 @@ class [[nodiscard]] generator { return iterator{m_coroutine}; } + /* + iterator begin() const; + detail::generator_sentinel end() const; + */ + detail::generator_sentinel end() noexcept { return detail::generator_sentinel{}; } + /* + // Not defined and not useful, but required for range-v3 + const_iterator begin() const; + const_iterator end() const; + */ + void swap(generator& other) noexcept { std::swap(m_coroutine, other.m_coroutine); } diff --git a/src/util/JoinAlgorithms/FindUndefRanges.h b/src/util/JoinAlgorithms/FindUndefRanges.h index bab300a763..7b3f3296cb 100644 --- a/src/util/JoinAlgorithms/FindUndefRanges.h +++ b/src/util/JoinAlgorithms/FindUndefRanges.h @@ -40,9 +40,9 @@ auto findSmallerUndefRangesForRowsWithoutUndef( using Row = typename std::iterator_traits::value_type; assert(row.size() == (*begin).size()); assert( - std::ranges::is_sorted(begin, end, std::ranges::lexicographical_compare)); - assert((std::ranges::all_of( - row, [](Id id) { return id != Id::makeUndefined(); }))); + ql::ranges::is_sorted(begin, end, ql::ranges::lexicographical_compare)); + assert((ql::ranges::all_of(row, + [](Id id) { return id != Id::makeUndefined(); }))); size_t numJoinColumns = row.size(); // TODO This can be done without copying. Row rowLower = row; @@ -56,7 +56,7 @@ auto findSmallerUndefRangesForRowsWithoutUndef( } auto [begOfUndef, endOfUndef] = std::equal_range( - begin, end, rowLower, std::ranges::lexicographical_compare); + begin, end, rowLower, ql::ranges::lexicographical_compare); for (auto it = begOfUndef; it != endOfUndef; ++it) { co_yield it; } @@ -80,7 +80,7 @@ auto findSmallerUndefRangesForRowsWithUndefInLastColumns( assert(row.size() == (*begin).size()); assert(numJoinColumns >= numLastUndefined); assert( - std::ranges::is_sorted(begin, end, std::ranges::lexicographical_compare)); + ql::ranges::is_sorted(begin, end, ql::ranges::lexicographical_compare)); const size_t numDefinedColumns = numJoinColumns - numLastUndefined; for (size_t i = 0; i < numDefinedColumns; ++i) { assert(row[i] != Id::makeUndefined()); @@ -107,11 +107,11 @@ auto findSmallerUndefRangesForRowsWithUndefInLastColumns( } auto begOfUndef = std::lower_bound(begin, end, rowLower, - std::ranges::lexicographical_compare); + ql::ranges::lexicographical_compare); rowLower[numDefinedColumns - 1] = Id::fromBits(rowLower[numDefinedColumns - 1].getBits() + 1); auto endOfUndef = std::lower_bound(begin, end, rowLower, - std::ranges::lexicographical_compare); + ql::ranges::lexicographical_compare); for (; begOfUndef != endOfUndef; ++begOfUndef) { resultMightBeUnsorted = true; co_yield begOfUndef; @@ -127,12 +127,12 @@ auto findSmallerUndefRangesArbitrary(const auto& row, It begin, It end, -> cppcoro::generator { assert(row.size() == (*begin).size()); assert( - std::ranges::is_sorted(begin, end, std::ranges::lexicographical_compare)); + ql::ranges::is_sorted(begin, end, ql::ranges::lexicographical_compare)); // To only get smaller entries, we first find a suitable upper bound in the // input range. We use `std::lower_bound` because the input row itself is not // a valid match. - end = std::lower_bound(begin, end, row, std::ranges::lexicographical_compare); + end = std::lower_bound(begin, end, row, ql::ranges::lexicographical_compare); const size_t numJoinColumns = row.size(); auto isCompatible = [&](const auto& otherRow) { @@ -171,8 +171,8 @@ auto findSmallerUndefRanges(const auto& row, It begin, It end, -> cppcoro::generator { size_t numLastUndefined = 0; assert(row.size() > 0); - auto it = std::ranges::rbegin(row); - auto rend = std::ranges::rend(row); + auto it = ql::ranges::rbegin(row); + auto rend = ql::ranges::rend(row); for (; it < rend; ++it) { if (*it != Id::makeUndefined()) { break; diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h index 7231f9396f..36af501651 100644 --- a/src/util/JoinAlgorithms/JoinAlgorithms.h +++ b/src/util/JoinAlgorithms/JoinAlgorithms.h @@ -4,11 +4,11 @@ #pragma once -#include #include #include #include +#include "backports/algorithm.h" #include "engine/idTable/IdTable.h" #include "global/Id.h" #include "util/Generator.h" @@ -160,7 +160,7 @@ template ) { return row != Id::makeUndefined(); } else { - return (std::ranges::none_of( + return (ql::ranges::none_of( row, [](Id id) { return id == Id::makeUndefined(); })); } }; @@ -525,7 +525,7 @@ void specialOptionalJoin( // TODO We could probably also apply this optimization if both // inputs contain UNDEF values only in the last column, and possibly // also not only for `OPTIONAL` joins. - auto endOfUndef = std::ranges::find_if_not(leftSub, &Id::isUndefined); + auto endOfUndef = ql::ranges::find_if_not(leftSub, &Id::isUndefined); auto findSmallerUndefRangeLeft = [leftSub, endOfUndef](auto&&...) -> cppcoro::generator { @@ -596,22 +596,22 @@ class BlockAndSubrange { bool empty() const { return subrange_.second == subrange_.first; } - // Return the currently specified subrange as a `std::ranges::subrange` + // Return the currently specified subrange as a `ql::ranges::subrange` // object. auto subrange() { - return std::ranges::subrange{fullBlock().begin() + subrange_.first, - fullBlock().begin() + subrange_.second}; + return ql::ranges::subrange{fullBlock().begin() + subrange_.first, + fullBlock().begin() + subrange_.second}; } // The const overload of the `subrange` method (see above). auto subrange() const { - return std::ranges::subrange{fullBlock().begin() + subrange_.first, - fullBlock().begin() + subrange_.second}; + return ql::ranges::subrange{fullBlock().begin() + subrange_.first, + fullBlock().begin() + subrange_.second}; } // Get a view that iterates over all the indices that belong to the subrange. auto getIndexRange() const { - return std::views::iota(subrange_.first, subrange_.second); + return ql::views::iota(subrange_.first, subrange_.second); } Range getIndices() const { return subrange_; } @@ -649,7 +649,7 @@ class BlockAndSubrange { // Overload of `setSubrange` for an actual subrange object. template void setSubrange(const Subrange& subrange) { - setSubrange(std::ranges::begin(subrange), std::ranges::end(subrange)); + setSubrange(ql::ranges::begin(subrange), ql::ranges::end(subrange)); } }; @@ -683,7 +683,7 @@ JoinSide(It, End, const Projection&) -> JoinSide; // keeping them valid until the join is completed. template auto makeJoinSide(Blocks& blocks, const auto& projection) { - return JoinSide{std::ranges::begin(blocks), std::ranges::end(blocks), + return JoinSide{ql::ranges::begin(blocks), ql::ranges::end(blocks), projection}; } @@ -799,14 +799,14 @@ struct BlockZipperJoinImpl { auto& end = side.end_; for (size_t numBlocksRead = 0; it != end && numBlocksRead < 3; ++it, ++numBlocksRead) { - if (std::ranges::empty(*it)) { + if (ql::ranges::empty(*it)) { continue; } if (!eq((*it)[0], currentEl)) { AD_CORRECTNESS_CHECK(lessThan_(currentEl, (*it)[0])); return true; } - AD_CORRECTNESS_CHECK(std::ranges::is_sorted(*it, lessThan_)); + AD_CORRECTNESS_CHECK(ql::ranges::is_sorted(*it, lessThan_)); side.currentBlocks_.emplace_back(std::move(*it)); } return it == end; @@ -850,7 +850,7 @@ struct BlockZipperJoinImpl { // Delete the part from the last block that is `<= lastProcessedElement`. decltype(auto) remainingBlock = blocks.at(0).subrange(); - auto beginningOfUnjoined = std::ranges::upper_bound( + auto beginningOfUnjoined = ql::ranges::upper_bound( remainingBlock, lastProcessedElement, lessThan_); blocks.at(0).setSubrange(beginningOfUnjoined, remainingBlock.end()); if (blocks.at(0).empty()) { @@ -867,7 +867,9 @@ struct BlockZipperJoinImpl { const ProjectedEl& currentEl) { AD_CORRECTNESS_CHECK(!currentBlocks.empty()); const auto& first = currentBlocks.at(0); - auto it = std::ranges::lower_bound(first.subrange(), currentEl, lessThan_); + // TODO ql::ranges::lower_bound doesn't work here. + auto it = std::lower_bound(first.subrange().begin(), first.subrange().end(), + currentEl, lessThan_); return std::tuple{std::ref(first.fullBlock()), first.subrange(), it}; } @@ -883,7 +885,7 @@ struct BlockZipperJoinImpl { // blocks on the right and add them to the result. void addCartesianProduct(const LeftBlocks& blocksLeft, const RightBlocks& blocksRight) { - // TODO use `std::views::cartesian_product`. + // TODO use `ql::views::cartesian_product`. for (const auto& lBlock : blocksLeft) { for (const auto& rBlock : blocksRight) { compatibleRowAction_.setInput(lBlock.fullBlock(), rBlock.fullBlock()); @@ -903,10 +905,10 @@ struct BlockZipperJoinImpl { const LeftBlocks& blocksLeft, const RightBlocks& blocksRight) { if constexpr (DoOptionalJoin) { if (!hasUndef(rightSide_) && - std::ranges::all_of( - blocksRight | std::views::transform( + ql::ranges::all_of( + blocksRight | ql::views::transform( [](const auto& inp) { return inp.subrange(); }), - std::ranges::empty)) { + ql::ranges::empty)) { for (const auto& lBlock : blocksLeft) { compatibleRowAction_.setOnlyLeftInputForOptionalJoin( lBlock.fullBlock()); @@ -938,8 +940,11 @@ struct BlockZipperJoinImpl { return result; } auto& last = result.back(); - last.setSubrange( - std::ranges::equal_range(last.subrange(), currentEl, lessThan_)); + // TODO `ql::ranges::equal_range` doesn't work here for some + // reason. + auto [begin, end] = std::equal_range( + last.subrange().begin(), last.subrange().end(), currentEl, lessThan_); + last.setSubrange(begin, end); return result; } @@ -1035,16 +1040,16 @@ struct BlockZipperJoinImpl { // yields iterators to the individual undefined values. if constexpr (potentiallyHasUndef) { [[maybe_unused]] auto res = zipperJoinWithUndef( - std::ranges::subrange{subrangeLeft.begin(), currentElItL}, - std::ranges::subrange{subrangeRight.begin(), currentElItR}, lessThan_, + ql::ranges::subrange{subrangeLeft.begin(), currentElItL}, + ql::ranges::subrange{subrangeRight.begin(), currentElItR}, lessThan_, addRowIndex, findUndefValues(fullBlockLeft, fullBlockRight, begL, begR), findUndefValues(fullBlockLeft, fullBlockRight, begL, begR), addNotFoundRowIndex); } else { [[maybe_unused]] auto res = zipperJoinWithUndef( - std::ranges::subrange{subrangeLeft.begin(), currentElItL}, - std::ranges::subrange{subrangeRight.begin(), currentElItR}, lessThan_, + ql::ranges::subrange{subrangeLeft.begin(), currentElItL}, + ql::ranges::subrange{subrangeRight.begin(), currentElItR}, lessThan_, addRowIndex, noop, noop, addNotFoundRowIndex); } compatibleRowAction_.flush(); @@ -1063,7 +1068,7 @@ struct BlockZipperJoinImpl { while (targetBuffer.empty() && it != end) { auto& el = *it; if (!el.empty()) { - AD_CORRECTNESS_CHECK(std::ranges::is_sorted(el, lessThan_)); + AD_CORRECTNESS_CHECK(ql::ranges::is_sorted(el, lessThan_)); targetBuffer.emplace_back(std::move(el)); } ++it; @@ -1245,7 +1250,7 @@ struct BlockZipperJoinImpl { // The reference of `it` is there on purpose. for (auto& it = side.it_; it != side.end_; ++it) { auto& el = *it; - if (std::ranges::empty(el) || !isUndefined_(el.front())) { + if (ql::ranges::empty(el) || !isUndefined_(el.front())) { return; } bool endIsUndefined = isUndefined_(el.back()); @@ -1253,11 +1258,12 @@ struct BlockZipperJoinImpl { if (!endIsUndefined) { auto& lastUndefinedBlock = side.undefBlocks_.back(); side.currentBlocks_.push_back(lastUndefinedBlock); - auto subrange = std::ranges::equal_range( - lastUndefinedBlock.subrange(), - lastUndefinedBlock.subrange().front(), lessThan_); - size_t undefCount = std::ranges::size(subrange); - lastUndefinedBlock.setSubrange(std::move(subrange)); + // TODO ql::ranges::equal_range doesn't work for some reason. + decltype(auto) subrange = lastUndefinedBlock.subrange(); + auto [begin, end] = std::equal_range(subrange.begin(), subrange.end(), + subrange.front(), lessThan_); + size_t undefCount = std::distance(begin, end); + lastUndefinedBlock.setSubrange(begin, end); auto& firstDefinedBlock = side.currentBlocks_.back(); firstDefinedBlock.setSubrange( firstDefinedBlock.fullBlock().begin() + undefCount, diff --git a/src/util/MemorySize/MemorySize.h b/src/util/MemorySize/MemorySize.h index 59d7e47a19..92e2f01465 100644 --- a/src/util/MemorySize/MemorySize.h +++ b/src/util/MemorySize/MemorySize.h @@ -7,7 +7,6 @@ #include -#include #include #include #include @@ -17,6 +16,7 @@ #include #include +#include "backports/algorithm.h" #include "util/ConstexprMap.h" #include "util/ConstexprUtils.h" #include "util/Exception.h" diff --git a/src/util/ParallelMultiwayMerge.h b/src/util/ParallelMultiwayMerge.h index 21f4d85d08..27db2e4e69 100644 --- a/src/util/ParallelMultiwayMerge.h +++ b/src/util/ParallelMultiwayMerge.h @@ -71,7 +71,7 @@ cppcoro::generator> lazyBinaryMerge( // Turn the ranges into `(iterator, end)` pairs. auto makeItPair = [](auto& range) { - return std::pair{std::ranges::begin(range), std::ranges::end(range)}; + return std::pair{ql::ranges::begin(range), ql::ranges::end(range)}; }; auto it1 = makeItPair(range1); @@ -126,7 +126,7 @@ cppcoro::generator> lazyBinaryMerge( auto yieldRemainder = [&buffer, &isBufferLargeEnough, &clearBuffer, &pushToBuffer](auto& itPair) -> cppcoro::generator> { - for (auto& el : std::ranges::subrange(itPair.first, itPair.second)) { + for (auto& el : ql::ranges::subrange(itPair.first, itPair.second)) { pushToBuffer(el); if (isBufferLargeEnough()) { co_yield buffer; @@ -194,20 +194,19 @@ cppcoro::generator> parallelMultiwayMergeImpl( maxMemPerNode, blocksize, moveIf(rangeOfRanges[0]), moveIf(rangeOfRanges[1]), comparison); } else { - size_t size = std::ranges::size(rangeOfRanges); + size_t size = ql::ranges::size(rangeOfRanges); size_t split = size / 2; auto beg = rangeOfRanges.begin(); auto splitIt = beg + split; auto end = rangeOfRanges.end(); auto join = [](auto&& view) { - return std::views::join(ad_utility::OwningView{AD_FWD(view)}); + return ql::views::join(ad_utility::OwningView{AD_FWD(view)}); }; auto parallelMerge = [join, blocksize, comparison, maxMemPerNode]( auto it, auto end) { return join(parallelMultiwayMergeImpl( - maxMemPerNode, blocksize, std::ranges::subrange{it, end}, - comparison)); + maxMemPerNode, blocksize, ql::ranges::subrange{it, end}, comparison)); }; return ad_utility::streams::runStreamAsync( @@ -233,7 +232,7 @@ cppcoro::generator> parallelMultiwayMerge( size_t blocksize = 100) { // There is one suboperation per input in the recursion tree, so we have to // divide the memory limit. - auto maxMemPerNode = memoryLimit / std::ranges::size(rangeOfRanges); + auto maxMemPerNode = memoryLimit / ql::ranges::size(rangeOfRanges); return detail::parallelMultiwayMergeImpl( maxMemPerNode, blocksize, AD_FWD(rangeOfRanges), std::move(comparison)); } diff --git a/src/util/PriorityQueue.h b/src/util/PriorityQueue.h index bbf4359be6..fd78cc63ca 100644 --- a/src/util/PriorityQueue.h +++ b/src/util/PriorityQueue.h @@ -16,7 +16,6 @@ #pragma once -#include #include #include #include @@ -25,6 +24,7 @@ #include "./Exception.h" #include "./HashMap.h" #include "./Log.h" +#include "backports/algorithm.h" namespace ad_utility { using std::make_shared; diff --git a/src/util/Random.h b/src/util/Random.h index 0be93b0f81..b5af0d3d66 100644 --- a/src/util/Random.h +++ b/src/util/Random.h @@ -6,7 +6,6 @@ #pragma once -#include #include #include #include @@ -15,6 +14,7 @@ #include #include +#include "backports/algorithm.h" #include "global/TypedIndex.h" namespace ad_utility { diff --git a/src/util/Serializer/ByteBufferSerializer.h b/src/util/Serializer/ByteBufferSerializer.h index 6ab5f80ad1..263621437d 100644 --- a/src/util/Serializer/ByteBufferSerializer.h +++ b/src/util/Serializer/ByteBufferSerializer.h @@ -5,12 +5,12 @@ #ifndef QLEVER_BYTEBUFFERSERIALIZER_H #define QLEVER_BYTEBUFFERSERIALIZER_H -#include #include #include #include "../Exception.h" #include "./Serializer.h" +#include "backports/algorithm.h" namespace ad_utility::serialization { /** diff --git a/src/util/Simple8bCode.h b/src/util/Simple8bCode.h index ff13bfb8e9..8af2011776 100644 --- a/src/util/Simple8bCode.h +++ b/src/util/Simple8bCode.h @@ -6,7 +6,7 @@ #include #include -#include +#include "backports/algorithm.h" namespace ad_utility { diff --git a/src/util/StringUtils.cpp b/src/util/StringUtils.cpp index 5ea3aaa85e..bf3aafe7b7 100644 --- a/src/util/StringUtils.cpp +++ b/src/util/StringUtils.cpp @@ -5,9 +5,15 @@ #include "util/StringUtils.h" +#include #include #include +#include "util/Algorithm.h" +#include "util/Exception.h" +#include "util/Forward.h" +#include "util/StringUtilsImpl.h" + namespace ad_utility { // ____________________________________________________________________________ string_view commonPrefix(string_view a, const string_view b) { @@ -55,10 +61,10 @@ bool isLanguageMatch(string& languageTag, string& languageRange) { if (languageRange.ends_with("*")) { languageRange.pop_back(); } - std::ranges::transform(languageTag, std::begin(languageTag), - [](unsigned char c) { return std::tolower(c); }); - std::ranges::transform(languageRange, std::begin(languageRange), - [](unsigned char c) { return std::tolower(c); }); + ql::ranges::transform(languageTag, std::begin(languageTag), + [](unsigned char c) { return std::tolower(c); }); + ql::ranges::transform(languageRange, std::begin(languageRange), + [](unsigned char c) { return std::tolower(c); }); return languageTag.compare(0, languageRange.length(), languageRange) == 0; } } @@ -192,5 +198,4 @@ std::string addIndentation(std::string_view str, absl::StrReplaceAll(str, {{"\n", absl::StrCat("\n", indentationSymbol)}})); } - } // namespace ad_utility diff --git a/src/util/StringUtils.h b/src/util/StringUtils.h index cd0fcaf250..493e7759cc 100644 --- a/src/util/StringUtils.h +++ b/src/util/StringUtils.h @@ -9,7 +9,7 @@ #include -#include "util/Algorithm.h" +#include "backports/algorithm.h" #include "util/Concepts.h" #include "util/ConstexprSmallString.h" #include "util/CtreHelpers.h" @@ -137,6 +137,12 @@ number. @param str The input string. @param separatorSymbol What symbol to put between groups of thousands. + +Note: To avoid cyclic dependencies, this function is defined in a separate file +`StringUtilsImpl.h`. This file is then included in the `StringUtils.cpp` with an +explicit instantiation for the default template argument `.`. The tests include +the impl file directly to exhaustively test the behavior for other template +arguments. */ template std::string insertThousandSeparator(const std::string_view str, @@ -201,11 +207,11 @@ void lazyStrJoin(std::ostream* stream, Range&& r, std::string_view separator) { // Add the remaining entries. ++begin; - std::ranges::for_each(begin, end, - [&stream, &separator](const auto& listItem) { - *stream << separator << listItem; - }, - {}); + ql::ranges::for_each(begin, end, + [&stream, &separator](const auto& listItem) { + *stream << separator << listItem; + }, + {}); } // _________________________________________________________________________ @@ -218,88 +224,6 @@ std::string lazyStrJoin(Range&& r, std::string_view separator) { return std::move(stream).str(); } -// ___________________________________________________________________________ -template -std::string insertThousandSeparator(const std::string_view str, - const char separatorSymbol) { - static const auto isDigit = [](const char c) { - // `char` is ASCII. So the number symbols are the codes from 48 to 57. - return '0' <= c && c <= '9'; - }; - AD_CONTRACT_CHECK(!isDigit(separatorSymbol) && - !isDigit(floatingPointSignifier)); - - /* - Create a `ctll::fixed_string` of `floatingPointSignifier`, that can be used - inside regex character classes, without being confused with one of the - reserved characters. - */ - static constexpr auto adjustFloatingPointSignifierForRegex = []() { - constexpr ctll::fixed_string floatingPointSignifierAsFixedString( - {floatingPointSignifier, '\0'}); - - // Inside a regex character class are fewer reserved character. - if constexpr (contains(R"--(^-[]\)--", floatingPointSignifier)) { - return "\\" + floatingPointSignifierAsFixedString; - } else { - return floatingPointSignifierAsFixedString; - } - }; - - /* - As string view doesn't support the option to insert new values between old - values, so we create a new string in the wanted format. - */ - std::ostringstream ostream; - - /* - Insert separator into the given string and add it into the `ostream`. Ignores - content of the given string, just works based on length. - */ - auto insertSeparator = [&separatorSymbol, - &ostream](const std::string_view s) { - // Nothing to do, if the string is to short. - AD_CORRECTNESS_CHECK(s.length() > 3); - - /* - For walking over the string view. - Our initialization value skips the leading digits, so that only the digits - remain, where we have to put the separator in front of every three chars. - */ - size_t currentIdx{s.length() % 3 == 0 ? 3 : s.length() % 3}; - ostream << s.substr(0, currentIdx); - for (; currentIdx < s.length(); currentIdx += 3) { - ostream << separatorSymbol << s.substr(currentIdx, 3); - } - }; - - /* - The pattern finds any digit sequence, that is long enough for inserting - thousand separators and is not the fractual part of a floating point. - */ - static constexpr ctll::fixed_string regexPatDigitSequence{ - "(?:^|[^\\d" + adjustFloatingPointSignifierForRegex() + - "])(?\\d{4,})"}; - auto parseIterator = std::begin(str); - std::ranges::for_each( - ctre::range(str), - [&parseIterator, &ostream, &insertSeparator](const auto& match) { - /* - The digit sequence, that must be transformed. Note: The string view - iterators point to entries in the `str` string. - */ - const std::string_view& digitSequence{match.template get<"digit">()}; - - // Insert the transformed digit sequence, and the string between it and - // the `parseIterator`, into the stream. - ostream << std::string_view(parseIterator, std::begin(digitSequence)); - insertSeparator(digitSequence); - parseIterator = std::end(digitSequence); - }); - ostream << std::string_view(std::move(parseIterator), std::end(str)); - return ostream.str(); -} - // The implementation of `constexprStrCat` below. namespace detail::constexpr_str_cat_impl { // We currently have a fixed upper bound of 100 characters on the inputs. @@ -308,8 +232,8 @@ namespace detail::constexpr_str_cat_impl { // more complicated. using ConstexprString = ad_utility::ConstexprSmallString<100>; -// Concatenate the elements of `arr` into a single array with an additional zero -// byte at the end. `sz` must be the sum of the sizes in `arr`, else the +// Concatenate the elements of `arr` into a single array with an additional +// zero byte at the end. `sz` must be the sum of the sizes in `arr`, else the // behavior is undefined. template constexpr std::array catImpl( @@ -324,8 +248,8 @@ constexpr std::array catImpl( } return buf; }; -// Concatenate the `strings` into a single `std::array` with an additional -// zero byte at the end. +// Concatenate the `strings` into a single `std::array` with an +// additional zero byte at the end. template constexpr auto constexprStrCatBufferImpl() { constexpr size_t sz = (size_t{0} + ... + strings.size()); @@ -344,8 +268,8 @@ constexpr inline auto constexprStrCatBufferVar = // Return the concatenation of the `strings` as a `string_view`. Can be // evaluated at compile time. The buffer that backs the returned `string_view` -// will be zero-terminated, so it is safe to pass pointers into the result into -// legacy C-APIs. +// will be zero-terminated, so it is safe to pass pointers into the result +// into legacy C-APIs. template constexpr std::string_view constexprStrCat() { const auto& b = diff --git a/src/util/StringUtilsImpl.h b/src/util/StringUtilsImpl.h new file mode 100644 index 0000000000..87db3aa645 --- /dev/null +++ b/src/util/StringUtilsImpl.h @@ -0,0 +1,97 @@ +// Copyright 2023, University of Freiburg, Chair of Algorithms and Data +// Structures. +// Authors: Andre Schlegel (schlegea@informatik.uni-freiburg.de) +// Johannes Kalmbach, kalmbach@cs.uni-freiburg.de + +#pragma once + +#include "util/Algorithm.h" +#include "util/Exception.h" +#include "util/StringUtils.h" + +namespace ad_utility { +// _____________________________________________________________________________ +template +std::string insertThousandSeparator(const std::string_view str, + const char separatorSymbol) { + static const auto isDigit = [](const char c) { + // `char` is ASCII. So the number symbols are the codes from 48 to 57. + return '0' <= c && c <= '9'; + }; + AD_CONTRACT_CHECK(!isDigit(separatorSymbol) && + !isDigit(floatingPointSignifier)); + + /* + Create a `ctll::fixed_string` of `floatingPointSignifier`, that can be used + inside regex character classes, without being confused with one of the + reserved characters. + */ + static constexpr auto adjustFloatingPointSignifierForRegex = []() { + constexpr ctll::fixed_string floatingPointSignifierAsFixedString( + {floatingPointSignifier, '\0'}); + + // Inside a regex character class are fewer reserved character. + if constexpr (contains(R"--(^-[]\)--", floatingPointSignifier)) { + return "\\" + floatingPointSignifierAsFixedString; + } else { + return floatingPointSignifierAsFixedString; + } + }; + + /* + As string view doesn't support the option to insert new values between old + values, so we create a new string in the wanted format. + */ + std::ostringstream ostream; + + /* + Insert separator into the given string and add it into the `ostream`. + Ignores content of the given string, just works based on length. + */ + auto insertSeparator = [&separatorSymbol, + &ostream](const std::string_view s) { + // Nothing to do, if the string is to short. + AD_CORRECTNESS_CHECK(s.length() > 3); + + /* + For walking over the string view. + Our initialization value skips the leading digits, so that only the digits + remain, where we have to put the separator in front of every three chars. + */ + size_t currentIdx{s.length() % 3 == 0 ? 3 : s.length() % 3}; + ostream << s.substr(0, currentIdx); + for (; currentIdx < s.length(); currentIdx += 3) { + ostream << separatorSymbol << s.substr(currentIdx, 3); + } + }; + + /* + The pattern finds any digit sequence, that is long enough for inserting + thousand separators and is not the fractual part of a floating point. + */ + static constexpr ctll::fixed_string regexPatDigitSequence{ + "(?:^|[^\\d" + adjustFloatingPointSignifierForRegex() + + "])(?\\d{4,})"}; + auto parseIterator = std::begin(str); + ql::ranges::for_each( + ctre::range(str), + [&parseIterator, &ostream, &insertSeparator](const auto& match) { + /* + The digit sequence, that must be transformed. Note: The string view + iterators point to entries in the `str` string. + */ + const std::string_view& digitSequence{match.template get<"digit">()}; + + // Insert the transformed digit sequence, and the string between it + // and the `parseIterator`, into the stream. + ostream << std::string_view(parseIterator, std::begin(digitSequence)); + insertSeparator(digitSequence); + parseIterator = std::end(digitSequence); + }); + ostream << std::string_view(std::move(parseIterator), std::end(str)); + return ostream.str(); +} + +template std::string insertThousandSeparator<'.'>(const std::string_view str, + const char separatorSymbol); +} // namespace ad_utility diff --git a/src/util/TaskQueue.h b/src/util/TaskQueue.h index 878f13c70e..c18b0a4e8d 100644 --- a/src/util/TaskQueue.h +++ b/src/util/TaskQueue.h @@ -145,7 +145,7 @@ class TaskQueue { // that set `startedFinishing_` from false to true. void finishImpl() { queuedTasks_.finish(); - std::ranges::for_each(threads_, [](auto& thread) { + ql::ranges::for_each(threads_, [](auto& thread) { // If `finish` was called from inside the queue, the calling thread cannot // join itself. AD_CORRECTNESS_CHECK(thread.joinable()); diff --git a/src/util/ThreadSafeQueue.h b/src/util/ThreadSafeQueue.h index f24e2717f1..6c6828d411 100644 --- a/src/util/ThreadSafeQueue.h +++ b/src/util/ThreadSafeQueue.h @@ -274,7 +274,7 @@ cppcoro::generator queueManager(size_t queueSize, std::vector threads; std::atomic numUnfinishedThreads{static_cast(numThreads)}; absl::Cleanup queueFinisher{[&queue] { queue.finish(); }}; - for ([[maybe_unused]] auto i : std::views::iota(0u, numThreads)) { + for ([[maybe_unused]] auto i : ql::views::iota(0u, numThreads)) { threads.emplace_back( detail::makeQueueTask(queue, producer, numUnfinishedThreads)); } diff --git a/src/util/Views.h b/src/util/Views.h index 8a075c179e..4666a28228 100644 --- a/src/util/Views.h +++ b/src/util/Views.h @@ -8,6 +8,8 @@ #include #include +#include "backports/algorithm.h" +#include "backports/concepts.h" #include "util/Generator.h" #include "util/Log.h" @@ -83,8 +85,8 @@ cppcoro::generator uniqueView(SortedView view) { // Takes a view of blocks and yields the elements of the same view, but removes // consecutive duplicates inside the blocks and across block boundaries. template >> + typename ValueType = ql::ranges::range_value_t< + ql::ranges::range_value_t>> cppcoro::generator uniqueBlockView( SortedBlockView view) { size_t numInputs = 0; @@ -97,7 +99,7 @@ cppcoro::generator uniqueBlockView( } numInputs += block.size(); auto beg = lastValueFromPreviousBlock - ? std::ranges::find_if( + ? ql::ranges::find_if( block, [&p = lastValueFromPreviousBlock.value()]( const auto& el) { return el != p; }) : block.begin(); @@ -113,11 +115,14 @@ cppcoro::generator uniqueBlockView( } // A view that owns its underlying storage. It is a replacement for -// `std::ranges::owning_view` which is not yet supported by `GCC 11`. The -// implementation is taken from libstdc++-13. -template -requires std::movable class OwningView - : public std::ranges::view_interface> { +// `std::ranges::owning_view` which is not yet supported by `GCC 11` and +// `range-v3`. The implementation is taken from libstdc++-13. The additional +// optional `supportsConst` argument explicitly disables const iteration for +// this view when set to false, see `OwningViewNoConst` below for details. +CPP_template(typename UnderlyingRange, bool supportConst = true)( + requires ql::ranges::range CPP_and + ql::concepts::movable) class OwningView + : public ql::ranges::view_interface> { private: UnderlyingRange underlyingRange_ = UnderlyingRange(); @@ -145,93 +150,107 @@ requires std::movable class OwningView return std::move(underlyingRange_); } - constexpr std::ranges::iterator_t begin() { - return std::ranges::begin(underlyingRange_); + constexpr ql::ranges::iterator_t begin() { + return ql::ranges::begin(underlyingRange_); } - constexpr std::ranges::sentinel_t end() { - return std::ranges::end(underlyingRange_); + constexpr ql::ranges::sentinel_t end() { + return ql::ranges::end(underlyingRange_); } constexpr auto begin() const - requires std::ranges::range { - return std::ranges::begin(underlyingRange_); + requires(supportConst && ql::ranges::range) { + return ql::ranges::begin(underlyingRange_); } - constexpr auto end() const requires std::ranges::range - { - return std::ranges::end(underlyingRange_); + constexpr auto end() const + requires(supportConst && ql::ranges::range) { + return ql::ranges::end(underlyingRange_); } constexpr bool empty() - requires requires { std::ranges::empty(underlyingRange_); } { - return std::ranges::empty(underlyingRange_); + requires requires { ql::ranges::empty(underlyingRange_); } { + return ql::ranges::empty(underlyingRange_); } constexpr bool empty() const - requires requires { std::ranges::empty(underlyingRange_); } { - return std::ranges::empty(underlyingRange_); + requires requires { ql::ranges::empty(underlyingRange_); } { + return ql::ranges::empty(underlyingRange_); } - constexpr auto size() requires std::ranges::sized_range { - return std::ranges::size(underlyingRange_); + constexpr auto size() requires ql::ranges::sized_range { + return ql::ranges::size(underlyingRange_); } constexpr auto size() const - requires std::ranges::sized_range { - return std::ranges::size(underlyingRange_); + requires ql::ranges::sized_range { + return ql::ranges::size(underlyingRange_); } - constexpr auto data() requires std::ranges::contiguous_range - { - return std::ranges::data(underlyingRange_); + constexpr auto data() requires ql::ranges::contiguous_range { + return ql::ranges::data(underlyingRange_); } constexpr auto data() const - requires std::ranges::contiguous_range { - return std::ranges::data(underlyingRange_); + requires ql::ranges::contiguous_range { + return ql::ranges::data(underlyingRange_); } }; +// Like `OwningView` above, but the const overloads to `begin()` and `end()` do +// not exist. This is currently used in the `CompressedExternalIdTable.h`, where +// have a deeply nested stack of views, one of which is `OnwingView` +// which doesn't properly propagate the possibility of const iteration in +// range-v3`. +template +struct OwningViewNoConst : OwningView { + using OwningView::OwningView; +}; + +template +OwningViewNoConst(T&&) -> OwningViewNoConst; + // Helper concept for `ad_utility::allView`. namespace detail { template -concept can_ref_view = - requires(Range&& range) { std::ranges::ref_view{AD_FWD(range)}; }; -} +CPP_requires(can_ref_view, + requires(Range&& range)(ql::ranges::ref_view{AD_FWD(range)})); +template +CPP_concept can_ref_view = CPP_requires_ref(can_ref_view, Range); +} // namespace detail -// A simple drop-in replacement for `std::views::all` which is required because -// GCC 11 doesn't support `std::owning_view` (see above). As soon as we don't -// support GCC 11 anymore, we can throw out those implementations. +// A simple drop-in replacement for `ql::views::all` which is required because +// GCC 11 and range-v3 currently don't support `std::owning_view` (see above). +// As soon as we don't support GCC 11 anymore, we can throw out those +// implementations. template constexpr auto allView(Range&& range) { if constexpr (std::ranges::view>) { return AD_FWD(range); } else if constexpr (detail::can_ref_view) { - return std::ranges::ref_view{AD_FWD(range)}; + return ql::ranges::ref_view{AD_FWD(range)}; } else { return ad_utility::OwningView{AD_FWD(range)}; } } // Returns a view that contains all the values in `[0, upperBound)`, similar to -// Python's `range` function. Avoids the common pitfall in `std::views::iota` +// Python's `range` function. Avoids the common pitfall in `ql::views::iota` // that the count variable is only derived from the first argument. For example, -// `std::views::iota(0, size_t(INT_MAX) + 1)` leads to undefined behavior +// `ql::views::iota(0, size_t(INT_MAX) + 1)` leads to undefined behavior // because of an integer overflow, but `ad_utility::integerRange(size_t(INT_MAX) // + 1)` is perfectly safe and behaves as expected. template auto integerRange(Int upperBound) { - return std::views::iota(Int{0}, upperBound); + return ql::views::iota(Int{0}, upperBound); } // The implementation of `inPlaceTransformView`, see below for details. namespace detail { -template > - Transformation> -requires std::ranges::view +template > + Transformation> +requires(ql::ranges::view && ql::ranges::input_range) auto inPlaceTransformViewImpl(Range range, Transformation transformation) { // Take a range and yield pairs of [pointerToElementOfRange, // boolThatIsInitiallyFalse]. The bool is yielded as a reference and if its @@ -251,7 +270,7 @@ auto inPlaceTransformViewImpl(Range range, Transformation transformation) { // Lift the transformation to work on the result of `makePtrAndBool` and to // only apply the transformation once for each element. - // Note: This works because `std::views::transform` calls the transformation + // Note: This works because `ql::views::transform` calls the transformation // each time an iterator is dereferenced, so the following lambda is called // multiple times for the same element if the same iterator is dereferenced // multiple times and we therefore have to remember whether the transformation @@ -268,31 +287,35 @@ auto inPlaceTransformViewImpl(Range range, Transformation transformation) { }; // Combine everything to the actual result range. - return std::views::transform( + return ql::views::transform( ad_utility::OwningView{makeElementPtrAndBool(std::move(range))}, actualTransformation); } } // namespace detail -// Similar to `std::views::transform` but for transformation functions that +// Similar to `ql::views::transform` but for transformation functions that // transform a value in place. The result is always only an input range, // independent of the actual range category of the input. -template > - Transformation> -auto inPlaceTransformView(Range&& range, Transformation transformation) { - return detail::inPlaceTransformViewImpl(std::views::all(AD_FWD(range)), +CPP_template(typename Range, typename Transformation)( + requires ql::ranges::input_range CPP_and + ad_utility::InvocableWithExactReturnType< + Transformation, void, + ql::ranges::range_reference_t< + Range>>) auto inPlaceTransformView(Range&& range, + Transformation + transformation) { + return detail::inPlaceTransformViewImpl(ql::views::all(AD_FWD(range)), std::move(transformation)); } /// Create a generator the consumes the input generator until it finds the given /// separator and the yields spans of the chunks of data received inbetween. -template -inline cppcoro::generator> reChunkAtSeparator( - Range generator, ElementType separator) { +CPP_template(typename Range, typename ElementType)( + requires ql::ranges::input_range) inline cppcoro:: + generator> reChunkAtSeparator( + Range generator, ElementType separator) { std::vector buffer; - for (std::ranges::input_range auto chunk : generator) { + for (QL_OPT_CONCEPT(ql::ranges::input_range) auto const& chunk : generator) { for (ElementType c : chunk) { if (c == separator) { co_yield std::span{buffer.data(), buffer.size()}; @@ -310,7 +333,14 @@ inline cppcoro::generator> reChunkAtSeparator( } // namespace ad_utility // Enabling of "borrowed" ranges for `OwningView`. +#ifdef QLEVER_CPP_17 +template +inline constexpr bool ::ranges::enable_borrowed_range< + ad_utility::OwningView> = enable_borrowed_range; + +#else template inline constexpr bool std::ranges::enable_borrowed_range> = std::ranges::enable_borrowed_range; +#endif diff --git a/src/util/http/MediaTypes.cpp b/src/util/http/MediaTypes.cpp index 096cb3a3c1..d91e7584df 100644 --- a/src/util/http/MediaTypes.cpp +++ b/src/util/http/MediaTypes.cpp @@ -151,12 +151,12 @@ std::optional getMediaTypeFromAcceptHeader( return detail::SUPPORTED_MEDIA_TYPES.at(0); } else if constexpr (ad_utility::isSimilar< T, MediaTypeWithQuality::TypeWithWildcard>) { - auto it = std::ranges::find_if( + auto it = ql::ranges::find_if( detail::SUPPORTED_MEDIA_TYPES, [&part](const auto& el) { return getType(el) == part._type; }); return it == detail::SUPPORTED_MEDIA_TYPES.end() ? noValue : *it; } else if constexpr (ad_utility::isSimilar) { - auto it = std::ranges::find(detail::SUPPORTED_MEDIA_TYPES, part); + auto it = ql::ranges::find(detail::SUPPORTED_MEDIA_TYPES, part); return it != detail::SUPPORTED_MEDIA_TYPES.end() ? part : noValue; } else { static_assert(ad_utility::alwaysFalse); @@ -178,7 +178,7 @@ std::optional getMediaTypeFromAcceptHeader( std::string getErrorMessageForSupportedMediaTypes() { return "Currently the following media types are supported: " + lazyStrJoin( - detail::SUPPORTED_MEDIA_TYPES | std::views::transform(toString), + detail::SUPPORTED_MEDIA_TYPES | ql::views::transform(toString), ", "); } diff --git a/test/AddCombinedRowToTableTest.cpp b/test/AddCombinedRowToTableTest.cpp index c4a6b1c874..2c409a830b 100644 --- a/test/AddCombinedRowToTableTest.cpp +++ b/test/AddCombinedRowToTableTest.cpp @@ -11,7 +11,7 @@ namespace { static constexpr auto U = Id::makeUndefined(); void testWithAllBuffersizes(const auto& testFunction) { - for (auto bufferSize : std::views::iota(0, 10)) { + for (auto bufferSize : ql::views::iota(0, 10)) { testFunction(bufferSize); } testFunction(100'000); diff --git a/test/AlgorithmTest.cpp b/test/AlgorithmTest.cpp index 485f6930cc..187e0aa82e 100644 --- a/test/AlgorithmTest.cpp +++ b/test/AlgorithmTest.cpp @@ -16,8 +16,8 @@ using namespace ad_utility; TEST(Algorithm, Contains) { std::vector v{1, 42, 5, 3}; ASSERT_TRUE( - std::ranges::all_of(v, [&v](const auto& el) { return contains(v, el); })); - ASSERT_TRUE(std::ranges::none_of( + ql::ranges::all_of(v, [&v](const auto& el) { return contains(v, el); })); + ASSERT_TRUE(ql::ranges::none_of( std::vector{ 28, 2, @@ -29,30 +29,30 @@ TEST(Algorithm, Contains) { StringLike s{"hal"}; { std::vector substrings{"h", "a", "l", "ha", "al", "hal"}; - ASSERT_TRUE(std::ranges::all_of( + ASSERT_TRUE(ql::ranges::all_of( substrings, [&s](const auto& el) { return contains(s, el); })); std::vector noSubstrings{"x", "hl", "hel"}; // codespell-ignore - ASSERT_TRUE(std::ranges::none_of( + ASSERT_TRUE(ql::ranges::none_of( noSubstrings, [&s](const auto& el) { return contains(s, el); })); } { std::vector substrings{"h", "a", "l", "ha", "al", "hal"}; - ASSERT_TRUE(std::ranges::all_of( + ASSERT_TRUE(ql::ranges::all_of( substrings, [&s](const auto& el) { return contains(s, el); })); std::vector noSubstrings{"x", "hl", "hel"}; // codespell-ignore - ASSERT_TRUE(std::ranges::none_of( + ASSERT_TRUE(ql::ranges::none_of( noSubstrings, [&s](const auto& el) { return contains(s, el); })); } std::vector subchars{'h', 'a', 'l'}; - ASSERT_TRUE(std::ranges::all_of( + ASSERT_TRUE(ql::ranges::all_of( subchars, [&s](const auto& el) { return contains(s, el); })); std::vector noSubchars{'i', 'b', 'm'}; - ASSERT_TRUE(std::ranges::none_of( + ASSERT_TRUE(ql::ranges::none_of( noSubchars, [&s](const auto& el) { return contains(s, el); })); }; testStringLike.template operator()(); @@ -104,7 +104,7 @@ TEST(Algorithm, Transform) { ASSERT_EQ((std::vector{"hix", "byex", "whyx"}), v3); ASSERT_EQ(3u, v.size()); // The individual elements of `v` were moved from. - ASSERT_TRUE(std::ranges::all_of(v, &std::string::empty)); + ASSERT_TRUE(ql::ranges::all_of(v, &std::string::empty)); } // _____________________________________________________________________________ @@ -114,8 +114,8 @@ TEST(Algorithm, Flatten) { ASSERT_EQ((std::vector{"hi", "bye", "why", "me"}), v3); ASSERT_EQ(3u, v.size()); // The individual elements of `v` were moved from. - ASSERT_TRUE(std::ranges::all_of(v, [](const auto& inner) { - return std::ranges::all_of(inner, &std::string::empty); + ASSERT_TRUE(ql::ranges::all_of(v, [](const auto& inner) { + return ql::ranges::all_of(inner, &std::string::empty); })); } @@ -168,8 +168,8 @@ TEST(AlgorithmTest, transformArray) { TEST(AlgorithmTest, lowerUpperBoundIterator) { std::vector input; FastRandomIntGenerator randomGenerator; - std::ranges::generate_n(std::back_inserter(input), 1000, - std::ref(randomGenerator)); + ql::ranges::generate_n(std::back_inserter(input), 1000, + std::ref(randomGenerator)); auto compForLowerBound = [](auto iterator, size_t value) { return *iterator < value; @@ -180,9 +180,9 @@ TEST(AlgorithmTest, lowerUpperBoundIterator) { for (auto value : input) { EXPECT_EQ(ad_utility::lower_bound_iterator(input.begin(), input.end(), value, compForLowerBound), - std::ranges::lower_bound(input, value)); + ql::ranges::lower_bound(input, value)); EXPECT_EQ(ad_utility::upper_bound_iterator(input.begin(), input.end(), value, compForUpperBound), - std::ranges::upper_bound(input, value)); + ql::ranges::upper_bound(input, value)); } } diff --git a/test/AsyncStreamTest.cpp b/test/AsyncStreamTest.cpp index d0fe3addf1..dfd2ac9af1 100644 --- a/test/AsyncStreamTest.cpp +++ b/test/AsyncStreamTest.cpp @@ -49,6 +49,6 @@ TEST(AsyncStream, EnsureBuffersArePassedCorrectly) { const std::vector testData{"Abc", "Def", "Ghi"}; auto generator = runStreamAsync(testData, 2); - ASSERT_TRUE(std::ranges::equal(testData.begin(), testData.end(), - generator.begin(), generator.end())); + ASSERT_TRUE(ql::ranges::equal(testData.begin(), testData.end(), + generator.begin(), generator.end())); } diff --git a/test/BenchmarkMeasurementContainerTest.cpp b/test/BenchmarkMeasurementContainerTest.cpp index a500c3df2c..dc3c9aaeb7 100644 --- a/test/BenchmarkMeasurementContainerTest.cpp +++ b/test/BenchmarkMeasurementContainerTest.cpp @@ -158,7 +158,7 @@ TEST(BenchmarkMeasurementContainerTest, ResultTable) { on creation, because you can't add columns after creation and a table without columns is quite the stupid idea. Additionally, operations on such an empty table can create segmentation faults. The string conversion of `Result` - uses `std::ranges::max`, which really doesn't play well with empty vectors. + uses `ql::ranges::max`, which really doesn't play well with empty vectors. */ ASSERT_ANY_THROW(ResultTable("1 by 0 table", {"Test"}, {})); @@ -352,11 +352,11 @@ TEST(BenchmarkMeasurementContainerTest, ResultGroupDeleteMember) { group.deleteMeasurement(*entryToDelete); group.deleteTable(*tableToDelete); auto getAddressOfObject = [](const auto& obj) { return obj.get(); }; - ASSERT_TRUE(std::ranges::find(group.resultEntries_, entryToDelete, - getAddressOfObject) == + ASSERT_TRUE(ql::ranges::find(group.resultEntries_, entryToDelete, + getAddressOfObject) == std::end(group.resultEntries_)); - ASSERT_TRUE(std::ranges::find(group.resultTables_, tableToDelete, - getAddressOfObject) == + ASSERT_TRUE(ql::ranges::find(group.resultTables_, tableToDelete, + getAddressOfObject) == std::end(group.resultTables_)); // Test, if trying to delete a non-existent member results in an error. diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 9dd3a733a9..c260f34d4e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -103,6 +103,7 @@ endfunction() add_subdirectory(engine) add_subdirectory(parser) add_subdirectory(index) +add_subdirectory(backports) addLinkAndDiscoverTest(ValueIdComparatorsTest util) diff --git a/test/CallFixedSizeTest.cpp b/test/CallFixedSizeTest.cpp index 52b865b27a..ba6e82c255 100644 --- a/test/CallFixedSizeTest.cpp +++ b/test/CallFixedSizeTest.cpp @@ -175,7 +175,7 @@ TEST(CallFixedSize, CallFixedSize2) { } }; // TODO the ranges of the loop can be greatly simplified - // using `std::views::iota`, but views don't work yet on clang. + // using `ql::views::iota`, but views don't work yet on clang. // TODO We can then also setup a lambda that does the loop, // going from 4*4 to just 4 lines of calling code. for (int i = 0; i <= m; ++i) { diff --git a/test/CompactStringVectorTest.cpp b/test/CompactStringVectorTest.cpp index e66d9b875f..f551fc54bd 100644 --- a/test/CompactStringVectorTest.cpp +++ b/test/CompactStringVectorTest.cpp @@ -50,7 +50,7 @@ TEST(CompactVectorOfStrings, Iterator) { s.build(input); auto it = s.begin(); - using std::ranges::equal; + using ql::ranges::equal; ASSERT_TRUE(equal(input[0], *it)); ASSERT_TRUE(equal(input[0], *it++)); ASSERT_TRUE(equal(input[1], *it)); diff --git a/test/ComparisonWithNanTest.cpp b/test/ComparisonWithNanTest.cpp index 6458de6d17..5cde3fc249 100644 --- a/test/ComparisonWithNanTest.cpp +++ b/test/ComparisonWithNanTest.cpp @@ -4,9 +4,9 @@ #include -#include #include +#include "backports/algorithm.h" #include "util/ComparisonWithNan.h" namespace { @@ -25,7 +25,7 @@ auto gt = ad_utility::makeComparatorForNans(std::greater{}); TEST(ComparisonWithNan, Sorting) { std::vector input{NaN, 3.0, -3.0, NaN, negInf, NaN, inf}; std::vector expected{negInf, -3.0, 3.0, inf, NaN, NaN, NaN}; - std::ranges::sort(input, lt); + ql::ranges::sort(input, lt); ASSERT_EQ(input.size(), expected.size()); for (size_t i = 0; i < input.size(); ++i) { auto a = input[i]; diff --git a/test/CompressedRelationsTest.cpp b/test/CompressedRelationsTest.cpp index e9ab85df14..3166664941 100644 --- a/test/CompressedRelationsTest.cpp +++ b/test/CompressedRelationsTest.cpp @@ -44,7 +44,7 @@ size_t getNumColumns(const std::vector& input) { return 2; } auto result = input.at(0).size(); - AD_CONTRACT_CHECK(std::ranges::all_of( + AD_CONTRACT_CHECK(ql::ranges::all_of( input, [result](const auto& vec) { return vec.size() == result; })); return result; } @@ -54,7 +54,7 @@ size_t getNumColumns(const std::vector& vec) { return 2; } auto result = getNumColumns(vec.at(0).col1And2_); - AD_CONTRACT_CHECK(std::ranges::all_of(vec, [&result](const auto& relation) { + AD_CONTRACT_CHECK(ql::ranges::all_of(vec, [&result](const auto& relation) { return getNumColumns(relation.col1And2_) == result; })); return result; @@ -107,13 +107,12 @@ compressedRelationTestWriteCompressedRelations( auto inputs, std::string filename, ad_utility::MemorySize blocksize) { // First check the invariants of the `inputs`. They must be sorted by the // `col0_` and for each of the `inputs` the `col1And2_` must also be sorted. - AD_CONTRACT_CHECK(std::ranges::is_sorted( + AD_CONTRACT_CHECK(ql::ranges::is_sorted( inputs, {}, [](const RelationInput& r) { return r.col0_; })); - AD_CONTRACT_CHECK(std::ranges::all_of(inputs, [](const RelationInput& r) { - return std::ranges::is_sorted( - r.col1And2_, [](const auto& a, const auto& b) { - return std::ranges::lexicographical_compare(a, b); - }); + AD_CONTRACT_CHECK(ql::ranges::all_of(inputs, [](const RelationInput& r) { + return ql::ranges::is_sorted(r.col1And2_, [](const auto& a, const auto& b) { + return ql::ranges::lexicographical_compare(a, b); + }); })); // First create the on-disk permutation. @@ -142,7 +141,7 @@ compressedRelationTestWriteCompressedRelations( }; for (const auto& arr : input.col1And2_) { std::vector row{V(input.col0_)}; - std::ranges::transform(arr, std::back_inserter(row), V); + ql::ranges::transform(arr, std::back_inserter(row), V); buffer.push_back(row); if (buffer.numRows() > writer.blocksize()) { addBlock(); @@ -283,14 +282,14 @@ void testCompressedRelations(const auto& inputsOriginalBeforeCopy, std::make_shared>(); // Check the contents of the metadata. - // TODO `std::ranges::to`. + // TODO `ql::ranges::to`. std::vector additionalColumns; - std::ranges::copy(std::views::iota(3ul, getNumColumns(inputs) + 1), - std::back_inserter(additionalColumns)); + ql::ranges::copy(ql::views::iota(3ul, getNumColumns(inputs) + 1), + std::back_inserter(additionalColumns)); auto getMetadata = [&, &metaData = metaData](size_t i) { Id col0 = V(inputs[i].col0_); - auto it = std::ranges::lower_bound(metaData, col0, {}, - &CompressedRelationMetadata::col0Id_); + auto it = ql::ranges::lower_bound(metaData, col0, {}, + &CompressedRelationMetadata::col0Id_); if (it != metaData.end() && it->col0Id_ == col0) { return *it; } diff --git a/test/ConfigManagerTest.cpp b/test/ConfigManagerTest.cpp index 0d90283201..315e02d7a3 100644 --- a/test/ConfigManagerTest.cpp +++ b/test/ConfigManagerTest.cpp @@ -435,7 +435,7 @@ TEST(ConfigManagerTest, ParseConfigWithSubManager) { const std::vector>& wantedValues) { m.parseConfig(j); - std::ranges::for_each( + ql::ranges::for_each( wantedValues, [](const std::pair& wantedValue) -> void { ASSERT_EQ(*wantedValue.first, wantedValue.second); }); @@ -2131,7 +2131,7 @@ TEST(ConfigManagerTest, ContainsOption) { auto checkContainmentStatus = [](const ConfigManager& m, const ContainmentStatusVector& optionsAndWantedStatus) { - std::ranges::for_each( + ql::ranges::for_each( optionsAndWantedStatus, [&m](const ContainmentStatusVector::value_type& p) { if (p.second) { @@ -2308,7 +2308,7 @@ TEST(ConfigManagerTest, ValidatorsSorting) { // For generating better messages, when failing a test. auto trace{generateLocationTrace(l, "checkOrder")}; - ASSERT_TRUE(std::ranges::equal( + ASSERT_TRUE(ql::ranges::equal( manager.validators(true), order.validators_, {}, [](const ConfigOptionValidatorManager& validatorManager) { return validatorManager.getDescription(); @@ -2434,9 +2434,9 @@ TEST(ConfigManagerTest, ConfigurationDocValidatorAssignment) { std::pair>>& pairVector) { // Simply insert all the entries. - std::ranges::for_each(pairVector, [&assignment](const auto& pair) { + ql::ranges::for_each(pairVector, [&assignment](const auto& pair) { const auto& [key, validatorVector] = pair; - std::ranges::for_each( + ql::ranges::for_each( validatorVector, [&assignment, &key](const ConfigOptionValidatorManager& validator) { @@ -2460,16 +2460,16 @@ TEST(ConfigManagerTest, ConfigurationDocValidatorAssignment) { ad_utility::source_location::current()) { // For generating better messages, when failing a test. auto trace{generateLocationTrace(l, "testPairVector")}; - std::ranges::for_each(pairVector, [&assignment](const auto& pair) { + ql::ranges::for_each(pairVector, [&assignment](const auto& pair) { const auto& [key, expectedValidatorVector] = pair; // Are the entries under `key` the objects in the expected vector? auto toPointer = [](const ConfigOptionValidatorManager& x) { return &x; }; - ASSERT_TRUE(std::ranges::equal(assignment.getEntriesUnderKey(key), - expectedValidatorVector, {}, toPointer, - toPointer)); + ASSERT_TRUE(ql::ranges::equal(assignment.getEntriesUnderKey(key), + expectedValidatorVector, {}, toPointer, + toPointer)); }); }; diff --git a/test/DeltaTriplesTest.cpp b/test/DeltaTriplesTest.cpp index 10a4792d29..88ec0c76e5 100644 --- a/test/DeltaTriplesTest.cpp +++ b/test/DeltaTriplesTest.cpp @@ -47,7 +47,7 @@ class DeltaTriplesTest : public ::testing::Test { std::vector makeTurtleTriples( const std::vector& turtles) { RdfStringParser> parser; - std::ranges::for_each(turtles, [&parser](const std::string& turtle) { + ql::ranges::for_each(turtles, [&parser](const std::string& turtle) { parser.parseUtf8String(turtle); }); AD_CONTRACT_CHECK(parser.getTriples().size() == turtles.size()); diff --git a/test/FindUndefRangesTest.cpp b/test/FindUndefRangesTest.cpp index a3f3ee269f..a63afc8d5d 100644 --- a/test/FindUndefRangesTest.cpp +++ b/test/FindUndefRangesTest.cpp @@ -65,7 +65,7 @@ void testSmallerUndefRangesForRowsWithoutUndef( const std::vector& positions, source_location l = source_location::current()) { auto t = generateLocationTrace(l); - ASSERT_TRUE(std::ranges::is_sorted(range)); + ASSERT_TRUE(ql::ranges::is_sorted(range)); std::vector foundPositions; // TODO also actually test the bool; [[maybe_unused]] bool outOfOrder; @@ -112,7 +112,7 @@ void testSmallerUndefRangesForRowsWithUndefInLastColumns( const std::vector& positions, source_location l = source_location::current()) { auto t = generateLocationTrace(l); - ASSERT_TRUE(std::ranges::is_sorted(range)); + ASSERT_TRUE(ql::ranges::is_sorted(range)); std::vector foundPositions; // TODO also actually test the bool; [[maybe_unused]] bool outOfOrder; diff --git a/test/GeoPointTest.cpp b/test/GeoPointTest.cpp index 3d5dc60b7f..a9b39e0f87 100644 --- a/test/GeoPointTest.cpp +++ b/test/GeoPointTest.cpp @@ -11,6 +11,7 @@ #include "parser/GeoPoint.h" #include "util/GTestHelpers.h" #include "util/GeoSparqlHelpers.h" +#include "util/HashSet.h" // _____________________________________________________________________________ TEST(GeoPoint, GeoPoint) { diff --git a/test/GroupByTest.cpp b/test/GroupByTest.cpp index a2c28a1b5e..0c2e314e46 100644 --- a/test/GroupByTest.cpp +++ b/test/GroupByTest.cpp @@ -1207,10 +1207,10 @@ TEST_F(GroupByOptimizations, hashMapOptimizationMinMaxSumIntegers) { auto unsignedLongToValueId = [](unsigned long value) { return ValueId::makeFromInt(static_cast(value)); }; - std::ranges::transform(firstColumn.begin(), firstColumn.end(), - firstTableColumn.begin(), unsignedLongToValueId); - std::ranges::transform(secondColumn.begin(), secondColumn.end(), - secondTableColumn.begin(), unsignedLongToValueId); + ql::ranges::transform(firstColumn.begin(), firstColumn.end(), + firstTableColumn.begin(), unsignedLongToValueId); + ql::ranges::transform(secondColumn.begin(), secondColumn.end(), + secondTableColumn.begin(), unsignedLongToValueId); auto values = ad_utility::makeExecutionTree( qec, std::move(testTable), variables, false); diff --git a/test/HttpTest.cpp b/test/HttpTest.cpp index 05e3e7cf7b..12e5233c7e 100644 --- a/test/HttpTest.cpp +++ b/test/HttpTest.cpp @@ -22,7 +22,7 @@ namespace { /// Join all of the bytes into a big string. std::string toString(cppcoro::generator> generator) { std::string result; - for (std::byte byte : generator | std::ranges::views::join) { + for (std::byte byte : generator | ql::ranges::views::join) { result.push_back(static_cast(byte)); } return result; diff --git a/test/IdTableHelpersTest.cpp b/test/IdTableHelpersTest.cpp index b6cbcebbd5..842cc1ea32 100644 --- a/test/IdTableHelpersTest.cpp +++ b/test/IdTableHelpersTest.cpp @@ -11,6 +11,7 @@ #include #include "./util/IdTableHelpers.h" +#include "backports/algorithm.h" #include "engine/idTable/IdTable.h" #include "global/ValueId.h" #include "util/Algorithm.h" @@ -29,30 +30,30 @@ elements will not be ignored. @param setToCalculateFor The container to calculate all sub-sets for. Will only be read. */ -template >> -std::vector> calculateAllSubSets(R&& setToCalculateFor) { +CPP_template(typename R, + typename E = std::iter_value_t>)( + requires ql::ranges::forward_range) + std::vector> calculateAllSubSets(R&& setToCalculateFor) { // Getting rid of duplicated elements. std::vector> calculatedSubSets; // There will be exactly $setToCalculateFor.size()^2$ items added. calculatedSubSets.reserve( - ad_utility::pow(2, std::ranges::size(setToCalculateFor))); + ad_utility::pow(2, ql::ranges::size(setToCalculateFor))); // The empty set is always a sub-set. calculatedSubSets.push_back({}); // Calculate all sub-sets. - std::ranges::for_each( - setToCalculateFor, [&calculatedSubSets](const E& entry) { - ad_utility::appendVector( - calculatedSubSets, - ad_utility::transform(calculatedSubSets, - [&entry](std::vector subSet) { - subSet.push_back(entry); - return subSet; - })); - }); + ql::ranges::for_each(setToCalculateFor, [&calculatedSubSets](const E& entry) { + ad_utility::appendVector( + calculatedSubSets, + ad_utility::transform(calculatedSubSets, + [&entry](std::vector subSet) { + subSet.push_back(entry); + return subSet; + })); + }); return calculatedSubSets; } @@ -64,8 +65,8 @@ TEST(IdTableHelpersHelpersTest, calculateAllSubSets) { std::vector> result{calculateAllSubSets(input)}; // For comparison, we have to sort both vectors. - std::ranges::sort(expectedOutput, std::ranges::lexicographical_compare); - std::ranges::sort(result, std::ranges::lexicographical_compare); + ql::ranges::sort(expectedOutput, ql::ranges::lexicographical_compare); + ql::ranges::sort(result, ql::ranges::lexicographical_compare); ASSERT_EQ(expectedOutput, result); }; @@ -96,8 +97,8 @@ void generalIdTableCheck(const IdTable& table, ASSERT_EQ(table.numColumns(), expectedNumberOfColumns); if (allEntriesWereSet) { - ASSERT_TRUE(std::ranges::all_of(table, [](const auto& row) { - return std::ranges::all_of(row, [](const ValueId& entry) { + ASSERT_TRUE(ql::ranges::all_of(table, [](const auto& row) { + return ql::ranges::all_of(row, [](const ValueId& entry) { return ad_utility::testing::VocabId(0) <= entry && entry <= ad_utility::testing::VocabId(ValueId::maxIndex); }); @@ -136,7 +137,7 @@ TEST(IdTableHelpersTest, createRandomlyFilledIdTableWithoutGenerators) { // Checks, if all entries of are within a given inclusive range. auto checkColumn = [](const IdTable& table, const size_t& columnNumber, const size_t& lowerBound, const size_t& upperBound) { - ASSERT_TRUE(std::ranges::all_of( + ASSERT_TRUE(ql::ranges::all_of( table.getColumn(columnNumber), [&lowerBound, &upperBound](const ValueId& entry) { return ad_utility::testing::VocabId(lowerBound) <= entry && @@ -165,7 +166,7 @@ TEST(IdTableHelpersTest, createRandomlyFilledIdTableWithoutGenerators) { `JoinColumnAndBounds`, in the case of generating tables with 40 rows and 10 columns. */ - std::ranges::for_each( + ql::ranges::for_each( calculateAllSubSets(std::vector{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}), [&checkColumn, &result](const std::vector& joinColumns) { result = createRandomlyFilledIdTable( @@ -177,10 +178,10 @@ TEST(IdTableHelpersTest, createRandomlyFilledIdTableWithoutGenerators) { generalIdTableCheck(result, 40, 10, true); // Are the join columns like we wanted them? - std::ranges::for_each(joinColumns, - [&result, &checkColumn](const size_t& jc) { - checkColumn(result, jc, jc * 10, jc * 10 + 9); - }); + ql::ranges::for_each(joinColumns, + [&result, &checkColumn](const size_t& jc) { + checkColumn(result, jc, jc * 10, jc * 10 + 9); + }); }); } @@ -229,7 +230,7 @@ TEST(IdTableHelpersTest, createRandomlyFilledIdTableWithGenerators) { // Exhaustive test, if the creation of a randomly filled table works, // regardless of the amount of join columns and their position. - std::ranges::for_each( + ql::ranges::for_each( calculateAllSubSets(std::vector{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}), [&createCountUpGenerator, &compareColumnsWithVectors](const std::vector& joinColumns) { @@ -247,7 +248,7 @@ TEST(IdTableHelpersTest, createRandomlyFilledIdTableWithGenerators) { // have the correct content. generalIdTableCheck(resultMultiGenerator, 10, 10, true); generalIdTableCheck(resultSingleGenerator, 10, 10, true); - std::ranges::for_each( + ql::ranges::for_each( joinColumns, [&resultMultiGenerator, &resultSingleGenerator, &joinColumns, &compareColumnsWithVectors](const size_t& num) { @@ -255,7 +256,7 @@ TEST(IdTableHelpersTest, createRandomlyFilledIdTableWithGenerators) { {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}); const size_t indexOfTheColumn = - std::ranges::find(joinColumns, num) - joinColumns.begin(); + ql::ranges::find(joinColumns, num) - joinColumns.begin(); compareColumnsWithVectors( resultSingleGenerator, num, {indexOfTheColumn, indexOfTheColumn + joinColumns.size(), @@ -294,7 +295,7 @@ TEST(IdTableHelpersTest, generateIdTable) { std::vector row(width); // Fill the row. - std::ranges::fill(row, ad_utility::testing::VocabId(i)); + ql::ranges::fill(row, ad_utility::testing::VocabId(i)); i++; return row; @@ -312,7 +313,7 @@ TEST(IdTableHelpersTest, generateIdTable) { std::vector row(i < 3 ? 5 : 20); // Fill the row. - std::ranges::fill(row, ad_utility::testing::VocabId(4)); + ql::ranges::fill(row, ad_utility::testing::VocabId(4)); i++; return row; @@ -322,7 +323,7 @@ TEST(IdTableHelpersTest, generateIdTable) { IdTable table{generateIdTable(5, 5, createCountUpGenerator(5))}; generalIdTableCheck(table, 5, 5, true); for (size_t row = 0; row < 5; row++) { - ASSERT_TRUE(std::ranges::all_of(table[row], [&row](const auto& entry) { + ASSERT_TRUE(ql::ranges::all_of(table[row], [&row](const auto& entry) { return entry == ad_utility::testing::VocabId(row); })); } @@ -337,7 +338,7 @@ TEST(IdTableHelpersTest, randomSeed) { constexpr size_t NUM_ROWS = 100; constexpr size_t NUM_COLUMNS = 200; - std::ranges::for_each( + ql::ranges::for_each( createArrayOfRandomSeeds<5>(), [](const ad_utility::RandomSeed seed) { // Simply generate and compare. ASSERT_EQ( diff --git a/test/IdTableTest.cpp b/test/IdTableTest.cpp index 483f3bcb92..441048a54e 100644 --- a/test/IdTableTest.cpp +++ b/test/IdTableTest.cpp @@ -164,7 +164,7 @@ TEST(IdTable, rowIterators) { ASSERT_FALSE( std::is_sorted(std::as_const(row).begin(), std::as_const(row).end())); - std::ranges::sort(row.begin(), row.end()); + ql::ranges::sort(row.begin(), row.end()); ASSERT_EQ(-1, row[0]); ASSERT_EQ(0, row[1]); ASSERT_EQ(1, row[2]); @@ -203,8 +203,8 @@ TEST(IdTable, rowIterators) { std::sort(std::move(row).begin(), std::move(row).end()); // The following calls all would not compile: // std::sort(row.begin(), row.end()); - // std::ranges::sort(row); - // std::ranges::sort(std::move(row)); + // ql::ranges::sort(row); + // ql::ranges::sort(std::move(row)); ASSERT_EQ(-1, row[0]); ASSERT_EQ(0, row[1]); ASSERT_EQ(1, row[2]); @@ -570,8 +570,7 @@ TEST(IdTable, sortTest) { // Now try the actual sort test = orig.clone(); - std::ranges::sort(test, std::less<>{}, - [](const auto& row) { return row[0]; }); + ql::ranges::sort(test, std::less<>{}, [](const auto& row) { return row[0]; }); // The sorted order of the orig tables should be: // 3, 2, 0, 4, 5, 1 @@ -1100,8 +1099,8 @@ TEST(IdTable, shrinkToFit) { TEST(IdTable, staticAsserts) { static_assert(std::is_trivially_copyable_v::iterator>); static_assert(std::is_trivially_copyable_v::const_iterator>); - static_assert(std::ranges::random_access_range); - static_assert(std::ranges::random_access_range>); + static_assert(ql::ranges::random_access_range); + static_assert(ql::ranges::random_access_range>); } TEST(IdTable, constructorsAreSfinaeFriendly) { diff --git a/test/JoinAlgorithmsTest.cpp b/test/JoinAlgorithmsTest.cpp index 1829de1519..6996191e59 100644 --- a/test/JoinAlgorithmsTest.cpp +++ b/test/JoinAlgorithmsTest.cpp @@ -72,7 +72,7 @@ void testJoin(const NestedBlock& a, const NestedBlock& b, JoinResult expected, zipperJoinForBlocksWithoutUndef(a, b, compare, adder); } // The result must be sorted on the first column - EXPECT_TRUE(std::ranges::is_sorted(result, std::less<>{}, ad_utility::first)); + EXPECT_TRUE(ql::ranges::is_sorted(result, std::less<>{}, ad_utility::first)); // The exact order of the elements with the same first column is not important // and depends on implementation details. We therefore do not enforce it here. EXPECT_THAT(result, ::testing::UnorderedElementsAreArray(expected)); @@ -89,7 +89,7 @@ void testJoin(const NestedBlock& a, const NestedBlock& b, JoinResult expected, auto adder = makeRowAdder(result); zipperJoinForBlocksWithoutUndef(b, a, compare, adder); EXPECT_TRUE( - std::ranges::is_sorted(result, std::less<>{}, ad_utility::first)); + ql::ranges::is_sorted(result, std::less<>{}, ad_utility::first)); EXPECT_THAT(result, ::testing::UnorderedElementsAreArray(expected)); } } diff --git a/test/JoinTest.cpp b/test/JoinTest.cpp index d5a0117078..6ab090214f 100644 --- a/test/JoinTest.cpp +++ b/test/JoinTest.cpp @@ -97,14 +97,14 @@ void runTestCasesForAllJoinAlgorithms( // For sorting IdTableAndJoinColumn by their join column. auto sortByJoinColumn = [](IdTableAndJoinColumn& idTableAndJC) { - std::ranges::sort(idTableAndJC.idTable, {}, - [&idTableAndJC](const auto& row) { - return row[idTableAndJC.joinColumn]; - }); + ql::ranges::sort(idTableAndJC.idTable, {}, + [&idTableAndJC](const auto& row) { + return row[idTableAndJC.joinColumn]; + }); }; // Random shuffle both tables, run hashJoin, check result. - std::ranges::for_each(testSet, [](JoinTestCase& testCase) { + ql::ranges::for_each(testSet, [](JoinTestCase& testCase) { randomShuffle(testCase.leftInput.idTable.begin(), testCase.leftInput.idTable.end()); randomShuffle(testCase.rightInput.idTable.begin(), @@ -115,7 +115,7 @@ void runTestCasesForAllJoinAlgorithms( // Sort the larger table by join column, run hashJoin, check result (this time // it's sorted). - std::ranges::for_each(testSet, [&sortByJoinColumn](JoinTestCase& testCase) { + ql::ranges::for_each(testSet, [&sortByJoinColumn](JoinTestCase& testCase) { IdTableAndJoinColumn& largerInputTable = (testCase.leftInput.idTable.size() >= testCase.rightInput.idTable.size()) @@ -128,7 +128,7 @@ void runTestCasesForAllJoinAlgorithms( // Sort both tables, run merge join and hash join, check result. (Which has to // be sorted.) - std::ranges::for_each(testSet, [&sortByJoinColumn](JoinTestCase& testCase) { + ql::ranges::for_each(testSet, [&sortByJoinColumn](JoinTestCase& testCase) { sortByJoinColumn(testCase.leftInput); sortByJoinColumn(testCase.rightInput); testCase.resultMustBeSortedByJoinColumn = true; @@ -213,7 +213,7 @@ std::vector createJoinTestSet() { IdTable createIdTableOfSizeWithValue(size_t size, Id value) { IdTable idTable{1, ad_utility::testing::makeAllocator()}; idTable.resize(size); - std::ranges::fill(idTable.getColumn(0), value); + ql::ranges::fill(idTable.getColumn(0), value); return idTable; } } // namespace diff --git a/test/LocalVocabTest.cpp b/test/LocalVocabTest.cpp index c192a8b35f..369b61f713 100644 --- a/test/LocalVocabTest.cpp +++ b/test/LocalVocabTest.cpp @@ -187,7 +187,7 @@ TEST(LocalVocab, merge) { auto id2 = vocE.getBlankNodeIndex(&bnm); auto vocabs3 = std::vector{&std::as_const(localVocabMerged2), &std::as_const(vocF)}; - vocE.mergeWith(vocabs3 | std::views::transform( + vocE.mergeWith(vocabs3 | ql::views::transform( [](const LocalVocab* l) -> const LocalVocab& { return *l; })); @@ -219,8 +219,8 @@ TEST(LocalVocab, propagation) { return LiteralOrIri::literalWithoutQuotes(word); } }; - std::ranges::transform(expectedWordsAsStrings, - std::back_inserter(expectedWords), toLitOrIri); + ql::ranges::transform(expectedWordsAsStrings, + std::back_inserter(expectedWords), toLitOrIri); std::shared_ptr resultTable = operation.getResult(); ASSERT_TRUE(resultTable) << "Operation: " << operation.getDescriptor() << std::endl; @@ -228,7 +228,7 @@ TEST(LocalVocab, propagation) { resultTable->localVocab().getAllWordsForTesting(); // We currently allow the local vocab to have multiple IDs for the same // word, so we have to deduplicate first. - std::ranges::sort(localVocabWords); + ql::ranges::sort(localVocabWords); localVocabWords.erase(std::ranges::unique(localVocabWords).begin(), localVocabWords.end()); ASSERT_THAT(localVocabWords, diff --git a/test/LocatedTriplesTest.cpp b/test/LocatedTriplesTest.cpp index 4a46644aff..9fb9933ef5 100644 --- a/test/LocatedTriplesTest.cpp +++ b/test/LocatedTriplesTest.cpp @@ -20,7 +20,7 @@ int g = 123948; void addGraphColumn(IdTable& block) { block.addEmptyColumn(); - std::ranges::fill(block.getColumn(block.numColumns() - 1), V(g)); + ql::ranges::fill(block.getColumn(block.numColumns() - 1), V(g)); } auto IT = [](const auto& c1, const auto& c2, const auto& c3, int graph = g) { diff --git a/test/MemorySizeTest.cpp b/test/MemorySizeTest.cpp index 22b2c614c7..cde91e5c35 100644 --- a/test/MemorySizeTest.cpp +++ b/test/MemorySizeTest.cpp @@ -207,7 +207,7 @@ TEST(MemorySize, AsString) { ASSERT_STREQ(stream.str().c_str(), testCase.stringRepresentation_.data()); }; - std::ranges::for_each(generalAsStringTestCases(), doTest); + ql::ranges::for_each(generalAsStringTestCases(), doTest); // Check, if it always uses the right unit. doTest({99'999_B, "99999 B"}); @@ -232,7 +232,7 @@ TEST(MemorySize, Parse) { }; // General testing. - std::ranges::for_each(generalAsStringTestCases(), doTest); + ql::ranges::for_each(generalAsStringTestCases(), doTest); // Does `Byte` only work with whole, positive numbers? doExceptionTest("-46 B"); @@ -240,21 +240,21 @@ TEST(MemorySize, Parse) { doExceptionTest("-4.2 B"); // Nothing should work with negative numbers. - std::ranges::for_each(generalAsStringTestCases(), doExceptionTest, - [](const MemorySizeAndStringRepresentation& testCase) { - return absl::StrCat("-", - testCase.stringRepresentation_); - }); + ql::ranges::for_each(generalAsStringTestCases(), doExceptionTest, + [](const MemorySizeAndStringRepresentation& testCase) { + return absl::StrCat("-", + testCase.stringRepresentation_); + }); // Byte sizes can only be set with `B`. - std::ranges::for_each(std::vector{"42 BYTE", "42 BYTe", "42 BYtE", "42 BYte", - "42 ByTE", "42 ByTe", "42 BytE", "42 Byte", - "42 bYTE", "42 bYTe", "42 bYtE", "42 bYte", - "42 byTE", "42 byTe", "42 bytE", "42 byte"}, - doExceptionTest); + ql::ranges::for_each(std::vector{"42 BYTE", "42 BYTe", "42 BYtE", "42 BYte", + "42 ByTE", "42 ByTe", "42 BytE", "42 Byte", + "42 bYTE", "42 bYTe", "42 bYtE", "42 bYte", + "42 byTE", "42 byTe", "42 bytE", "42 byte"}, + doExceptionTest); // Is our grammar truly case insensitive? - std::ranges::for_each( + ql::ranges::for_each( std::vector{{42_B, "42 B"}, {42_B, "42 b"}, {42_kB, "42 KB"}, @@ -277,7 +277,7 @@ TEST(MemorySize, Parse) { // Does our short hand (memory unit without the `B` at the end) work? And is // it case insensitive? - std::ranges::for_each( + ql::ranges::for_each( std::vector{{42_kB, "42 K"}, {42_kB, "42 k"}, {42_MB, "42 M"}, @@ -289,7 +289,7 @@ TEST(MemorySize, Parse) { doTest); // Check if whitespace between unit and amount is truly optional - std::ranges::for_each( + ql::ranges::for_each( std::vector{{42_B, "42B"}, {42_B, "42b"}, {42_kB, "42KB"}, @@ -310,7 +310,7 @@ TEST(MemorySize, Parse) { {42_TB, "42tb"}}, doTest); - std::ranges::for_each( + ql::ranges::for_each( std::vector{{42_kB, "42K"}, {42_kB, "42k"}, {42_MB, "42M"}, @@ -322,13 +322,13 @@ TEST(MemorySize, Parse) { doTest); // Test if multiple spaces are fine too - std::ranges::for_each( + ql::ranges::for_each( std::vector{{42_kB, "42 K"}, {42_kB, "42 k"}}, doTest); // We only take memory units up to `TB`. Not further. - std::ranges::for_each(std::vector{"42 P", "42 PB"}, doExceptionTest); + ql::ranges::for_each(std::vector{"42 P", "42 PB"}, doExceptionTest); } TEST(MemorySize, ArithmeticOperators) { diff --git a/test/OrderByTest.cpp b/test/OrderByTest.cpp index cef7d1f1e4..046f55d1f6 100644 --- a/test/OrderByTest.cpp +++ b/test/OrderByTest.cpp @@ -61,9 +61,9 @@ void testOrderBy(IdTable input, const IdTable& expected, // Apply the current permutation of the `sortColumns` to `expected` and // `input`. for (size_t i = 0; i < sortColumns.size(); ++i) { - std::ranges::copy(input.getColumn(i), - permutedInput.getColumn(sortColumns[i].first).begin()); - std::ranges::copy( + ql::ranges::copy(input.getColumn(i), + permutedInput.getColumn(sortColumns[i].first).begin()); + ql::ranges::copy( expected.getColumn(i), permutedExpected.getColumn(sortColumns[i].first).begin()); // Also put the information which columns are descending into the correct @@ -192,7 +192,7 @@ TEST(OrderBy, mixedDatatypes) { testOrderBy(makeIdTableFromVector(input), makeIdTableFromVector(expected), {false}); - std::ranges::reverse(expected); + ql::ranges::reverse(expected); testOrderBy(makeIdTableFromVector(input), makeIdTableFromVector(expected), {true}); } diff --git a/test/ParallelMultiwayMergeTest.cpp b/test/ParallelMultiwayMergeTest.cpp index 95c416ef90..2174a9ec35 100644 --- a/test/ParallelMultiwayMergeTest.cpp +++ b/test/ParallelMultiwayMergeTest.cpp @@ -15,10 +15,10 @@ using namespace ad_utility::memory_literals; // Join a range of ranges into a single vector, e.g. `array> // -> vector`. auto join = [](Range&& range) { - std::vector>> + std::vector>> result; - auto view = std::views::join(ad_utility::OwningView{AD_FWD(range)}); - std::ranges::copy(view, std::back_inserter(result)); + auto view = ql::views::join(ad_utility::OwningView{AD_FWD(range)}); + ql::ranges::copy(view, std::back_inserter(result)); return result; }; @@ -34,22 +34,22 @@ void testRandomInts() { numRowsGen = ad_utility::SlowRandomIntGenerator( minVecSize, maxVecSize)]() mutable { std::vector res(numRowsGen()); - std::ranges::generate(res, gen); - std::ranges::sort(res); + ql::ranges::generate(res, gen); + ql::ranges::sort(res); return res; }; std::vector> input(numVecs); - std::ranges::generate(input, generateRandomVec); + ql::ranges::generate(input, generateRandomVec); auto expected = join(std::vector>{input}); - std::ranges::sort(expected); + ql::ranges::sort(expected); std::vector result; - std::ranges::copy(std::views::join(ad_utility::OwningView{ - ad_utility::parallelMultiwayMerge( - 1_GB, input, std::less<>{}, blocksize)}), - std::back_inserter(result)); + ql::ranges::copy(ql::views::join(ad_utility::OwningView{ + ad_utility::parallelMultiwayMerge( + 1_GB, input, std::less<>{}, blocksize)}), + std::back_inserter(result)); EXPECT_THAT(result, ::testing::ElementsAreArray(expected)); } diff --git a/test/PrefilterExpressionIndexTest.cpp b/test/PrefilterExpressionIndexTest.cpp index 2b31304d4f..5eeb1baaca 100644 --- a/test/PrefilterExpressionIndexTest.cpp +++ b/test/PrefilterExpressionIndexTest.cpp @@ -218,7 +218,7 @@ class PrefilterExpressionOnMetadataTest : public ::testing::Test { std::vector expectedAdjusted; // This is for convenience, we automatically insert all mixed and possibly // incomplete blocks which must be always returned. - std::ranges::set_union( + ql::ranges::set_union( expected, useBlocksIncomplete ? mixedAndIncompleteBlocks : mixedBlocks, std::back_inserter(expectedAdjusted), [](const BlockMetadata& b1, const BlockMetadata& b2) { diff --git a/test/QueryPlannerTestHelpers.h b/test/QueryPlannerTestHelpers.h index 48fb419571..ea22a00431 100644 --- a/test/QueryPlannerTestHelpers.h +++ b/test/QueryPlannerTestHelpers.h @@ -357,8 +357,8 @@ static constexpr auto GroupBy = // TODO Also test the aliases. auto aliasesToStrings = [](const std::vector& aliases) { std::vector result; - std::ranges::transform(aliases, std::back_inserter(result), - &Alias::getDescriptor); + ql::ranges::transform(aliases, std::back_inserter(result), + &Alias::getDescriptor); return result; }; diff --git a/test/RandomTest.cpp b/test/RandomTest.cpp index fdcae374e8..034d7ac4c1 100644 --- a/test/RandomTest.cpp +++ b/test/RandomTest.cpp @@ -51,7 +51,7 @@ void testSeed( // For every seed test, if the random number generators return the same // numbers. - std::ranges::for_each( + ql::ranges::for_each( createArrayOfRandomSeeds(), [&randomNumberGeneratorFactory](const RandomSeed seed) { // What type of generator does the factory create? @@ -62,7 +62,7 @@ void testSeed( // The generators, that should create the same numbers. std::array generators; - std::ranges::generate( + ql::ranges::generate( generators, [&randomNumberGeneratorFactory, &seed]() { return std::invoke(randomNumberGeneratorFactory, seed); }); @@ -71,10 +71,10 @@ void testSeed( for (size_t numCall = 0; numCall < NUM_RANDOM_NUMBER; numCall++) { const NumberType expectedNumber = std::invoke(generators.front()); - std::ranges::for_each(std::views::drop(generators, 1), - [&expectedNumber](GeneratorType& g) { - ASSERT_EQ(std::invoke(g), expectedNumber); - }); + ql::ranges::for_each(ql::views::drop(generators, 1), + [&expectedNumber](GeneratorType& g) { + ASSERT_EQ(std::invoke(g), expectedNumber); + }); } }); } @@ -115,8 +115,8 @@ void testSeedWithRange( // For generating better messages, when failing a test. auto trace{generateLocationTrace(l, "testSeedWithRange")}; - std::ranges::for_each(ranges, [&randomNumberGeneratorFactory]( - const NumericalRange& r) { + ql::ranges::for_each(ranges, [&randomNumberGeneratorFactory]( + const NumericalRange& r) { testSeed([&r, &randomNumberGeneratorFactory](RandomSeed seed) { return std::invoke(randomNumberGeneratorFactory, r.minimum_, r.maximum_, seed); @@ -147,7 +147,7 @@ void testRange( constexpr size_t NUM_RANDOM_NUMBER = 500; static_assert(NUM_RANDOM_NUMBER > 1); - std::ranges::for_each(ranges, [](const NumericalRange& r) { + ql::ranges::for_each(ranges, [](const NumericalRange& r) { Generator generator(r.minimum_, r.maximum_); const auto& generatedNumber = std::invoke(generator); ASSERT_LE(generatedNumber, r.maximum_); @@ -244,25 +244,24 @@ TEST(RandomShuffleTest, Seed) { For every random seed test, if the shuffled array is the same, if given identical input and seed. */ - std::ranges::for_each( + ql::ranges::for_each( createArrayOfRandomSeeds(), [](const RandomSeed seed) { std::array, NUM_SHUFFLED_ARRAY> inputArrays{}; // Fill the first input array with random values, then copy it into the // other 'slots'. - std::ranges::generate(inputArrays.front(), - FastRandomIntGenerator{}); - std::ranges::fill(std::views::drop(inputArrays, 1), - inputArrays.front()); + ql::ranges::generate(inputArrays.front(), + FastRandomIntGenerator{}); + ql::ranges::fill(ql::views::drop(inputArrays, 1), inputArrays.front()); // Shuffle and compare, if they are all the same. - std::ranges::for_each( + ql::ranges::for_each( inputArrays, [&seed](std::array& inputArray) { randomShuffle(inputArray.begin(), inputArray.end(), seed); }); - std::ranges::for_each( - std::views::drop(inputArrays, 1), + ql::ranges::for_each( + ql::views::drop(inputArrays, 1), [&inputArrays](const std::array& inputArray) { ASSERT_EQ(inputArrays.front(), inputArray); }); diff --git a/test/RdfParserTest.cpp b/test/RdfParserTest.cpp index 6969961003..bbde3375a5 100644 --- a/test/RdfParserTest.cpp +++ b/test/RdfParserTest.cpp @@ -1132,7 +1132,7 @@ TEST(RdfParserTest, multifileParser) { Parser p{specs}; std::vector result; while (auto batch = p.getBatch()) { - std::ranges::copy(batch.value(), std::back_inserter(result)); + ql::ranges::copy(batch.value(), std::back_inserter(result)); } EXPECT_THAT(result, ::testing::UnorderedElementsAreArray(expected)); }; diff --git a/test/RelationalExpressionTest.cpp b/test/RelationalExpressionTest.cpp index 7c53af843b..50e369109f 100644 --- a/test/RelationalExpressionTest.cpp +++ b/test/RelationalExpressionTest.cpp @@ -169,7 +169,7 @@ auto expectUndefined = [](const SparqlExpression& expression, AD_CORRECTNESS_CHECK( (std::holds_alternative>(result))); const auto& vec = std::get>(result); - EXPECT_TRUE(std::ranges::all_of( + EXPECT_TRUE(ql::ranges::all_of( vec, [](Id id) { return id == Id::makeUndefined(); })); } }; @@ -679,8 +679,8 @@ void testWithExplicitResult(auto leftValue, auto rightValue, source_location l = source_location::current()) { auto t = generateLocationTrace(l); std::vector expected; - std::ranges::transform(expectedAsBool, std::back_inserter(expected), - Id::makeFromBool); + ql::ranges::transform(expectedAsBool, std::back_inserter(expected), + Id::makeFromBool); testWithExplicitIdResult(std::move(leftValue), std::move(rightValue), expected); diff --git a/test/ResultTableColumnOperationsTest.cpp b/test/ResultTableColumnOperationsTest.cpp index a04ac895ff..9fc246c548 100644 --- a/test/ResultTableColumnOperationsTest.cpp +++ b/test/ResultTableColumnOperationsTest.cpp @@ -317,7 +317,7 @@ TEST(ResultTableColumnOperations, calculateSpeedupOfColumn) { }; // Test things for a range of speedups. - std::ranges::for_each( + ql::ranges::for_each( std::array{2.f, 16.f, 73.696f, 4.2f}, [&fillColumnsForSpeedup](const float wantedSpeedup, ad_utility::source_location l = diff --git a/test/SortTest.cpp b/test/SortTest.cpp index 9461576a45..ab51138815 100644 --- a/test/SortTest.cpp +++ b/test/SortTest.cpp @@ -54,10 +54,10 @@ void testSort(IdTable input, const IdTable& expected, // Apply the current permutation of the `sortColumns` to `expected` and // `input`. for (size_t i = 0; i < sortColumns.size(); ++i) { - std::ranges::copy(input.getColumn(sortColumns[i]), - permutedInput.getColumn(i).begin()); - std::ranges::copy(expected.getColumn(sortColumns[i]), - permutedExpected.getColumn(i).begin()); + ql::ranges::copy(input.getColumn(sortColumns[i]), + permutedInput.getColumn(i).begin()); + ql::ranges::copy(expected.getColumn(sortColumns[i]), + permutedExpected.getColumn(i).begin()); } for (size_t i = 0; i < 5; ++i) { diff --git a/test/StringUtilsTest.cpp b/test/StringUtilsTest.cpp index e448aff7e1..b7c26fd5e6 100644 --- a/test/StringUtilsTest.cpp +++ b/test/StringUtilsTest.cpp @@ -17,6 +17,7 @@ #include "util/Forward.h" #include "util/Generator.h" #include "util/StringUtils.h" +#include "util/StringUtilsImpl.h" using ad_utility::constantTimeEquals; using ad_utility::constexprStrCat; @@ -109,22 +110,22 @@ TEST(StringUtilsTest, listToString) { multiValueVector, " -> "); /* - `std::ranges::views` can cause dangling pointers, if a `std::identity` is + `ql::ranges::views` can cause dangling pointers, if a `std::identity` is called with one, that returns r-values. */ /* - TODO Do a test, where the `std::views::transform` uses an r-value vector, + TODO Do a test, where the `ql::views::transform` uses an r-value vector, once we no longer support `gcc-11`. The compiler has a bug, where it doesn't allow that code, even though it's correct. */ - auto plus10View = std::views::transform( + auto plus10View = ql::views::transform( multiValueVector, [](const int& num) -> int { return num + 10; }); doTestForAllOverloads("50,51,52,53", plus10View, plus10View, ","); - auto identityView = std::views::transform(multiValueVector, std::identity{}); + auto identityView = ql::views::transform(multiValueVector, std::identity{}); doTestForAllOverloads("40,41,42,43", identityView, identityView, ","); - // Test, that uses an actual `std::ranges::input_range`. That is, a range who + // Test, that uses an actual `ql::ranges::input_range`. That is, a range who // doesn't know it's own size and can only be iterated once. // Returns the content of a given vector, element by element. diff --git a/test/ThreadSafeQueueTest.cpp b/test/ThreadSafeQueueTest.cpp index 055c01f4db..2588ce5551 100644 --- a/test/ThreadSafeQueueTest.cpp +++ b/test/ThreadSafeQueueTest.cpp @@ -146,7 +146,7 @@ TEST(ThreadSafeQueue, Concurrency) { // order, for the `ThreadSafeQueue` the order is unspecified and we only // check the content. if (ad_utility::isInstantiation) { - std::ranges::sort(result); + ql::ranges::sort(result); } EXPECT_THAT(result, ::testing::ElementsAreArray( std::views::iota(0UL, numValues * numThreads))); @@ -253,13 +253,13 @@ TEST(ThreadSafeQueue, DisablePush) { if (ad_utility::similarToInstantiation) { // When terminating early, we cannot actually say much about the result, // other than that it contains no duplicate values - std::ranges::sort(result); + ql::ranges::sort(result); EXPECT_TRUE(std::unique(result.begin(), result.end()) == result.end()); } else { // For the ordered queue we have the guarantee that all the pushed values // were in order. EXPECT_THAT(result, - ::testing::ElementsAreArray(std::views::iota(0U, 400U))); + ::testing::ElementsAreArray(ql::views::iota(0U, 400U))); } }; runWithBothQueueTypes(runTest); @@ -309,7 +309,7 @@ TEST(ThreadSafeQueue, SafeExceptionHandling) { // 1. Queue, 2. WorkerThreads, 3. `Cleanup` that finishes the queue. absl::Cleanup cleanup{[&queue] { queue.finish(); }}; - for ([[maybe_unused]] auto i : std::views::iota(0u, numValues)) { + for ([[maybe_unused]] auto i : ql::views::iota(0u, numValues)) { auto opt = queue.pop(); if (!opt) { return; @@ -400,7 +400,7 @@ TEST(ThreadSafeQueue, queueManager) { // order, for the `ThreadSafeQueue` the order is unspecified and we only // check the content. if (ad_utility::isInstantiation) { - std::ranges::sort(result); + ql::ranges::sort(result); } EXPECT_THAT(result, ::testing::ElementsAreArray( std::views::iota(0UL, numValues))); diff --git a/test/ViewsTest.cpp b/test/ViewsTest.cpp index addf37af87..24fc744afb 100644 --- a/test/ViewsTest.cpp +++ b/test/ViewsTest.cpp @@ -107,7 +107,7 @@ TEST(Views, uniqueBlockView) { i = nextI; } - auto unique = std::views::join( + auto unique = ql::views::join( ad_utility::OwningView{ad_utility::uniqueBlockView(inputs)}); std::vector result; for (const auto& element : unique) { @@ -124,14 +124,14 @@ TEST(Views, uniqueBlockView) { TEST(Views, owningView) { using namespace ad_utility; // Static asserts for the desired concepts. - static_assert(std::ranges::input_range>>); + static_assert(ql::ranges::input_range>>); static_assert( - !std::ranges::forward_range>>); - static_assert(std::ranges::random_access_range>>); + !ql::ranges::forward_range>>); + static_assert(ql::ranges::random_access_range>>); auto toVec = [](auto& range) { std::vector result; - std::ranges::copy(range, std::back_inserter(result)); + ql::ranges::copy(range, std::back_inserter(result)); return result; }; @@ -166,7 +166,7 @@ TEST(Views, integerRange) { } std::vector actual; - std::ranges::copy(ad_utility::integerRange(42u), std::back_inserter(actual)); + ql::ranges::copy(ad_utility::integerRange(42u), std::back_inserter(actual)); ASSERT_THAT(actual, ::testing::ElementsAreArray(expected)); } @@ -202,7 +202,7 @@ std::string_view toView(std::span span) { TEST(Views, verifyLineByLineWorksWithMinimalChunks) { auto range = std::string_view{"\nabc\ndefghij\n"} | - std::views::transform([](char c) { return std::ranges::single_view(c); }); + ql::views::transform([](char c) { return ql::ranges::single_view(c); }); auto lineByLineGenerator = ad_utility::reChunkAtSeparator(std::move(range), '\n'); @@ -224,8 +224,8 @@ TEST(Views, verifyLineByLineWorksWithMinimalChunks) { // __________________________________________________________________________ TEST(Views, verifyLineByLineWorksWithNoTrailingNewline) { - auto range = std::string_view{"abc"} | std::views::transform([](char c) { - return std::ranges::single_view(c); + auto range = std::string_view{"abc"} | ql::views::transform([](char c) { + return ql::ranges::single_view(c); }); auto lineByLineGenerator = diff --git a/test/backports/CMakeLists.txt b/test/backports/CMakeLists.txt new file mode 100644 index 0000000000..3e3f2e9865 --- /dev/null +++ b/test/backports/CMakeLists.txt @@ -0,0 +1,6 @@ + +add_executable(AlgorithmBackportTests algorithmTest.cpp) +add_executable(DebugJoinView DebugJoinView.cpp) +qlever_target_link_libraries(DebugJoinView) +target_link_libraries(AlgorithmBackportTests GTest::gtest GTest::gmock_main) +gtest_discover_tests(AlgorithmBackportTests AlgorithmBackportTests) \ No newline at end of file diff --git a/test/backports/DebugJoinView.cpp b/test/backports/DebugJoinView.cpp new file mode 100644 index 0000000000..ff372cc7fa --- /dev/null +++ b/test/backports/DebugJoinView.cpp @@ -0,0 +1,38 @@ +// +// Created by kalmbacj on 12/10/24. +// + +#include + +#include "engine/idTable/IdTable.h" +#include "util/Generator.h" +#include "util/Views.h" + +cppcoro::generator> inner() { return {}; } + +auto joinOwning() { return ql::views::join(ad_utility::OwningView{inner()}); } + +/* +auto joinOwning() { + return +ql::views::join(ad_utility::OwningView{std::vector>{}}); +} +*/ + +auto vec() { + std::vector vec; + vec.push_back(joinOwning()); + return vec; +} + +auto joinOuter() { + // return ad_utility::OwningView{vec()}; + // return ql::views::join(ad_utility::OwningView{vec()}); + return ql::views::join(ad_utility::OwningViewNoConst{vec()}); +} + +int main() { + auto view = joinOuter(); + [[maybe_unused]] auto it = view.begin(); +} diff --git a/test/backports/algorithmTest.cpp b/test/backports/algorithmTest.cpp new file mode 100644 index 0000000000..0e213c192d --- /dev/null +++ b/test/backports/algorithmTest.cpp @@ -0,0 +1,9 @@ +// +// Created by kalmbacj on 12/6/24. +// + +#include + +#include "backports/algorithm.h" + +TEST(Range, Sort) {} diff --git a/test/engine/BindTest.cpp b/test/engine/BindTest.cpp index cecec3c4c7..34ef0eb370 100644 --- a/test/engine/BindTest.cpp +++ b/test/engine/BindTest.cpp @@ -95,7 +95,7 @@ TEST( auto* qec = ad_utility::testing::getQec(); IdTable table{1, ad_utility::makeUnlimitedAllocator()}; table.resize(Bind::CHUNK_SIZE + 1); - std::ranges::fill(table, row); + ql::ranges::fill(table, row); auto valuesTree = ad_utility::makeExecutionTree( qec, table.clone(), Vars{Variable{"?a"}}, false, std::vector{}, LocalVocab{}, std::nullopt, true); @@ -109,7 +109,7 @@ TEST( row = IdTable::row_type{2}; row[0] = val; row[1] = val; - std::ranges::fill(table, row); + ql::ranges::fill(table, row); { qec->getQueryTreeCache().clearAll(); auto result = bind.getResult(false, ComputationMode::FULLY_MATERIALIZED); diff --git a/test/engine/CartesianProductJoinTest.cpp b/test/engine/CartesianProductJoinTest.cpp index 90488dca5c..8727aa223a 100644 --- a/test/engine/CartesianProductJoinTest.cpp +++ b/test/engine/CartesianProductJoinTest.cpp @@ -406,8 +406,8 @@ class CartesianProductJoinLazyTest // `start` to `end` wrapped as Ids. static void fillColumn(IdTable& table, size_t column, int64_t start, int64_t end) { - std::ranges::copy( - std::views::iota(start, end) | std::views::transform(Id::makeFromInt), + ql::ranges::copy( + ql::views::iota(start, end) | ql::views::transform(Id::makeFromInt), table.getColumn(column).begin()); } }; @@ -476,8 +476,8 @@ TEST_P(CartesianProductJoinLazyTest, leftTableBiggerThanChunk) { bigTable.addEmptyColumn(); bigTable.addEmptyColumn(); auto fillWithVocabValue = [&bigTable](size_t column, uint64_t vocabIndex) { - std::ranges::fill(bigTable.getColumn(column), - Id::makeFromVocabIndex(VocabIndex::make(vocabIndex))); + ql::ranges::fill(bigTable.getColumn(column), + Id::makeFromVocabIndex(VocabIndex::make(vocabIndex))); }; fillWithVocabValue(3, 100); diff --git a/test/engine/DistinctTest.cpp b/test/engine/DistinctTest.cpp index 0b66ca9748..c20d0ba5c6 100644 --- a/test/engine/DistinctTest.cpp +++ b/test/engine/DistinctTest.cpp @@ -74,7 +74,7 @@ TEST(Distinct, testChunkEdgeCases) { { input.resize(1); row[0] = Id::makeFromInt(0); - std::ranges::fill(input, row); + ql::ranges::fill(input, row); IdTable result = distinct.outOfPlaceDistinct<1>(input); ASSERT_EQ(makeIdTableFromVector({{0}}, &Id::makeFromInt), result); @@ -83,7 +83,7 @@ TEST(Distinct, testChunkEdgeCases) { { input.resize(Distinct::CHUNK_SIZE + 1); row[0] = Id::makeFromInt(0); - std::ranges::fill(input, row); + ql::ranges::fill(input, row); IdTable result = distinct.outOfPlaceDistinct<1>(input); ASSERT_EQ(makeIdTableFromVector({{0}}, &Id::makeFromInt), result); @@ -92,7 +92,7 @@ TEST(Distinct, testChunkEdgeCases) { { input.resize(Distinct::CHUNK_SIZE + 1); row[0] = Id::makeFromInt(0); - std::ranges::fill(input, row); + ql::ranges::fill(input, row); input.at(Distinct::CHUNK_SIZE, 0) = Id::makeFromInt(1); IdTable result = distinct.outOfPlaceDistinct<1>(input); @@ -102,7 +102,7 @@ TEST(Distinct, testChunkEdgeCases) { { input.resize(2 * Distinct::CHUNK_SIZE); row[0] = Id::makeFromInt(0); - std::ranges::fill(input, row); + ql::ranges::fill(input, row); IdTable result = distinct.outOfPlaceDistinct<1>(input); ASSERT_EQ(makeIdTableFromVector({{0}}, &Id::makeFromInt), result); @@ -111,7 +111,7 @@ TEST(Distinct, testChunkEdgeCases) { { input.resize(2 * Distinct::CHUNK_SIZE + 2); row[0] = Id::makeFromInt(0); - std::ranges::fill(input, row); + ql::ranges::fill(input, row); input.at(2 * Distinct::CHUNK_SIZE + 1, 0) = Id::makeFromInt(1); IdTable result = distinct.outOfPlaceDistinct<1>(input); diff --git a/test/engine/IndexScanTest.cpp b/test/engine/IndexScanTest.cpp index 6663beaf3f..2c526787a3 100644 --- a/test/engine/IndexScanTest.cpp +++ b/test/engine/IndexScanTest.cpp @@ -64,7 +64,7 @@ void testLazyScan(Permutation::IdTableGenerator partialLazyScanResult, if (limitOffset.isUnconstrained()) { for (auto [lower, upper] : expectedRows) { - for (auto index : std::views::iota(lower, upper)) { + for (auto index : ql::views::iota(lower, upper)) { expected.push_back(resFullScan.at(index)); } } diff --git a/test/engine/ValuesForTesting.h b/test/engine/ValuesForTesting.h index 6009ccd216..c02a9826bc 100644 --- a/test/engine/ValuesForTesting.h +++ b/test/engine/ValuesForTesting.h @@ -63,10 +63,9 @@ class ValuesForTesting : public Operation { resultSortedColumns_{std::move(sortedColumns)}, localVocab_{std::move(localVocab)}, multiplicity_{std::nullopt} { - AD_CONTRACT_CHECK( - std::ranges::all_of(tables_, [this](const IdTable& table) { - return variables_.size() == table.numColumns(); - })); + AD_CONTRACT_CHECK(ql::ranges::all_of(tables_, [this](const IdTable& table) { + return variables_.size() == table.numColumns(); + })); size_t totalRows = 0; for (const IdTable& idTable : tables_) { totalRows += idTable.numRows(); @@ -125,7 +124,7 @@ class ValuesForTesting : public Operation { // ___________________________________________________________________________ string getCacheKeyImpl() const override { std::stringstream str; - auto numRowsView = tables_ | std::views::transform(&IdTable::numRows); + auto numRowsView = tables_ | ql::views::transform(&IdTable::numRows); auto totalNumRows = std::reduce(numRowsView.begin(), numRowsView.end(), 0); auto numCols = tables_.empty() ? 0 : tables_.at(0).numColumns(); str << "Values for testing with " << numCols << " columns and " @@ -177,7 +176,7 @@ class ValuesForTesting : public Operation { vector getChildren() override { return {}; } bool knownEmptyResult() override { - return std::ranges::all_of( + return ql::ranges::all_of( tables_, [](const IdTable& table) { return table.empty(); }); } @@ -189,9 +188,9 @@ class ValuesForTesting : public Operation { continue; } bool containsUndef = - std::ranges::any_of(tables_, [&i](const IdTable& table) { - return std::ranges::any_of(table.getColumn(i), - [](Id id) { return id.isUndefined(); }); + ql::ranges::any_of(tables_, [&i](const IdTable& table) { + return ql::ranges::any_of(table.getColumn(i), + [](Id id) { return id.isUndefined(); }); }); using enum ColumnIndexAndTypeInfo::UndefStatus; m[variables_.at(i).value()] = ColumnIndexAndTypeInfo{ diff --git a/test/engine/idTable/CompressedExternalIdTableTest.cpp b/test/engine/idTable/CompressedExternalIdTableTest.cpp index 49c5f2a23f..a5d4fde240 100644 --- a/test/engine/idTable/CompressedExternalIdTableTest.cpp +++ b/test/engine/idTable/CompressedExternalIdTableTest.cpp @@ -33,10 +33,10 @@ auto idTableFromBlockGenerator = [](auto& generator) -> CopyableIdTable<0> { size_t numColumns = result.numColumns(); size_t size = result.size(); result.resize(result.size() + block.size()); - for (auto i : std::views::iota(0U, numColumns)) { + for (auto i : ql::views::iota(0U, numColumns)) { decltype(auto) blockCol = block.getColumn(i); decltype(auto) resultCol = result.getColumn(i); - std::ranges::copy(blockCol, resultCol.begin() + size); + ql::ranges::copy(blockCol, resultCol.begin() + size); } } return result; @@ -75,8 +75,8 @@ TEST(CompressedExternalIdTable, compressedExternalIdTableWriter) { using namespace ::testing; std::vector> result; - auto tr = std::ranges::transform_view(generators, idTableFromBlockGenerator); - std::ranges::copy(tr, std::back_inserter(result)); + auto tr = ql::ranges::transform_view(generators, idTableFromBlockGenerator); + ql::ranges::copy(tr, std::back_inserter(result)); ASSERT_THAT(result, ElementsAreArray(tables)); } @@ -103,7 +103,7 @@ void testExternalSorterImpl(size_t numDynamicColumns, size_t numRows, writer.push(row); } - std::ranges::sort(randomTable, SortByOSP{}); + ql::ranges::sort(randomTable, SortByOSP{}); if (mergeMultipleTimes) { writer.moveResultOnMerge() = false; @@ -114,7 +114,7 @@ void testExternalSorterImpl(size_t numDynamicColumns, size_t numRows, // number of inputs. auto blocksize = k == 1 ? 1 : 17; using namespace ::testing; - auto generator = k == 0 ? std::views::join(ad_utility::OwningView{ + auto generator = k == 0 ? ql::views::join(ad_utility::OwningView{ writer.getSortedBlocks(blocksize)}) : writer.sortedView(); if (mergeMultipleTimes || k == 0) { diff --git a/test/index/PatternCreatorTest.cpp b/test/index/PatternCreatorTest.cpp index 3cdfa9b790..698b2e1ec7 100644 --- a/test/index/PatternCreatorTest.cpp +++ b/test/index/PatternCreatorTest.cpp @@ -106,7 +106,7 @@ auto createExamplePatterns(PatternCreator& creator) { push({V(3), V(11), V(29)}, false, 0); push({V(3), V(11), V(45)}, false, 0); - std::ranges::sort(expected, SortByOSP{}); + ql::ranges::sort(expected, SortByOSP{}); auto tripleOutputs = std::move(creator).getTripleSorter(); auto& triples = *tripleOutputs.triplesWithSubjectPatternsSortedByOsp_; static constexpr size_t numCols = NumColumnsIndexBuilding + 1; @@ -160,7 +160,7 @@ void assertPatternContents(const std::string& filename, expectedTriples.push_back(std::array{V(0), pat, I(0)}); expectedTriples.push_back(std::array{V(1), pat, I(1)}); expectedTriples.push_back(std::array{V(3), pat, I(0)}); - std::ranges::sort(expectedTriples, SortByPSO{}); + ql::ranges::sort(expectedTriples, SortByPSO{}); EXPECT_THAT(addedTriples, ::testing::ElementsAreArray(expectedTriples)); } diff --git a/test/util/IdTableHelpers.cpp b/test/util/IdTableHelpers.cpp index 0b3b0a6a2e..55ac6209f9 100644 --- a/test/util/IdTableHelpers.cpp +++ b/test/util/IdTableHelpers.cpp @@ -23,16 +23,16 @@ void compareIdTableWithExpectedContent( std::stringstream traceMessage{}; auto writeIdTableToStream = [&traceMessage](const IdTable& idTable) { - std::ranges::for_each(idTable, - [&traceMessage](const auto& row) { - // TODO Use std::views::join_with for both - // loops. - for (size_t i = 0; i < row.numColumns(); i++) { - traceMessage << row[i] << " "; - } - traceMessage << "\n"; - }, - {}); + ql::ranges::for_each(idTable, + [&traceMessage](const auto& row) { + // TODO Use ql::views::join_with for both + // loops. + for (size_t i = 0; i < row.numColumns(); i++) { + traceMessage << row[i] << " "; + } + traceMessage << "\n"; + }, + {}); }; traceMessage << "compareIdTableWithExpectedContent comparing IdTable\n"; @@ -48,13 +48,13 @@ void compareIdTableWithExpectedContent( if (resultMustBeSortedByJoinColumn) { // Is the table sorted by join column? - ASSERT_TRUE(std::ranges::is_sorted(localTable.getColumn(joinColumn))); + ASSERT_TRUE(ql::ranges::is_sorted(localTable.getColumn(joinColumn))); } // Sort both the table and the expectedContent, so that both have a definite // form for comparison. - std::ranges::sort(localTable, std::ranges::lexicographical_compare); - std::ranges::sort(localExpectedContent, std::ranges::lexicographical_compare); + ql::ranges::sort(localTable, ql::ranges::lexicographical_compare); + ql::ranges::sort(localExpectedContent, ql::ranges::lexicographical_compare); ASSERT_EQ(localTable, localExpectedContent); } @@ -76,7 +76,7 @@ IdTable generateIdTable( table.resize(numberRows); // Fill the table. - std::ranges::for_each( + ql::ranges::for_each( /* The iterator of an `IdTable` dereference to an `row_reference_restricted`, which only allows write access, if it is a r-value. Otherwise, we can't @@ -90,7 +90,7 @@ IdTable generateIdTable( std::vector generatedRow = rowGenerator(); AD_CONTRACT_CHECK(generatedRow.size() == numberColumns); - std::ranges::copy(generatedRow, AD_FWD(row).begin()); + ql::ranges::copy(generatedRow, AD_FWD(row).begin()); }); return table; @@ -105,22 +105,22 @@ IdTable createRandomlyFilledIdTable( AD_CONTRACT_CHECK(numberRows > 0 && numberColumns > 0); // Views for clearer access. - auto joinColumnNumberView = std::views::keys(joinColumnWithGenerator); - auto joinColumnGeneratorView = std::views::values(joinColumnWithGenerator); + auto joinColumnNumberView = ql::views::keys(joinColumnWithGenerator); + auto joinColumnGeneratorView = ql::views::values(joinColumnWithGenerator); // Are all the join column numbers within the max column number? - AD_CONTRACT_CHECK(std::ranges::all_of( + AD_CONTRACT_CHECK(ql::ranges::all_of( joinColumnNumberView, [&numberColumns](const size_t num) { return num < numberColumns; })); // Are there no duplicates in the join column numbers? std::vector sortedJoinColumnNumbers = ad_utility::transform(joinColumnNumberView, std::identity{}); - std::ranges::sort(sortedJoinColumnNumbers); + ql::ranges::sort(sortedJoinColumnNumbers); AD_CONTRACT_CHECK(std::ranges::unique(sortedJoinColumnNumbers).empty()); // Are all the functions for generating join column entries not nullptr? - AD_CONTRACT_CHECK(std::ranges::all_of( + AD_CONTRACT_CHECK(ql::ranges::all_of( joinColumnGeneratorView, [](auto func) { return func != nullptr; })); // The random number generators for normal entries. @@ -134,10 +134,10 @@ IdTable createRandomlyFilledIdTable( // Assigning the column number to a generator function. std::vector*> columnToGenerator( numberColumns, &normalEntryGenerator); - std::ranges::for_each(joinColumnWithGenerator, - [&columnToGenerator](auto& pair) { - columnToGenerator.at(pair.first) = &pair.second; - }); + ql::ranges::for_each(joinColumnWithGenerator, + [&columnToGenerator](auto& pair) { + columnToGenerator.at(pair.first) = &pair.second; + }); // Creating the table. return generateIdTable( @@ -192,7 +192,7 @@ IdTable createRandomlyFilledIdTable( Is the lower bound smaller, or equal, to the upper bound? And is the upper bound smaller, or equal, to the maximum size of an IdTable entry? */ - AD_CONTRACT_CHECK(std::ranges::all_of( + AD_CONTRACT_CHECK(ql::ranges::all_of( joinColumnsAndBounds, [](const JoinColumnAndBounds& j) { return j.lowerBound_ <= j.upperBound_ && j.upperBound_ <= maxIdSize; })); diff --git a/test/util/RandomTestHelpers.h b/test/util/RandomTestHelpers.h index 202fccfa41..e45ff48c69 100644 --- a/test/util/RandomTestHelpers.h +++ b/test/util/RandomTestHelpers.h @@ -35,7 +35,7 @@ inline std::array createArrayOfRandomSeeds( ad_utility::RandomSeed::make(std::random_device{}())) { RandomSeedGenerator generator{std::move(seed)}; std::array seeds{}; - std::ranges::generate(seeds, - [&generator]() { return std::invoke(generator); }); + ql::ranges::generate(seeds, + [&generator]() { return std::invoke(generator); }); return seeds; } From 27f4799ae3faf1e1ffd86c86fd98e9bf90ef027f Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Sat, 14 Dec 2024 22:52:29 +0100 Subject: [PATCH 04/14] Implement DESCRIBE (#1624) Implement DESCRIBE according to the Concise Bounded Description (CBD) specification: https://www.w3.org/submissions/2005/SUBM-CBD-20050603 . That is, do not only show those triples where the subject is one of the resources to describe but recursively expand reification nodes. The current implementation recursively expands blank nodes. Here is an example query on Wikidata, where that makes a difference: https://qlever.cs.uni-freiburg.de/wikidata/obesyx In a future PR, add an option to configure which other nodes to consider for expansion. For example, for Wikidata, we would also like to expand all object IRIs that start with the prefix `http://www.wikidata.org/entity/statement/`. Co-authored-by: Hannah Bast --- src/engine/CMakeLists.txt | 3 +- src/engine/CheckUsePatternTrick.cpp | 6 +- src/engine/Describe.cpp | 247 ++++++++++++++++++ src/engine/Describe.h | 78 ++++++ src/engine/Operation.h | 9 +- src/engine/QueryPlanner.cpp | 14 + src/engine/QueryPlanner.h | 1 + src/engine/idTable/IdTable.h | 5 +- src/parser/CMakeLists.txt | 4 +- src/parser/DatasetClauses.cpp | 19 ++ src/parser/DatasetClauses.h | 24 ++ src/parser/GraphPatternOperation.cpp | 7 +- src/parser/GraphPatternOperation.h | 27 +- src/parser/ParsedQuery.cpp | 21 +- src/parser/ParsedQuery.h | 24 +- .../sparqlParser/SparqlQleverVisitor.cpp | 92 ++++++- src/parser/sparqlParser/SparqlQleverVisitor.h | 18 +- test/QueryPlannerTest.cpp | 20 +- test/QueryPlannerTestHelpers.h | 12 +- test/SparqlAntlrParserTest.cpp | 55 +++- test/SparqlAntlrParserTestHelpers.h | 29 ++ test/engine/CMakeLists.txt | 1 + test/engine/DescribeTest.cpp | 180 +++++++++++++ 23 files changed, 812 insertions(+), 84 deletions(-) create mode 100644 src/engine/Describe.cpp create mode 100644 src/engine/Describe.h create mode 100644 src/parser/DatasetClauses.cpp create mode 100644 src/parser/DatasetClauses.h create mode 100644 test/engine/DescribeTest.cpp diff --git a/src/engine/CMakeLists.txt b/src/engine/CMakeLists.txt index cbfb3344c3..5789f50023 100644 --- a/src/engine/CMakeLists.txt +++ b/src/engine/CMakeLists.txt @@ -13,5 +13,6 @@ add_library(engine VariableToColumnMap.cpp ExportQueryExecutionTrees.cpp CartesianProductJoin.cpp TextIndexScanForWord.cpp TextIndexScanForEntity.cpp TextLimit.cpp LazyGroupBy.cpp GroupByHashMapOptimization.cpp SpatialJoin.cpp - CountConnectedSubgraphs.cpp SpatialJoinAlgorithms.cpp PathSearch.cpp ExecuteUpdate.cpp) + CountConnectedSubgraphs.cpp SpatialJoinAlgorithms.cpp PathSearch.cpp ExecuteUpdate.cpp + Describe.cpp) qlever_target_link_libraries(engine util index parser sparqlExpressions http SortPerformanceEstimator Boost::iostreams s2) diff --git a/src/engine/CheckUsePatternTrick.cpp b/src/engine/CheckUsePatternTrick.cpp index 866caa2f9b..e7da58ea14 100644 --- a/src/engine/CheckUsePatternTrick.cpp +++ b/src/engine/CheckUsePatternTrick.cpp @@ -72,9 +72,9 @@ bool isVariableContainedInGraphPatternOperation( } else if constexpr (std::is_same_v) { return ad_utility::contains(arg.visibleVariables_, variable); } else { - static_assert(std::is_same_v || - std::is_same_v || - std::is_same_v); + static_assert( + std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v); // The `TransPath` is set up later in the query planning, when this // function should not be called anymore. AD_FAIL(); diff --git a/src/engine/Describe.cpp b/src/engine/Describe.cpp new file mode 100644 index 0000000000..5fb01dbe6c --- /dev/null +++ b/src/engine/Describe.cpp @@ -0,0 +1,247 @@ +// Copyright 2024, University of Freiburg, +// Chair of Algorithms and Data Structures +// Author: Johannes Kalmbach + +#include "engine/Describe.h" + +#include "../../test/engine/ValuesForTesting.h" +#include "engine/IndexScan.h" +#include "engine/Join.h" + +// _____________________________________________________________________________ +Describe::Describe(QueryExecutionContext* qec, + std::shared_ptr subtree, + parsedQuery::Describe describe) + : Operation{qec}, + subtree_{std::move(subtree)}, + describe_{std::move(describe)} { + // If the DESCRIBE query has no WHERE clause, `subtree_` is the neutral + // element, but never `nullptr`. + AD_CORRECTNESS_CHECK(subtree_ != nullptr); +} + +// _____________________________________________________________________________ +std::vector Describe::getChildren() { + return {subtree_.get()}; +} + +// _____________________________________________________________________________ +string Describe::getCacheKeyImpl() const { + // The cache key must represent the `resources_` (the variables and IRIs of + // the DESCRIBE clause) and the `subtree_` (the WHERE clause). + std::string result = absl::StrCat("DESCRIBE ", subtree_->getCacheKey(), " "); + for (const auto& resource : describe_.resources_) { + if (std::holds_alternative(resource)) { + result.append( + std::get(resource).toStringRepresentation()); + } else { + result.append(absl::StrCat( + "column #", + subtree_->getVariableColumnOrNullopt(std::get(resource)) + .value_or(static_cast(-1)), + " ")); + } + } + + // Add the names of the default graphs (from the FROM clauses) to the cache + // key, in a deterministic order. + // + // NOTE: The default and named graphs are also part of the cache key of the + // `subtree_`. However, the named graphs only determine the result for + // `subtree_` (the resources to be described), whereas the default graphs + // also determine which triples for these resources become part of the result. + const auto& defaultGraphs = describe_.datasetClauses_.defaultGraphs_; + if (defaultGraphs.has_value()) { + std::vector graphIdVec; + std::ranges::transform(defaultGraphs.value(), + std::back_inserter(graphIdVec), + &TripleComponent::toRdfLiteral); + std::ranges::sort(graphIdVec); + absl::StrAppend(&result, + "\nFiltered by Graphs:", absl::StrJoin(graphIdVec, " ")); + } + return result; +} + +// _____________________________________________________________________________ +string Describe::getDescriptor() const { return "DESCRIBE"; } + +// _____________________________________________________________________________ +size_t Describe::getResultWidth() const { return 3; } + +// As DESCRIBE is never part of the query planning (it is always the root +// operation), we can return dummy values for the following functions. +size_t Describe::getCostEstimate() { return 2 * subtree_->getCostEstimate(); } +uint64_t Describe::getSizeEstimateBeforeLimit() { + return subtree_->getSizeEstimate() * 2; +} +float Describe::getMultiplicity([[maybe_unused]] size_t col) { return 1.0f; } +bool Describe::knownEmptyResult() { return false; } + +// The result cannot easily be sorted, as it involves recursive expanding of +// graphs. +vector Describe::resultSortedOn() const { return {}; } + +// The result always has three variables `?subject`, `?predicate`, `?object`. +// +// NOTE: These variable names are hardcoded in the implicit CONSTRUCT query +// created in `SparqlQleverVisitor::visitDescribe`. +VariableToColumnMap Describe::computeVariableToColumnMap() const { + using V = Variable; + auto col = makeAlwaysDefinedColumn; + return {{V("?subject"), col(0)}, + {V("?predicate"), col(1)}, + {V("?object"), col(2)}}; +} + +// A helper function for the recursive BFS. Return those `Id`s from `input` (an +// `IdTable` with one column) that are blank nodes and not in `alreadySeen`, +// with duplicates removed. The returned `Id`s are added to `alreadySeen`. +static IdTable getNewBlankNodes( + const auto& allocator, ad_utility::HashSetWithMemoryLimit& alreadySeen, + std::span input) { + IdTable result{1, allocator}; + result.resize(input.size()); + decltype(auto) resultColumn = result.getColumn(0); + size_t i = 0; + for (Id id : input) { + if (id.getDatatype() != Datatype::BlankNodeIndex) { + continue; + } + auto [it, isNew] = alreadySeen.emplace(id); + if (!isNew) { + continue; + } + resultColumn[i] = id; + ++i; + } + result.resize(i); + return result; +} + +// _____________________________________________________________________________ +void Describe::recursivelyAddBlankNodes( + IdTable& finalResult, LocalVocab& localVocab, + ad_utility::HashSetWithMemoryLimit& alreadySeen, IdTable blankNodes) { + AD_CORRECTNESS_CHECK(blankNodes.numColumns() == 1); + + // If there are no more `blankNodes` to explore, we are done. + if (blankNodes.empty()) { + return; + } + + // Expand the `blankNodes` by joining them with the full index and add the + // resulting triples to the `finalResult`. + // + // TODO Make the result of DESCRIBE lazy, then we can avoid the + // additional copy here. + auto table = + makeAndExecuteJoinWithFullIndex(std::move(blankNodes), localVocab); + finalResult.insertAtEnd(table); + + // Compute the set of newly found blank nodes and recurse. + auto newBlankNodes = + getNewBlankNodes(allocator(), alreadySeen, table.getColumn(2)); + recursivelyAddBlankNodes(finalResult, localVocab, alreadySeen, + std::move(newBlankNodes)); +} + +// _____________________________________________________________________________ +IdTable Describe::makeAndExecuteJoinWithFullIndex( + IdTable input, LocalVocab& localVocab) const { + AD_CORRECTNESS_CHECK(input.numColumns() == 1); + + // Create a `Join` operation that joins `input` (with column `?subject`) with + // the full index (with columns `?subject`, `?predicate`, `?object`) on the + // `?subject` column. + using V = Variable; + auto subjectVar = V{"?subject"}; + auto valuesOp = ad_utility::makeExecutionTree( + getExecutionContext(), std::move(input), + std::vector>{subjectVar}); + SparqlTripleSimple triple{subjectVar, V{"?predicate"}, V{"?object"}}; + auto indexScan = ad_utility::makeExecutionTree( + getExecutionContext(), Permutation::SPO, triple, + describe_.datasetClauses_.defaultGraphs_); + auto joinColValues = valuesOp->getVariableColumn(subjectVar); + auto joinColScan = indexScan->getVariableColumn(subjectVar); + auto join = ad_utility::makeExecutionTree( + getExecutionContext(), std::move(valuesOp), std::move(indexScan), + joinColValues, joinColScan); + + // Compute the result of the `join` and select the columns `?subject`, + // `?predicate`, `?object`. + // + // NOTE: Typically, the join result has already those exact columns, in which + // case the `selectColumns` operation is a no-op. Note sure when this is not + // the case, but better safe than sorry. + auto result = join->getResult(); + IdTable resultTable = result->idTable().clone(); + ColumnIndex s = join->getVariableColumn(V{"?subject"}); + ColumnIndex p = join->getVariableColumn(V{"?predicate"}); + ColumnIndex o = join->getVariableColumn(V{"?object"}); + resultTable.setColumnSubset(std::vector{s, p, o}); + + // The `indexScan` might have added some delta triples with local vocab IDs, + // so make sure to merge them into the `localVocab`. + localVocab.mergeWith(std::span{&result->localVocab(), 1}); + + return resultTable; +} + +// _____________________________________________________________________________ +IdTable Describe::getIdsToDescribe(const Result& result, + LocalVocab& localVocab) const { + // First collect the `Id`s in a hash set, in order to remove duplicates. + ad_utility::HashSetWithMemoryLimit idsToDescribe{allocator()}; + const auto& vocab = getIndex().getVocab(); + for (const auto& resource : describe_.resources_) { + if (std::holds_alternative(resource)) { + // For an IRI, add the corresponding ID to `idsToDescribe`. + idsToDescribe.insert( + TripleComponent{std::get(resource)}.toValueId( + vocab, localVocab)); + } else { + // For a variable, add all IDs that match the variable in the `result` of + // the WHERE clause to `idsToDescribe`. + const auto& var = std::get(resource); + auto column = subtree_->getVariableColumnOrNullopt(var); + if (!column.has_value()) { + continue; + } + for (Id id : result.idTable().getColumn(column.value())) { + idsToDescribe.insert(id); + } + } + } + + // Copy the `Id`s from the hash set to an `IdTable`. + IdTable idsAsTable{1, allocator()}; + idsAsTable.resize(idsToDescribe.size()); + std::ranges::copy(idsToDescribe, idsAsTable.getColumn(0).begin()); + return idsAsTable; +} + +// _____________________________________________________________________________ +ProtoResult Describe::computeResult([[maybe_unused]] bool requestLaziness) { + LocalVocab localVocab; + // Compute the results of the WHERE clause and extract the `Id`s to describe. + // + // TODO Would we benefit from computing `resultOfWhereClause` lazily? + // Probably not, because we have to deduplicate the whole input anyway. + auto resultOfWhereClause = subtree_->getResult(); + auto idsAsTable = getIdsToDescribe(*resultOfWhereClause, localVocab); + + // Get all triples with the `Id`s as subject. + auto resultTable = + makeAndExecuteJoinWithFullIndex(std::move(idsAsTable), localVocab); + + // Recursively follow all blank nodes. + ad_utility::HashSetWithMemoryLimit alreadySeen{allocator()}; + auto blankNodes = + getNewBlankNodes(allocator(), alreadySeen, resultTable.getColumn(2)); + recursivelyAddBlankNodes(resultTable, localVocab, alreadySeen, + std::move(blankNodes)); + + return {std::move(resultTable), resultSortedOn(), std::move(localVocab)}; +} diff --git a/src/engine/Describe.h b/src/engine/Describe.h new file mode 100644 index 0000000000..da0fa95bf6 --- /dev/null +++ b/src/engine/Describe.h @@ -0,0 +1,78 @@ +// Copyright 2024, University of Freiburg, +// Chair of Algorithms and Data Structures +// Author: Johannes Kalmbach + +#pragma once + +#include "engine/Operation.h" +#include "parser/GraphPatternOperation.h" + +// Operation for DESCRIBE queries according to the Concise Bounded Description +// (CBD) specification: https://www.w3.org/submissions/2005/SUBM-CBD-20050603 . +// +// NOTE: The current implementation recursively expands blank nodes. This can +// be expanded to other reification schemes relatively easily (for example, +// for Wikidata, also expand all object IRIs that start with the prefix +// `http://www.wikidata.org/entity/statement/`). +class Describe : public Operation { + private: + // The query execution tree for computing the WHERE clause of the DESCRIBE. + // Must be the neutral element if the DESCRIBE query has no WHERE clause. + std::shared_ptr subtree_; + + // The specification of the DESCRIBE clause. + parsedQuery::Describe describe_; + + public: + // Create a new DESCRIBE operation. + Describe(QueryExecutionContext* qec, + std::shared_ptr subtree, + parsedQuery::Describe describe); + + // Getter for testing. + const auto& getDescribe() const { return describe_; } + + // The following functions override those from the base class `Operation`. + std::vector getChildren() override; + string getCacheKeyImpl() const override; + string getDescriptor() const override; + size_t getResultWidth() const override; + size_t getCostEstimate() override; + + private: + uint64_t getSizeEstimateBeforeLimit() override; + + public: + float getMultiplicity(size_t col) override; + bool knownEmptyResult() override; + + private: + [[nodiscard]] vector resultSortedOn() const override; + ProtoResult computeResult(bool requestLaziness) override; + VariableToColumnMap computeVariableToColumnMap() const override; + + // Add all triples where the subject is one of the `blankNodes` (an `IdTable` + // with one column) to the `finalResult`. Recursively continue for all newly + // found blank nodes (objects of the newly found triples, which are not + // contained in `alreadySeen`). This is a recursive implementation of + // breadth-first-search (BFS) where `blankNodes` is the set of start nodes, + // and `alreadySeen` is the set of nodes which have already been explored, + // which is needed to handle cycles in the graph. + void recursivelyAddBlankNodes( + IdTable& finalResult, LocalVocab& localVocab, + ad_utility::HashSetWithMemoryLimit& alreadySeen, IdTable blankNodes); + + // Join the `input` (an `IdTable` with one column) with the full index on the + // subject column. The result has three columns: the subject, predicate, and + // object of each triple, where the subject is contained in `input`. This + // includes delta triples with local vocab IDs, which are added to the + // `localVocab`. + IdTable makeAndExecuteJoinWithFullIndex(IdTable input, + LocalVocab& localVocab) const; + + // Get the set of (unique) IDs that match one of the variables or IRIs in + // the DESCRIBE clause and the `result` of the WHERE clause. For example, if + // the query is `DESCRIBE ?y WHERE { ?y

}`, return `` and all + // IRIs that match `?y` in the WHERE clause, with all duplicates removed. + IdTable getIdsToDescribe(const Result& result, LocalVocab& localVocab) const; +}; diff --git a/src/engine/Operation.h b/src/engine/Operation.h index 9e649cb0a4..893e85ca22 100644 --- a/src/engine/Operation.h +++ b/src/engine/Operation.h @@ -1,8 +1,7 @@ -// Copyright 2015, University of Freiburg, -// Chair of Algorithms and Data Structures. -// Author: -// 2015-2017 Björn Buchhold (buchhold@informatik.uni-freiburg.de) -// 2018- Johannes Kalmbach (kalmbach@informatik.uni-freiburg.de) +// Copyright 2015 - 2024, University of Freiburg +// Chair of Algorithms and Data Structures +// Authors: Björn Buchhold [2015 - 2017] +// Johannes Kalmbach [2018 - 2024] #pragma once diff --git a/src/engine/QueryPlanner.cpp b/src/engine/QueryPlanner.cpp index 9e0cd56083..9261172f4e 100644 --- a/src/engine/QueryPlanner.cpp +++ b/src/engine/QueryPlanner.cpp @@ -19,6 +19,7 @@ #include "engine/CheckUsePatternTrick.h" #include "engine/CountAvailablePredicates.h" #include "engine/CountConnectedSubgraphs.h" +#include "engine/Describe.h" #include "engine/Distinct.h" #include "engine/Filter.h" #include "engine/GroupBy.h" @@ -2349,6 +2350,8 @@ void QueryPlanner::GraphPatternPlanner::graphPatternOperationVisitor(Arg& arg) { visitGroupOptionalOrMinus(std::move(candidates)); } else if constexpr (std::is_same_v) { visitPathSearch(arg); + } else if constexpr (std::is_same_v) { + visitDescribe(arg); } else if constexpr (std::is_same_v) { visitSpatialSearch(arg); } else { @@ -2587,3 +2590,14 @@ void QueryPlanner::GraphPatternPlanner::optimizeCommutatively() { candidatePlans_.push_back(std::move(lastRow)); planner_.checkCancellation(); } + +// _______________________________________________________________ +void QueryPlanner::GraphPatternPlanner::visitDescribe( + parsedQuery::Describe& describe) { + auto tree = std::make_shared( + planner_.createExecutionTree(describe.whereClause_.get(), true)); + auto describeOp = + makeSubtreePlan(planner_._qec, std::move(tree), describe); + candidatePlans_.push_back(std::vector{std::move(describeOp)}); + planner_.checkCancellation(); +} diff --git a/src/engine/QueryPlanner.h b/src/engine/QueryPlanner.h index 643343b72f..52ee540a0a 100644 --- a/src/engine/QueryPlanner.h +++ b/src/engine/QueryPlanner.h @@ -542,6 +542,7 @@ class QueryPlanner { void visitSpatialSearch(parsedQuery::SpatialQuery& config); void visitUnion(parsedQuery::Union& un); void visitSubquery(parsedQuery::Subquery& subquery); + void visitDescribe(parsedQuery::Describe& describe); // This function is called for groups, optional, or minus clauses. // The `candidates` are the result of planning the pattern inside the diff --git a/src/engine/idTable/IdTable.h b/src/engine/idTable/IdTable.h index 4ffe33138a..9e57073602 100644 --- a/src/engine/idTable/IdTable.h +++ b/src/engine/idTable/IdTable.h @@ -1,5 +1,6 @@ -// Copyright 2021, University of Freiburg, Chair of Algorithms and Data -// Structures. Author: Johannes Kalmbach +// Copyright 2021 - 2024, University of Freiburg +// Chair of Algorithms and Data Structures +// Author: Johannes Kalmbach #pragma once diff --git a/src/parser/CMakeLists.txt b/src/parser/CMakeLists.txt index cb6da1e6ed..be4b3db44c 100644 --- a/src/parser/CMakeLists.txt +++ b/src/parser/CMakeLists.txt @@ -29,6 +29,8 @@ add_library(parser GraphPattern.cpp data/Variable.cpp Iri.cpp Literal.cpp - LiteralOrIri.cpp) + LiteralOrIri.cpp + DatasetClauses.cpp +) qlever_target_link_libraries(parser sparqlParser parserData sparqlExpressions rdfEscaping re2::re2 util engine index) diff --git a/src/parser/DatasetClauses.cpp b/src/parser/DatasetClauses.cpp new file mode 100644 index 0000000000..4cf9b5526a --- /dev/null +++ b/src/parser/DatasetClauses.cpp @@ -0,0 +1,19 @@ +// Copyright 2024, University of Freiburg +// Chair of Algorithms and Data Structures +// Author: Johannes Kalmbach + +#include "parser/DatasetClauses.h" + +// _____________________________________________________________________________ +parsedQuery::DatasetClauses parsedQuery::DatasetClauses::fromClauses( + const std::vector& clauses) { + DatasetClauses result; + for (auto& [dataset, isNamed] : clauses) { + auto& graphs = isNamed ? result.namedGraphs_ : result.defaultGraphs_; + if (!graphs.has_value()) { + graphs.emplace(); + } + graphs.value().insert(dataset); + } + return result; +} diff --git a/src/parser/DatasetClauses.h b/src/parser/DatasetClauses.h new file mode 100644 index 0000000000..7cc3936e68 --- /dev/null +++ b/src/parser/DatasetClauses.h @@ -0,0 +1,24 @@ +// Copyright 2024, University of Freiburg +// Chair of Algorithms and Data Structures +// Author: Johannes Kalmbach + +#pragma once +#include + +#include "index/ScanSpecification.h" +#include "parser/sparqlParser/DatasetClause.h" + +namespace parsedQuery { +// A struct for the FROM clause (default graphs) and FROM NAMED clauses (named +// graphs). +struct DatasetClauses { + ScanSpecificationAsTripleComponent::Graphs defaultGraphs_{}; + ScanSpecificationAsTripleComponent::Graphs namedGraphs_{}; + + // Divide the dataset clause from `clauses` into default and named graphs, as + // needed for a `DatasetClauses` object. + static DatasetClauses fromClauses(const std::vector& clauses); + + bool operator==(const DatasetClauses& other) const = default; +}; +} // namespace parsedQuery diff --git a/src/parser/GraphPatternOperation.cpp b/src/parser/GraphPatternOperation.cpp index d4385591fb..9a36998fb8 100644 --- a/src/parser/GraphPatternOperation.cpp +++ b/src/parser/GraphPatternOperation.cpp @@ -53,14 +53,15 @@ auto m(auto&&... args) { // Special member functions for the `Subquery` class Subquery::Subquery() : _subquery{m()} {} Subquery::Subquery(const ParsedQuery& pq) : _subquery{m(pq)} {} -Subquery::Subquery(ParsedQuery&& pq) : _subquery{m(std::move(pq))} {} -Subquery::Subquery(Subquery&& pq) : _subquery{m(std::move(pq.get()))} {} +Subquery::Subquery(ParsedQuery&& pq) noexcept : _subquery{m(std::move(pq))} {} +Subquery::Subquery(Subquery&& pq) noexcept + : _subquery{m(std::move(pq.get()))} {} Subquery::Subquery(const Subquery& pq) : _subquery{m(pq.get())} {} Subquery& Subquery::operator=(const Subquery& pq) { _subquery = m(pq.get()); return *this; } -Subquery& Subquery::operator=(Subquery&& pq) { +Subquery& Subquery::operator=(Subquery&& pq) noexcept { _subquery = m(std::move(pq.get())); return *this; } diff --git a/src/parser/GraphPatternOperation.h b/src/parser/GraphPatternOperation.h index d02d60cd3d..8f7d4a8505 100644 --- a/src/parser/GraphPatternOperation.h +++ b/src/parser/GraphPatternOperation.h @@ -11,6 +11,7 @@ #include "engine/PathSearch.h" #include "engine/SpatialJoin.h" #include "engine/sparqlExpressions/SparqlExpressionPimpl.h" +#include "parser/DatasetClauses.h" #include "parser/GraphPattern.h" #include "parser/PathQuery.h" #include "parser/SpatialQuery.h" @@ -120,20 +121,34 @@ class Subquery { public: // TODO Make this an abstraction `TypeErasingPimpl`. - // Deliberately not explicit, because semantically those are copy/move - // constructors. + // NOTE: The first two constructors are deliberately not `explicit` because + // we want to used them like copy/move constructors (semantically, a + // `Subquery` is like a `ParsedQuery`, just with an own type). Subquery(const ParsedQuery&); - Subquery(ParsedQuery&&); + Subquery(ParsedQuery&&) noexcept; Subquery(); ~Subquery(); Subquery(const Subquery&); - Subquery(Subquery&&); + Subquery(Subquery&&) noexcept; Subquery& operator=(const Subquery&); - Subquery& operator=(Subquery&&); + Subquery& operator=(Subquery&&) noexcept; ParsedQuery& get(); const ParsedQuery& get() const; }; +// A SPARQL `DESCRIBE` query. +struct Describe { + using VarOrIri = std::variant; + // The resources (variables or IRIs) that are to be described, for example + // `?x` and `` in `DESCRIBE ?x `. + std::vector resources_; + // The FROM clauses of the DESCRIBE query + DatasetClauses datasetClauses_; + // The WHERE clause of the DESCRIBE query. It is used to compute the values + // for the variables in the DESCRIBE clause. + Subquery whereClause_; +}; + struct TransPath { // The name of the left and right end of the transitive operation TripleComponent _left; @@ -164,7 +179,7 @@ struct Bind { using GraphPatternOperationVariant = std::variant; + GroupGraphPattern, Describe>; struct GraphPatternOperation : public GraphPatternOperationVariant, public VisitMixin { diff --git a/src/parser/ParsedQuery.cpp b/src/parser/ParsedQuery.cpp index 71351b43d6..6bee76da2c 100644 --- a/src/parser/ParsedQuery.cpp +++ b/src/parser/ParsedQuery.cpp @@ -1,6 +1,7 @@ -// Copyright 2014, University of Freiburg, -// Chair of Algorithms and Data Structures. -// Author: Björn Buchhold (buchhold@informatik.uni-freiburg.de) +// Copyright 2014 - 2024, University of Freiburg +// Chair of Algorithms and Data Structures +// Authors: Björn Buchhold [2014 - 2017] +// Johannes Kalmbach #include "ParsedQuery.h" @@ -23,20 +24,6 @@ using std::string; using std::vector; -// _____________________________________________________________________________ -parsedQuery::DatasetClauses parsedQuery::DatasetClauses::fromClauses( - const std::vector& clauses) { - DatasetClauses result; - for (auto& [dataset, isNamed] : clauses) { - auto& graphs = isNamed ? result.namedGraphs_ : result.defaultGraphs_; - if (!graphs.has_value()) { - graphs.emplace(); - } - graphs.value().insert(dataset); - } - return result; -} - // _____________________________________________________________________________ string SparqlPrefix::asString() const { std::ostringstream os; diff --git a/src/parser/ParsedQuery.h b/src/parser/ParsedQuery.h index ef8e2b6637..5629f49833 100644 --- a/src/parser/ParsedQuery.h +++ b/src/parser/ParsedQuery.h @@ -1,6 +1,8 @@ -// Copyright 2014, University of Freiburg, -// Chair of Algorithms and Data Structures. -// Author: Björn Buchhold (buchhold@informatik.uni-freiburg.de) +// Copyright 2014 - 2024, University of Freiburg +// Chair of Algorithms and Data Structures +// Authors: Björn Buchhold [2014 - 2017] +// Johannes Kalmbach + #pragma once #include @@ -14,6 +16,7 @@ #include "index/ScanSpecification.h" #include "parser/Alias.h" #include "parser/ConstructClause.h" +#include "parser/DatasetClauses.h" #include "parser/GraphPattern.h" #include "parser/GraphPatternOperation.h" #include "parser/PropertyPath.h" @@ -37,21 +40,6 @@ using std::string; using std::vector; -// Forward declaration -struct DatasetClause; - -namespace parsedQuery { -// A struct for the FROM and FROM NAMED clauses; -struct DatasetClauses { - // FROM clauses. - ScanSpecificationAsTripleComponent::Graphs defaultGraphs_{}; - // FROM NAMED clauses. - ScanSpecificationAsTripleComponent::Graphs namedGraphs_{}; - - static DatasetClauses fromClauses(const std::vector& clauses); -}; -} // namespace parsedQuery - // Data container for prefixes class SparqlPrefix { public: diff --git a/src/parser/sparqlParser/SparqlQleverVisitor.cpp b/src/parser/sparqlParser/SparqlQleverVisitor.cpp index 5629f1452c..28054466db 100644 --- a/src/parser/sparqlParser/SparqlQleverVisitor.cpp +++ b/src/parser/sparqlParser/SparqlQleverVisitor.cpp @@ -269,9 +269,7 @@ ParsedQuery Visitor::visit(Parser::ConstructQueryContext* ctx) { if (ctx->constructTemplate()) { query._clause = visit(ctx->constructTemplate()) .value_or(parsedQuery::ConstructClause{}); - auto [pattern, visibleVariables] = visit(ctx->whereClause()); - query._rootGraphPattern = std::move(pattern); - query.registerVariablesVisibleInQueryBody(visibleVariables); + visitWhereClause(ctx->whereClause(), query); } else { query._clause = parsedQuery::ConstructClause{ visitOptional(ctx->triplesTemplate()).value_or(Triples{})}; @@ -282,8 +280,69 @@ ParsedQuery Visitor::visit(Parser::ConstructQueryContext* ctx) { } // ____________________________________________________________________________________ -ParsedQuery Visitor::visit(const Parser::DescribeQueryContext* ctx) { - reportNotSupported(ctx, "DESCRIBE queries are"); +ParsedQuery Visitor::visit(Parser::DescribeQueryContext* ctx) { + auto describeClause = parsedQuery::Describe{}; + auto describedResources = visitVector(ctx->varOrIri()); + + // Convert the describe resources (variables or IRIs) from the format that the + // parser delivers to the one that the `parsedQuery::Describe` struct expects. + std::vector describedVariables; + for (GraphTerm& resource : describedResources) { + if (std::holds_alternative(resource)) { + const auto& variable = std::get(resource); + describeClause.resources_.emplace_back(variable); + describedVariables.push_back(variable); + } else { + AD_CORRECTNESS_CHECK(std::holds_alternative(resource)); + auto iri = + TripleComponent::Iri::fromIriref(std::get(resource).toSparql()); + describeClause.resources_.emplace_back(std::move(iri)); + } + } + + // Parse the FROM and FROM NAMED clauses. + auto datasetClauses = parsedQuery::DatasetClauses::fromClauses( + visitVector(ctx->datasetClause())); + describeClause.datasetClauses_ = datasetClauses; + + // Parse the WHERE clause and construct a SELECT query from it. For `DESCRIBE + // *`, add each visible variable as a resource to describe. + visitWhereClause(ctx->whereClause(), parsedQuery_); + if (describedResources.empty()) { + const auto& visibleVariables = + parsedQuery_.selectClause().getVisibleVariables(); + std::ranges::copy(visibleVariables, + std::back_inserter(describeClause.resources_)); + describedVariables = visibleVariables; + } + auto& selectClause = parsedQuery_.selectClause(); + selectClause.setSelected(std::move(describedVariables)); + describeClause.whereClause_ = std::move(parsedQuery_); + + // Set up the final `ParsedQuery` object for the DESCRIBE query. The clause is + // a CONSTRUCT query of the form `CONSTRUCT { ?subject ?predicate ?object} { + // ... }`, with the `parsedQuery::Describe` object from above as the root + // graph pattern. The solution modifiers (in particular ORDER BY) are part of + // the CONSTRUCT query. + // + // NOTE: The dataset clauses are stored once in `parsedQuery_.datasetClauses_` + // (which pertains to the CONSTRUCT query that computes the result of the + // DESCRIBE), and once in `parsedQuery_.describeClause_.datasetClauses_` + // (which pertains to the SELECT query that computes the resources to be + // described). + parsedQuery_ = ParsedQuery{}; + parsedQuery_.addSolutionModifiers(visit(ctx->solutionModifier())); + parsedQuery_._rootGraphPattern._graphPatterns.emplace_back( + std::move(describeClause)); + parsedQuery_.datasetClauses_ = datasetClauses; + auto constructClause = ParsedQuery::ConstructClause{}; + using G = GraphTerm; + using V = Variable; + constructClause.triples_.push_back( + std::array{G(V("?subject")), G(V("?predicate")), G(V("?object"))}); + parsedQuery_._clause = std::move(constructClause); + + return parsedQuery_; } // ____________________________________________________________________________________ @@ -291,9 +350,7 @@ ParsedQuery Visitor::visit(Parser::AskQueryContext* ctx) { parsedQuery_._clause = ParsedQuery::AskClause{}; parsedQuery_.datasetClauses_ = parsedQuery::DatasetClauses::fromClauses( visitVector(ctx->datasetClause())); - auto [pattern, visibleVariables] = visit(ctx->whereClause()); - parsedQuery_._rootGraphPattern = std::move(pattern); - parsedQuery_.registerVariablesVisibleInQueryBody(visibleVariables); + visitWhereClause(ctx->whereClause(), parsedQuery_); // NOTE: It can make sense to have solution modifiers with an ASK query, for // example, a GROUP BY with a HAVING. auto getSolutionModifiers = [this, ctx]() { @@ -1115,9 +1172,7 @@ ParsedQuery Visitor::visit(Parser::SelectQueryContext* ctx) { parsedQuery_._clause = visit(ctx->selectClause()); parsedQuery_.datasetClauses_ = parsedQuery::DatasetClauses::fromClauses( visitVector(ctx->datasetClause())); - auto [pattern, visibleVariables] = visit(ctx->whereClause()); - parsedQuery_._rootGraphPattern = std::move(pattern); - parsedQuery_.registerVariablesVisibleInQueryBody(visibleVariables); + visitWhereClause(ctx->whereClause(), parsedQuery_); parsedQuery_.addSolutionModifiers(visit(ctx->solutionModifier())); return parsedQuery_; } @@ -1126,10 +1181,8 @@ ParsedQuery Visitor::visit(Parser::SelectQueryContext* ctx) { Visitor::SubQueryAndMaybeValues Visitor::visit(Parser::SubSelectContext* ctx) { ParsedQuery& query = parsedQuery_; query._clause = visit(ctx->selectClause()); - auto [pattern, visibleVariables] = visit(ctx->whereClause()); - query._rootGraphPattern = std::move(pattern); + visitWhereClause(ctx->whereClause(), query); query.setNumInternalVariables(numInternalVariables_); - query.registerVariablesVisibleInQueryBody(visibleVariables); query.addSolutionModifiers(visit(ctx->solutionModifier())); numInternalVariables_ = query.getNumInternalVariables(); auto values = visit(ctx->valuesClause()); @@ -2620,3 +2673,14 @@ TripleComponent SparqlQleverVisitor::visitGraphTerm( } }); } + +// _____________________________________________________________________________ +void SparqlQleverVisitor::visitWhereClause( + Parser::WhereClauseContext* whereClauseContext, ParsedQuery& query) { + if (!whereClauseContext) { + return; + } + auto [pattern, visibleVariables] = visit(whereClauseContext); + query._rootGraphPattern = std::move(pattern); + query.registerVariablesVisibleInQueryBody(visibleVariables); +} diff --git a/src/parser/sparqlParser/SparqlQleverVisitor.h b/src/parser/sparqlParser/SparqlQleverVisitor.h index 6c44a1caa2..fb1cb9c05c 100644 --- a/src/parser/sparqlParser/SparqlQleverVisitor.h +++ b/src/parser/sparqlParser/SparqlQleverVisitor.h @@ -140,12 +140,7 @@ class SparqlQleverVisitor { ParsedQuery visit(Parser::ConstructQueryContext* ctx); - // The parser rules for which the visit overload is annotated [[noreturn]] - // will always throw an exception because the corresponding feature is not - // (yet) supported by QLever. If they have return types other than void this - // is to make the usage of abstractions like `visitAlternative` easier. - [[noreturn]] static ParsedQuery visit( - const Parser::DescribeQueryContext* ctx); + ParsedQuery visit(Parser::DescribeQueryContext* ctx); ParsedQuery visit(Parser::AskQueryContext* ctx); @@ -159,6 +154,12 @@ class SparqlQleverVisitor { PatternAndVisibleVariables visit(Parser::WhereClauseContext* ctx); + // Parse the WHERE clause represented by the `whereClauseContext` and store + // its result (the GroupGraphPattern representing the where clause + the set + // of visible variables) in the `query`. + void visitWhereClause(Parser::WhereClauseContext* whereClauseContext, + ParsedQuery& query); + SolutionModifiers visit(Parser::SolutionModifierContext* ctx); vector visit(Parser::GroupClauseContext* ctx); @@ -342,6 +343,11 @@ class SparqlQleverVisitor { PropertyPath visit(Parser::PathEltOrInverseContext* ctx); + // NOTE: The `visit` overloads marked `[[noreturn]]` always throw an exception + // because the corresponding feature is not (yet) supported by QLever. Most + // of them have a return type of `void`. Some of the don't, in order to make + // the usage of abstractions like `visitAlternative` easier. + [[noreturn]] static void visit(Parser::PathModContext* ctx); PropertyPath visit(Parser::PathPrimaryContext* ctx); diff --git a/test/QueryPlannerTest.cpp b/test/QueryPlannerTest.cpp index c9a104bdcb..32261e871e 100644 --- a/test/QueryPlannerTest.cpp +++ b/test/QueryPlannerTest.cpp @@ -1,7 +1,7 @@ // Copyright 2015 - 2024, University of Freiburg // Chair of Algorithms and Data Structures // Authors: Björn Buchhold [2015 - 2017] -// Johannes Kalmbach [2018 - 2024] +// Johannes Kalmbach [2018 - 2024] #include @@ -2884,3 +2884,21 @@ TEST(QueryPlanner, WarningsOnUnboundVariables) { h::QetWithWarnings({"?a was used in the expression of a BIND"}, testing::_)); } + +// ___________________________________________________________________________ +TEST(QueryPlanner, Describe) { + // Note: We deliberately don't test the contents of the actual DESCRIBE + // clause, because they have been extensively tested already in + // `SparqlAntlrParserTest.cpp` where we have access to proper matchers for + // them. + h::expect("DESCRIBE ", h::Describe(::testing::_, h::NeutralElement())); + h::expect("DESCRIBE ?x", h::Describe(::testing::_, h::NeutralElement())); + h::expect( + "Describe ?y { ?y

}", + h::Describe(::testing::_, h::IndexScanFromStrings("?y", "

", ""))); + h::expect( + "Describe ?y FROM { ?y

}", + h::Describe(::testing::_, h::IndexScanFromStrings( + "?y", "

", "", {}, + ad_utility::HashSet{""}))); +} diff --git a/test/QueryPlannerTestHelpers.h b/test/QueryPlannerTestHelpers.h index ea22a00431..dfc037645b 100644 --- a/test/QueryPlannerTestHelpers.h +++ b/test/QueryPlannerTestHelpers.h @@ -14,6 +14,7 @@ #include "engine/Bind.h" #include "engine/CartesianProductJoin.h" #include "engine/CountAvailablePredicates.h" +#include "engine/Describe.h" #include "engine/Filter.h" #include "engine/GroupBy.h" #include "engine/IndexScan.h" @@ -395,10 +396,19 @@ constexpr auto OrderBy = [](const ::OrderBy::SortedVariables& sortedVariables, // Match a `UNION` operation. constexpr auto Union = MatchTypeAndOrderedChildren<::Union>; +// Match a `DESCRIBE` operation +inline QetMatcher Describe( + const Matcher& describeMatcher, + const QetMatcher& childMatcher) { + return RootOperation<::Describe>( + AllOf(children(childMatcher), + AD_PROPERTY(::Describe, getDescribe, describeMatcher))); +} + // inline QetMatcher QetWithWarnings( const std::vector& warningSubstrings, - QetMatcher actualMatcher) { + const QetMatcher& actualMatcher) { auto warningMatchers = ad_utility::transform( warningSubstrings, [](const std::string& s) { return ::testing::HasSubstr(s); }); diff --git a/test/SparqlAntlrParserTest.cpp b/test/SparqlAntlrParserTest.cpp index d737c90e00..0803f96f03 100644 --- a/test/SparqlAntlrParserTest.cpp +++ b/test/SparqlAntlrParserTest.cpp @@ -1,9 +1,8 @@ // Copyright 2021 - 2024, University of Freiburg // Chair of Algorithms and Data Structures -// Authors: -// 2021 - Johannes Kalmbach -// 2022 - Julian Mundhahs -// 2022 - Hannah Bast +// Authors: Johannes Kalmbach +// Julian Mundhahs +// Hannah Bast #include @@ -1367,8 +1366,52 @@ TEST(SparqlParser, Query) { {Var{"?s"}, Var{"?p"}, Var{"?o"}}, "{ ?s ?p ?o }", "PREFIX doof: ")))); - // DESCRIBE queries are not yet supported. - expectQueryFails("DESCRIBE *"); + // Tests around DESCRIBE. + { + // The tested DESCRIBE queries all describe ``, `?y`, and ``. + using Resources = std::vector; + auto Iri = [](const auto& x) { + return TripleComponent::Iri::fromIriref(x); + }; + Resources xyz{Iri(""), Var{"?y"}, Iri("")}; + + // A matcher for `?y ?v`. + auto graphPatternMatcher = + m::GraphPattern(m::Triples({{Var{"?y"}, "", Var{"?v"}}})); + + // A matcher for the subquery `SELECT ?y { ?y ?v }`, which we will + // use to compute the values for `?y` that are to be described. + auto selectQueryMatcher1 = + m::SelectQuery(m::Select({Var{"?y"}}), graphPatternMatcher); + + // DESCRIBE with neither FROM nor FROM NAMED clauses. + expectQuery("DESCRIBE ?y { ?y ?v }", + m::DescribeQuery(m::Describe(xyz, {}, selectQueryMatcher1))); + + // `DESCRIBE *` query that is equivalent to `DESCRIBE ?y { ... }`. + auto selectQueryMatcher2 = + m::SelectQuery(m::Select({Var{"?y"}, Var{"?v"}}), graphPatternMatcher); + Resources yv{Var{"?y"}, Var{"?v"}}; + expectQuery("DESCRIBE * { ?y ?v }", + m::DescribeQuery(m::Describe(yv, {}, selectQueryMatcher2))); + + // DESCRIBE with FROM and FROM NAMED clauses. + // + // NOTE: The clauses are relevant *both* for the retrieval of the resources + // to describe (the `Id`s matching `?y`), as well as for computing the + // triples for each of these resources. + using Graphs = ScanSpecificationAsTripleComponent::Graphs; + Graphs expectedDefaultGraphs; + Graphs expectedNamedGraphs; + expectedDefaultGraphs.emplace({Iri("")}); + expectedNamedGraphs.emplace({Iri("")}); + parsedQuery::DatasetClauses expectedClauses{expectedDefaultGraphs, + expectedNamedGraphs}; + expectQuery( + "DESCRIBE ?y FROM FROM NAMED ", + m::DescribeQuery(m::Describe(xyz, expectedClauses, ::testing::_), + expectedDefaultGraphs, expectedNamedGraphs)); + } // Test the various places where warnings are added in a query. expectQuery("SELECT ?x {} GROUP BY ?x ORDER BY ?y", diff --git a/test/SparqlAntlrParserTestHelpers.h b/test/SparqlAntlrParserTestHelpers.h index 489e2b81e9..b898659acc 100644 --- a/test/SparqlAntlrParserTestHelpers.h +++ b/test/SparqlAntlrParserTestHelpers.h @@ -14,6 +14,7 @@ #include "engine/sparqlExpressions/SparqlExpressionPimpl.h" #include "parser/Alias.h" +#include "parser/DatasetClauses.h" #include "parser/ParsedQuery.h" #include "parser/SparqlParserHelpers.h" #include "parser/TripleComponent.h" @@ -676,6 +677,21 @@ inline auto Triples = [](const vector& triples) testing::UnorderedElementsAreArray(triples))); }; +// Match a `Describe` clause. +inline auto Describe = [](const std::vector& resources, + const p::DatasetClauses& datasetClauses, + const Matcher& subquery) + -> Matcher { + using namespace ::testing; + auto getSubquery = [](const p::Subquery& subquery) -> const ParsedQuery& { + return subquery.get(); + }; + return detail::GraphPatternOperation(AllOf( + AD_FIELD(p::Describe, resources_, Eq(resources)), + AD_FIELD(p::Describe, datasetClauses_, Eq(datasetClauses)), + AD_FIELD(p::Describe, whereClause_, ResultOf(getSubquery, subquery)))); +}; + namespace detail { inline auto Optional = [](auto&& subMatcher) -> Matcher { @@ -892,6 +908,19 @@ inline auto ConstructQuery( RootGraphPattern(m)); } +// A matcher for a `DescribeQuery` +inline auto DescribeQuery( + const Matcher& describeMatcher, + const ScanSpecificationAsTripleComponent::Graphs& defaultGraphs = + std::nullopt, + const ScanSpecificationAsTripleComponent::Graphs& namedGraphs = + std::nullopt) { + using Var = ::Variable; + return ConstructQuery({{Var{"?subject"}, Var{"?predicate"}, Var{"?object"}}}, + GraphPattern(describeMatcher), defaultGraphs, + namedGraphs); +} + // _____________________________________________________________________________ inline auto VisibleVariables = [](const std::vector<::Variable>& elems) -> Matcher { diff --git a/test/engine/CMakeLists.txt b/test/engine/CMakeLists.txt index 47704485d6..fef9ffed39 100644 --- a/test/engine/CMakeLists.txt +++ b/test/engine/CMakeLists.txt @@ -11,3 +11,4 @@ addLinkAndDiscoverTest(CountConnectedSubgraphsTest) addLinkAndDiscoverTest(BindTest engine) addLinkAndRunAsSingleTest(SpatialJoinAlgorithmsTest engine) addLinkAndDiscoverTestSerial(QueryExecutionTreeTest engine) +addLinkAndDiscoverTestSerial(DescribeTest engine) diff --git a/test/engine/DescribeTest.cpp b/test/engine/DescribeTest.cpp new file mode 100644 index 0000000000..76618a0f40 --- /dev/null +++ b/test/engine/DescribeTest.cpp @@ -0,0 +1,180 @@ +// Copyright 2024, University of Freiburg +// Chair of Algorithms and Data Structures +// Author: Johannes Kalmbach + +#include + +#include "../util/GTestHelpers.h" +#include "../util/IndexTestHelpers.h" +#include "engine/Describe.h" +#include "engine/IndexScan.h" +#include "engine/NeutralElementOperation.h" +#include "engine/QueryExecutionTree.h" + +using namespace ad_utility::testing; + +namespace { +// Return a matcher that matches a `span` or something similar that +// contains `expectedNumUnique` elements. +auto numUnique = [](size_t expectedNumUnique) { + using namespace ::testing; + auto getNumUnique = [](const auto& col) { + return ad_utility::HashSet(col.begin(), col.end()).size(); + }; + return ResultOf(getNumUnique, expectedNumUnique); +}; +} // namespace + +// Test DESCRIBE query with a fixed IRI and sveral blank nodes that need to be +// expanded. +TEST(Describe, recursiveBlankNodes) { + auto qec = getQec( + "

." + "

_:g1 ." + "_:g1 ." + "_:g1 _:g1 ." + "_:g1 _:g2 ." + "_:g2

." + "

." + "_:g4

_:g5 ."); + parsedQuery::Describe parsedDescribe; + parsedDescribe.resources_.push_back(TripleComponent::Iri::fromIriref("")); + Describe describe{qec, + ad_utility::makeExecutionTree(qec), + parsedDescribe}; + auto res = describe.computeResultOnlyForTesting(); + const auto& table = res.idTable(); + // The expected result is as follows: + // + //

+ //

_:g1 + // _:g1 + // _:g1 _:g1 + // _:g1 _:g2 + // _:g2

+ // + // However, we cannot control the names given to the blank nodes, but we can + // at least check the statistics. + EXPECT_EQ(table.size(), 6); + EXPECT_THAT(table.getColumn(0), numUnique(3)); + EXPECT_THAT(table.getColumn(1), numUnique(2)); + EXPECT_THAT(table.getColumn(2), numUnique(5)); +} + +// Test DESCRIBE query with a fixed IRI and a variable in the DESCRIBE clause, +// and various blank nodes that need to be expanded. +TEST(Describe, describeWithVariable) { + auto qec = getQec( + "

." + "

_:g1 ." + "_:g1 ." + "

." + " _:g1 ." + " _:g2 ." + "_:g2 ." + " ." + " ."); + + // On the above knowledge graph, evaluate `DESCRIBE ?x { ?x

}`. + parsedQuery::Describe parsedDescribe; + parsedDescribe.resources_.push_back(TripleComponent::Iri::fromIriref("")); + parsedDescribe.resources_.push_back(Variable{"?x"}); + SparqlTripleSimple triple{Variable{"?x"}, + TripleComponent::Iri::fromIriref("

"), + TripleComponent::Iri::fromIriref("")}; + Describe describe{qec, + ad_utility::makeExecutionTree( + qec, Permutation::Enum::POS, triple), + parsedDescribe}; + auto res = describe.computeResultOnlyForTesting(); + const auto& table = res.idTable(); + // The expected result is as follows (the resources are ``, which is + // explicitly requested, and `` and ``, which match `?x` in the WHERE + // clause): + // + //

+ //

_:g1 + // _:g1 + //

+ // _:g1 [note that _:g1 has already been expanded above] + // _:g2 + // _:g2 + // + // + // However, we cannot control the names given to the blank nodes, but we can + // at least check the statistics. + EXPECT_EQ(table.size(), 8); + EXPECT_THAT(table.getColumn(0), numUnique(5)); + EXPECT_THAT(table.getColumn(1), numUnique(3)); + EXPECT_THAT(table.getColumn(2), numUnique(5)); +} + +// Test DESCRIBE query with a variable but not WHERE clause (which should +// return an empty result). +TEST(Describe, describeWithVariableButNoWhereClause) { + auto qec = getQec("

"); + parsedQuery::Describe parsedDescribe; + parsedDescribe.resources_.push_back(Variable{"?x"}); + auto noWhere = ad_utility::makeExecutionTree(qec); + Describe describe{qec, noWhere, parsedDescribe}; + auto result = describe.computeResultOnlyForTesting(); + EXPECT_EQ(result.idTable().size(), 0); + EXPECT_EQ(result.idTable().numColumns(), 3); +} + +// TODO Add tests with inputs from a different graph, but those are +// Currently hard to do with the given `getQec` function. + +// Test the various member functions of the `Describe` operation. +TEST(Describe, simpleMembers) { + auto qec = getQec( + "

." + "

_:g1 ." + "_:g1 ." + "_:g1 _:g1 ." + "_:g1 _:g2 ." + "_:g2

." + "

." + "_:g4

_:g5 ."); + parsedQuery::Describe parsedDescribe; + parsedDescribe.resources_.push_back(TripleComponent::Iri::fromIriref("")); + Describe describe{qec, + ad_utility::makeExecutionTree(qec), + parsedDescribe}; + + EXPECT_EQ(describe.getDescriptor(), "DESCRIBE"); + EXPECT_EQ(describe.getResultWidth(), 3); + EXPECT_EQ(describe.getCostEstimate(), 0u); + EXPECT_EQ(describe.getSizeEstimate(), 2u); + EXPECT_FLOAT_EQ(describe.getMultiplicity(42), 1.0f); + EXPECT_FALSE(describe.knownEmptyResult()); + + // Test the cache key. + using namespace ::testing; + EXPECT_THAT( + describe.getCacheKey(), + AllOf(HasSubstr("DESCRIBE"), HasSubstr(""), Not(HasSubstr("

")), + HasSubstr("Neutral Element"), Not(HasSubstr("Filtered")))); + + // Test the cache key of the same query, but with a FROM clause. + auto parsedDescribe2 = parsedDescribe; + parsedDescribe2.datasetClauses_.defaultGraphs_.emplace( + {TripleComponent::Iri::fromIriref("")}); + Describe describe2{ + qec, ad_utility::makeExecutionTree(qec), + parsedDescribe2}; + EXPECT_THAT(describe2.getCacheKey(), + AllOf(HasSubstr("DESCRIBE"), HasSubstr(""), + Not(HasSubstr("

")), HasSubstr("Neutral Element"), + HasSubstr("Filtered by Graphs:"))); + + auto col = makeAlwaysDefinedColumn; + VariableToColumnMap expected{{Variable{"?subject"}, col(0)}, + {Variable{"?predicate"}, col(1)}, + {Variable{"?object"}, col(2)}}; + + auto children = describe.getChildren(); + ASSERT_EQ(children.size(), 1); + EXPECT_THAT(children.at(0)->getRootOperation()->getDescriptor(), + Eq("NeutralElement")); +} From a97905ebaa4eff2005171706ee520449ce767f18 Mon Sep 17 00:00:00 2001 From: Felix Meisen <85636111+Flixtastic@users.noreply.github.com> Date: Mon, 16 Dec 2024 11:47:00 +0100 Subject: [PATCH 05/14] `ql:contains-word` now can show the score of the word match in the respective text (#1397) The fulltext index of QLever has forever been able to associate the occurence of a word in a text with a score. This PR adds the functionality to actually retrieve this score and to use it in the remainder of the query. Currently the score is bound to a variable the name of which is automatically determined from the involved literals and variables. The easiest way to get the names of these variables is to use `SELECT *` or to look at the runtime information tree. --- e2e/scientists_queries.yaml | 33 +++- src/engine/QueryPlanner.cpp | 4 +- src/engine/TextIndexScanForEntity.cpp | 4 +- src/engine/TextIndexScanForWord.cpp | 14 +- src/index/FTSAlgorithms.cpp | 7 +- src/index/Index.cpp | 5 + src/index/Index.h | 1 + src/index/IndexImpl.Text.cpp | 25 ++- src/index/IndexImpl.cpp | 1 + src/index/IndexImpl.h | 15 +- src/parser/ContextFileParser.h | 1 + src/parser/data/Variable.cpp | 41 +++-- src/parser/data/Variable.h | 14 +- .../sparqlParser/SparqlQleverVisitor.cpp | 5 +- test/QueryPlannerTest.cpp | 60 ++++--- test/QueryPlannerTestHelpers.h | 2 +- test/engine/TextIndexScanForWordTest.cpp | 165 ++++++++++++++++-- test/engine/TextIndexScanTestHelpers.h | 50 +++++- test/util/IndexTestHelpers.cpp | 42 ++++- test/util/IndexTestHelpers.h | 9 +- 20 files changed, 400 insertions(+), 98 deletions(-) diff --git a/e2e/scientists_queries.yaml b/e2e/scientists_queries.yaml index 1fc78430be..421329c58b 100644 --- a/e2e/scientists_queries.yaml +++ b/e2e/scientists_queries.yaml @@ -55,31 +55,43 @@ queries: ?t ql:contains-word "RElaT* phySIKalische rela*" } checks: - - num_cols: 5 - - selected: [ "?x", "?ql_score_t_var_x", "?t", "?ql_matchingword_t_relat", "?ql_matchingword_t_rela" ] + - num_cols: 8 + - selected: [ "?x", "?ql_score_t_var_x", "?t", "?ql_score_prefix_t_RElaT", "?ql_matchingword_t_relat", "?ql_score_word_t_phySIKalische", "?ql_score_prefix_t_rela", "?ql_matchingword_t_rela" ] - contains_row: - "" - null - null + - null - "relationship" + - null + - null - "relationship" - contains_row: - "" - null - null + - null - "relationship" + - null + - null - "relativity" - contains_row: - "" - null - null + - null - "relativity" + - null + - null - "relationship" - contains_row: - "" - null - null + - null - "relativity" + - null + - null - "relativity" - query: algo-star-female-scientists @@ -151,7 +163,7 @@ queries: } TEXTLIMIT 2 checks: - - num_cols: 7 + - num_cols: 9 - num_rows: 18 - query: algor-star-female-born-before-1940 @@ -192,7 +204,7 @@ queries: } ORDER BY DESC(?ql_score_text_fixedEntity__60_Ada_95_Lovelace_62_) checks: - - num_cols: 5 + - num_cols: 6 - num_rows: 7 - contains_row: - "" @@ -202,6 +214,7 @@ queries: Charles Babbage, also known as' the father of computers', and in particular, Babbage's work on the Analytical Engine." - null + - null - "relationship" - order_numeric: {"dir": "DESC", "var" : "?ql_score_text_fixedEntity__60_Ada_95_Lovelace_62_"} @@ -219,7 +232,7 @@ queries: ORDER BY DESC(?ql_score_text_fixedEntity__60_Ada_95_Lovelace_62_) TEXTLIMIT 2 checks: - - num_cols: 5 + - num_cols: 6 - num_rows: 3 - contains_row: - "" @@ -229,6 +242,7 @@ queries: Charles Babbage, also known as' the father of computers', and in particular, Babbage's work on the Analytical Engine." - null + - null - "relationship" - order_numeric: {"dir": "DESC", "var" : "?ql_score_text_fixedEntity__60_Ada_95_Lovelace_62_"} @@ -246,7 +260,7 @@ queries: } TEXTLIMIT 1 checks: - - num_cols: 6 + - num_cols: 7 - num_rows: 2 - contains_row: - "" @@ -255,6 +269,7 @@ queries: with Somerville to visit Babbage as often as she could." - null - null + - null - "relationship" @@ -1391,10 +1406,10 @@ queries: ?t ql:contains-word "algo* herm* primary" } checks: - - num_cols: 5 + - num_cols: 8 - num_rows: 1 - - selected: [ "?x", "?ql_score_t_var_x", "?t", "?ql_matchingword_t_algo", "?ql_matchingword_t_herm" ] - - contains_row: [ "",null,"Hermann's algorithm for primary decomposition is still in use now.","algorithm","hermann" ] + - selected: [ "?x", "?ql_score_t_var_x", "?t", "?ql_score_prefix_t_algo", "?ql_matchingword_t_algo", "?ql_score_prefix_t_herm", "?ql_matchingword_t_herm", "?ql_score_word_t_primary" ] + - contains_row: [ "",null,"Hermann's algorithm for primary decomposition is still in use now.",null,"algorithm",null,"hermann",null ] - query : select_asterisk_regex-lastname-stein diff --git a/src/engine/QueryPlanner.cpp b/src/engine/QueryPlanner.cpp index 9261172f4e..9dd6b5599c 100644 --- a/src/engine/QueryPlanner.cpp +++ b/src/engine/QueryPlanner.cpp @@ -1002,14 +1002,14 @@ QueryPlanner::SubtreePlan QueryPlanner::getTextLeafPlan( : *(node._variables.begin()); plan = makeSubtreePlan(_qec, cvar, evar, word); textLimits[cvar].entityVars_.push_back(evar); - textLimits[cvar].scoreVars_.push_back(cvar.getScoreVariable(evar)); + textLimits[cvar].scoreVars_.push_back(cvar.getEntityScoreVariable(evar)); } else { // Fixed entity case AD_CORRECTNESS_CHECK(node._variables.size() == 1); plan = makeSubtreePlan( _qec, cvar, node.triple_.o_.toString(), word); textLimits[cvar].scoreVars_.push_back( - cvar.getScoreVariable(node.triple_.o_.toString())); + cvar.getEntityScoreVariable(node.triple_.o_.toString())); } } else { plan = makeSubtreePlan(_qec, cvar, word); diff --git a/src/engine/TextIndexScanForEntity.cpp b/src/engine/TextIndexScanForEntity.cpp index 78c29e8734..6dbce07ef5 100644 --- a/src/engine/TextIndexScanForEntity.cpp +++ b/src/engine/TextIndexScanForEntity.cpp @@ -48,10 +48,10 @@ VariableToColumnMap TextIndexScanForEntity::computeVariableToColumnMap() const { }; addDefinedVar(textRecordVar_); if (hasFixedEntity()) { - addDefinedVar(textRecordVar_.getScoreVariable(fixedEntity())); + addDefinedVar(textRecordVar_.getEntityScoreVariable(fixedEntity())); } else { addDefinedVar(entityVariable()); - addDefinedVar(textRecordVar_.getScoreVariable(entityVariable())); + addDefinedVar(textRecordVar_.getEntityScoreVariable(entityVariable())); } return vcmap; } diff --git a/src/engine/TextIndexScanForWord.cpp b/src/engine/TextIndexScanForWord.cpp index 612780e28b..7c3f931f8f 100644 --- a/src/engine/TextIndexScanForWord.cpp +++ b/src/engine/TextIndexScanForWord.cpp @@ -18,13 +18,12 @@ ProtoResult TextIndexScanForWord::computeResult( IdTable idTable = getExecutionContext()->getIndex().getWordPostingsForTerm( word_, getExecutionContext()->getAllocator()); + // This filters out the word column. When the searchword is a prefix this + // column shows the word the prefix got extended to if (!isPrefix_) { - IdTable smallIdTable{getExecutionContext()->getAllocator()}; - smallIdTable.setNumColumns(1); - smallIdTable.resize(idTable.numRows()); - ql::ranges::copy(idTable.getColumn(0), smallIdTable.getColumn(0).begin()); - - return {std::move(smallIdTable), resultSortedOn(), LocalVocab{}}; + using CI = ColumnIndex; + idTable.setColumnSubset(std::array{CI{0}, CI{2}}); + return {std::move(idTable), resultSortedOn(), LocalVocab{}}; } // Add details to the runtimeInfo. This is has no effect on the result. @@ -46,12 +45,13 @@ VariableToColumnMap TextIndexScanForWord::computeVariableToColumnMap() const { addDefinedVar(textRecordVar_.getMatchingWordVariable( std::string_view(word_).substr(0, word_.size() - 1))); } + addDefinedVar(textRecordVar_.getWordScoreVariable(word_, isPrefix_)); return vcmap; } // _____________________________________________________________________________ size_t TextIndexScanForWord::getResultWidth() const { - return 1 + (isPrefix_ ? 1 : 0); + return 2 + (isPrefix_ ? 1 : 0); } // _____________________________________________________________________________ diff --git a/src/index/FTSAlgorithms.cpp b/src/index/FTSAlgorithms.cpp index 0589c5ffee..087f97a1fe 100644 --- a/src/index/FTSAlgorithms.cpp +++ b/src/index/FTSAlgorithms.cpp @@ -10,19 +10,21 @@ // _____________________________________________________________________________ IdTable FTSAlgorithms::filterByRange(const IdRange& idRange, const IdTable& idTablePreFilter) { - AD_CONTRACT_CHECK(idTablePreFilter.numColumns() == 2); + AD_CONTRACT_CHECK(idTablePreFilter.numColumns() == 3); LOG(DEBUG) << "Filtering " << idTablePreFilter.getColumn(0).size() << " elements by ID range...\n"; IdTable idTableResult{idTablePreFilter.getAllocator()}; - idTableResult.setNumColumns(2); + idTableResult.setNumColumns(3); idTableResult.resize(idTablePreFilter.getColumn(0).size()); decltype(auto) resultCidColumn = idTableResult.getColumn(0); decltype(auto) resultWidColumn = idTableResult.getColumn(1); + decltype(auto) resultSidColumn = idTableResult.getColumn(2); size_t nofResultElements = 0; decltype(auto) preFilterCidColumn = idTablePreFilter.getColumn(0); decltype(auto) preFilterWidColumn = idTablePreFilter.getColumn(1); + decltype(auto) preFilterSidColumn = idTablePreFilter.getColumn(2); // TODO Use views::zip. for (size_t i = 0; i < preFilterWidColumn.size(); ++i) { // TODO proper Ids for the text stuff. @@ -36,6 +38,7 @@ IdTable FTSAlgorithms::filterByRange(const IdRange& idRange, preFilterWidColumn[i].getWordVocabIndex() <= idRange.last()) { resultCidColumn[nofResultElements] = preFilterCidColumn[i]; resultWidColumn[nofResultElements] = preFilterWidColumn[i]; + resultSidColumn[nofResultElements] = preFilterSidColumn[i]; nofResultElements++; } } diff --git a/src/index/Index.cpp b/src/index/Index.cpp index 47fcad9c82..c70706b341 100644 --- a/src/index/Index.cpp +++ b/src/index/Index.cpp @@ -232,6 +232,11 @@ size_t Index::getNofEntityPostings() const { return pimpl_->getNofEntityPostings(); } +// ____________________________________________________________________________ +size_t Index::getNofNonLiteralsInTextIndex() const { + return pimpl_->getNofNonLiteralsInTextIndex(); +} + // ____________________________________________________________________________ Index::NumNormalAndInternal Index::numDistinctSubjects() const { return pimpl_->numDistinctSubjects(); diff --git a/src/index/Index.h b/src/index/Index.h index ec408f15df..e815b2a5bf 100644 --- a/src/index/Index.h +++ b/src/index/Index.h @@ -214,6 +214,7 @@ class Index { size_t getNofTextRecords() const; size_t getNofWordPostings() const; size_t getNofEntityPostings() const; + size_t getNofNonLiteralsInTextIndex() const; NumNormalAndInternal numDistinctSubjects() const; NumNormalAndInternal numDistinctObjects() const; diff --git a/src/index/IndexImpl.Text.cpp b/src/index/IndexImpl.Text.cpp index bd46c81e53..76c0015974 100644 --- a/src/index/IndexImpl.Text.cpp +++ b/src/index/IndexImpl.Text.cpp @@ -65,7 +65,7 @@ cppcoro::generator IndexImpl::wordsInTextRecords( if (!isLiteral(text)) { continue; } - ContextFileParser::Line entityLine{text, true, contextId, 1}; + ContextFileParser::Line entityLine{text, true, contextId, 1, true}; co_yield entityLine; std::string_view textView = text; textView = textView.substr(0, textView.rfind('"')); @@ -235,10 +235,12 @@ void IndexImpl::processWordsForInvertedLists(const string& contextFile, ad_utility::HashMap wordsInContext; ad_utility::HashMap entitiesInContext; auto currentContext = TextRecordIndex::make(0); + // The nofContexts can be misleading since it also counts empty contexts size_t nofContexts = 0; size_t nofWordPostings = 0; size_t nofEntityPostings = 0; size_t entityNotFoundErrorMsgCount = 0; + size_t nofLiterals = 0; for (auto line : wordsInTextRecords(contextFile, addWordsFromLiterals)) { if (line._contextId != currentContext) { @@ -258,6 +260,9 @@ void IndexImpl::processWordsForInvertedLists(const string& contextFile, // Note that `entitiesInContext` is a HashMap, so the `Id`s don't have // to be contiguous. entitiesInContext[Id::makeFromVocabIndex(eid)] += line._score; + if (line._isLiteralEntity) { + ++nofLiterals; + } } else { if (entityNotFoundErrorMsgCount < 20) { LOG(WARN) << "Entity from text not in KB: " << line._word << '\n'; @@ -294,6 +299,10 @@ void IndexImpl::processWordsForInvertedLists(const string& contextFile, textMeta_.setNofTextRecords(nofContexts); textMeta_.setNofWordPostings(nofWordPostings); textMeta_.setNofEntityPostings(nofEntityPostings); + nofNonLiteralsInTextIndex_ = nofContexts - nofLiterals; + configurationJson_["num-non-literals-text-index"] = + nofNonLiteralsInTextIndex_; + writeConfiguration(); writer.finish(); LOG(TRACE) << "END IndexImpl::passContextFileIntoVector" << std::endl; @@ -415,7 +424,7 @@ ContextListMetaData IndexImpl::writePostings(ad_utility::File& out, size_t n = 0; - WordToCodeMap wordCodeMap; + WordCodeMap wordCodeMap; WordCodebook wordCodebook; ScoreCodeMap scoreCodeMap; ScoreCodebook scoreCodebook; @@ -646,10 +655,11 @@ size_t IndexImpl::writeList(Numeric* data, size_t nofElements, // _____________________________________________________________________________ void IndexImpl::createCodebooks(const vector& postings, - IndexImpl::WordToCodeMap& wordCodemap, + IndexImpl::WordCodeMap& wordCodemap, IndexImpl::WordCodebook& wordCodebook, IndexImpl::ScoreCodeMap& scoreCodemap, IndexImpl::ScoreCodebook& scoreCodebook) const { + // There should be a more efficient way to do this (Felix Meisen) ad_utility::HashMap wfMap; ad_utility::HashMap sfMap; for (const auto& p : postings) { @@ -718,7 +728,7 @@ std::string_view IndexImpl::wordIdToString(WordIndex wordIndex) const { IdTable IndexImpl::readWordCl( const TextBlockMetaData& tbmd, const ad_utility::AllocatorWithLimit& allocator) const { - IdTable idTable{2, allocator}; + IdTable idTable{3, allocator}; vector cids = readGapComprList( tbmd._cl._nofElements, tbmd._cl._startContextlist, static_cast(tbmd._cl._startWordlist - tbmd._cl._startContextlist), @@ -734,6 +744,11 @@ IdTable IndexImpl::readWordCl( idTable.getColumn(1).begin(), [](WordIndex id) { return Id::makeFromWordVocabIndex(WordVocabIndex::make(id)); }); + std::ranges::transform( + readFreqComprList(tbmd._cl._nofElements, tbmd._cl._startScorelist, + static_cast(tbmd._cl._lastByte + 1 - + tbmd._cl._startScorelist)), + idTable.getColumn(2).begin(), &Id::makeFromInt); return idTable; } @@ -772,7 +787,7 @@ IdTable IndexImpl::getWordPostingsForTerm( const ad_utility::AllocatorWithLimit& allocator) const { LOG(DEBUG) << "Getting word postings for term: " << term << '\n'; IdTable idTable{allocator}; - idTable.setNumColumns(term.ends_with('*') ? 2 : 1); + idTable.setNumColumns(term.ends_with('*') ? 3 : 2); auto optionalTbmd = getTextBlockMetadataForWordOrPrefix(term); if (!optionalTbmd.has_value()) { return idTable; diff --git a/src/index/IndexImpl.cpp b/src/index/IndexImpl.cpp index ed8a6dd526..4f5ce915fe 100644 --- a/src/index/IndexImpl.cpp +++ b/src/index/IndexImpl.cpp @@ -1128,6 +1128,7 @@ void IndexImpl::readConfiguration() { loadDataMember("num-subjects", numSubjects_, NumNormalAndInternal{}); loadDataMember("num-objects", numObjects_, NumNormalAndInternal{}); loadDataMember("num-triples", numTriples_, NumNormalAndInternal{}); + loadDataMember("num-non-literals-text-index", nofNonLiteralsInTextIndex_, 0); // Initialize BlankNodeManager uint64_t numBlankNodesTotal; diff --git a/src/index/IndexImpl.h b/src/index/IndexImpl.h index 0d5b396ccc..b98d0d5788 100644 --- a/src/index/IndexImpl.h +++ b/src/index/IndexImpl.h @@ -158,6 +158,11 @@ class IndexImpl { NumNormalAndInternal numTriples_; string indexId_; + // Keeps track of the number of nonLiteral contexts in the index this is used + // in the test retrieval of the texts. This only works reliably if the + // wordsFile.tsv starts with contextId 1 and is continuous. + size_t nofNonLiteralsInTextIndex_; + // Global static pointers to the currently active index and comparator. // Those are used to compare LocalVocab entries with each other as well as // with Vocab entries. @@ -424,6 +429,9 @@ class IndexImpl { size_t getNofEntityPostings() const { return textMeta_.getNofEntityPostings(); } + size_t getNofNonLiteralsInTextIndex() const { + return nofNonLiteralsInTextIndex_; + } bool hasAllPermutations() const { return SPO().isLoaded(); } @@ -624,14 +632,17 @@ class IndexImpl { ad_utility::File& file) const; // TODO understand what the "codes" are, are they better just ints? - typedef ad_utility::HashMap WordToCodeMap; + // After using createCodebooks on these types, the lowest codes refer to the + // most frequent WordIndex/Score. The maps are mapping those codes to their + // respective frequency. + typedef ad_utility::HashMap WordCodeMap; typedef ad_utility::HashMap ScoreCodeMap; typedef vector WordCodebook; typedef vector ScoreCodebook; //! Creates codebooks for lists that are supposed to be entropy encoded. void createCodebooks(const vector& postings, - WordToCodeMap& wordCodemap, WordCodebook& wordCodebook, + WordCodeMap& wordCodemap, WordCodebook& wordCodebook, ScoreCodeMap& scoreCodemap, ScoreCodebook& scoreCodebook) const; diff --git a/src/parser/ContextFileParser.h b/src/parser/ContextFileParser.h index e00a268d24..ba8d7bac9c 100644 --- a/src/parser/ContextFileParser.h +++ b/src/parser/ContextFileParser.h @@ -21,6 +21,7 @@ class ContextFileParser { bool _isEntity; TextRecordIndex _contextId; Score _score; + bool _isLiteralEntity = false; }; explicit ContextFileParser(const string& contextFile, diff --git a/src/parser/data/Variable.cpp b/src/parser/data/Variable.cpp index 89e0e894a2..00371c3537 100644 --- a/src/parser/data/Variable.cpp +++ b/src/parser/data/Variable.cpp @@ -56,7 +56,7 @@ Variable::Variable(std::string name, bool checkName) : _name{std::move(name)} { } // _____________________________________________________________________________ -Variable Variable::getScoreVariable( +Variable Variable::getEntityScoreVariable( const std::variant& varOrEntity) const { std::string_view type; std::string entity; @@ -65,20 +65,29 @@ Variable Variable::getScoreVariable( entity = std::get(varOrEntity).name().substr(1); } else { type = "_fixedEntity_"; - // Converts input string to unambiguous result string not containing any - // special characters. "_" is used as an escaping character. - for (char c : std::get(varOrEntity)) { - if (isalpha(static_cast(c))) { - entity += c; - } else { - absl::StrAppend(&entity, "_", std::to_string(c), "_"); - } - } + appendEscapedWord(std::get(varOrEntity), entity); } return Variable{ absl::StrCat(SCORE_VARIABLE_PREFIX, name().substr(1), type, entity)}; } +// _____________________________________________________________________________ +Variable Variable::getWordScoreVariable(std::string_view word, + bool isPrefix) const { + std::string_view type; + std::string convertedWord; + if (isPrefix) { + word.remove_suffix(1); + type = "prefix_"; + } else { + type = "word_"; + } + convertedWord = "_"; + appendEscapedWord(word, convertedWord); + return Variable{absl::StrCat(SCORE_VARIABLE_PREFIX, type, name().substr(1), + convertedWord)}; +} + // _____________________________________________________________________________ Variable Variable::getMatchingWordVariable(std::string_view term) const { return Variable{ @@ -96,3 +105,15 @@ bool Variable::isValidVariableName(std::string_view var) { return false; } } + +// _____________________________________________________________________________ +void Variable::appendEscapedWord(std::string_view word, + std::string& target) const { + for (char c : word) { + if (isalpha(static_cast(c))) { + target += c; + } else { + absl::StrAppend(&target, "_", std::to_string(c), "_"); + } + } +} diff --git a/src/parser/data/Variable.h b/src/parser/data/Variable.h index a378cfd903..5d89d21aac 100644 --- a/src/parser/data/Variable.h +++ b/src/parser/data/Variable.h @@ -48,9 +48,17 @@ class Variable { // `?ql_someTextVar_fixedEntity_someFixedEntity`. // Note that if the the fixed entity contains non ascii characters they are // converted to numbers and escaped. - Variable getScoreVariable( + Variable getEntityScoreVariable( const std::variant& varOrEntity) const; + // Converts `?someTextVar` and `someWord` into + // `?ql_score_word_someTextVar_someWord. + // Converts `?someTextVar` and `somePrefix*` into + // `?ql_score_prefix_someTextVar_somePrefix`. + // Note that if the word contains non ascii characters they are converted to + // numbers and escaped. + Variable getWordScoreVariable(std::string_view word, bool isPrefix) const; + // Convert `?someVariable` into `?ql_matchingword_someVariable_someTerm` Variable getMatchingWordVariable(std::string_view term) const; @@ -72,4 +80,8 @@ class Variable { } static bool isValidVariableName(std::string_view var); + + // The method escapes all special chars in word to "_ASCIICODE_" and appends + // it at the end of target + void appendEscapedWord(std::string_view word, std::string& target) const; }; diff --git a/src/parser/sparqlParser/SparqlQleverVisitor.cpp b/src/parser/sparqlParser/SparqlQleverVisitor.cpp index 28054466db..f23530f820 100644 --- a/src/parser/sparqlParser/SparqlQleverVisitor.cpp +++ b/src/parser/sparqlParser/SparqlQleverVisitor.cpp @@ -1543,6 +1543,7 @@ void Visitor::setMatchingWordAndScoreVisibleIfPresent( } for (std::string_view s : std::vector( absl::StrSplit(name.substr(1, name.size() - 2), ' '))) { + addVisibleVariable(var->getWordScoreVariable(s, s.ends_with('*'))); if (!s.ends_with('*')) { continue; } @@ -1551,9 +1552,9 @@ void Visitor::setMatchingWordAndScoreVisibleIfPresent( } } else if (propertyPath->asString() == CONTAINS_ENTITY_PREDICATE) { if (const auto* entVar = std::get_if(&object)) { - addVisibleVariable(var->getScoreVariable(*entVar)); + addVisibleVariable(var->getEntityScoreVariable(*entVar)); } else { - addVisibleVariable(var->getScoreVariable(object.toSparql())); + addVisibleVariable(var->getEntityScoreVariable(object.toSparql())); } } } diff --git a/test/QueryPlannerTest.cpp b/test/QueryPlannerTest.cpp index 32261e871e..92ae4a32c0 100644 --- a/test/QueryPlannerTest.cpp +++ b/test/QueryPlannerTest.cpp @@ -2599,12 +2599,12 @@ TEST(QueryPlanner, TextLimit) { h::expect( "SELECT * WHERE { ?text ql:contains-word \"test*\" . ?text " "ql:contains-entity } TEXTLIMIT 10", - h::TextLimit( - 10, - h::Join(wordScan(Var{"?text"}, "test*"), - entityScan(Var{"?text"}, "", "test*")), - Var{"?text"}, vector{}, - vector{Var{"?text"}.getScoreVariable("")}), + h::TextLimit(10, + h::Join(wordScan(Var{"?text"}, "test*"), + entityScan(Var{"?text"}, "", "test*")), + Var{"?text"}, vector{}, + vector{ + Var{"?text"}.getEntityScoreVariable("")}), qec); // Contains entity @@ -2616,7 +2616,8 @@ TEST(QueryPlanner, TextLimit) { h::Join(wordScan(Var{"?text"}, "test*"), entityScan(Var{"?text"}, Var{"?scientist"}, "test*")), Var{"?text"}, vector{Var{"?scientist"}}, - vector{Var{"?text"}.getScoreVariable(Var{"?scientist"})}), + vector{ + Var{"?text"}.getEntityScoreVariable(Var{"?scientist"})}), qec); // Contains entity and fixed entity @@ -2624,15 +2625,15 @@ TEST(QueryPlanner, TextLimit) { "SELECT * WHERE { ?text ql:contains-entity ?scientist . ?text " "ql:contains-word \"test*\" . ?text ql:contains-entity } " "TEXTLIMIT 5", - h::TextLimit( - 5, - h::UnorderedJoins( - wordScan(Var{"?text"}, "test*"), - entityScan(Var{"?text"}, Var{"?scientist"}, "test*"), - entityScan(Var{"?text"}, "", "test*")), - Var{"?text"}, vector{Var{"?scientist"}}, - vector{Var{"?text"}.getScoreVariable(Var{"?scientist"}), - Var{"?text"}.getScoreVariable("")}), + h::TextLimit(5, + h::UnorderedJoins( + wordScan(Var{"?text"}, "test*"), + entityScan(Var{"?text"}, Var{"?scientist"}, "test*"), + entityScan(Var{"?text"}, "", "test*")), + Var{"?text"}, vector{Var{"?scientist"}}, + vector{ + Var{"?text"}.getEntityScoreVariable(Var{"?scientist"}), + Var{"?text"}.getEntityScoreVariable("")}), qec); // Contains two entities @@ -2647,8 +2648,9 @@ TEST(QueryPlanner, TextLimit) { entityScan(Var{"?text"}, Var{"?scientist"}, "test*"), entityScan(Var{"?text"}, Var{"?scientist2"}, "test*")), Var{"?text"}, vector{Var{"?scientist"}, Var{"?scientist2"}}, - vector{Var{"?text"}.getScoreVariable(Var{"?scientist"}), - Var{"?text"}.getScoreVariable(Var{"?scientist2"})}), + vector{ + Var{"?text"}.getEntityScoreVariable(Var{"?scientist"}), + Var{"?text"}.getEntityScoreVariable(Var{"?scientist2"})}), qec); // Contains two text variables. Also checks if the textlimit at an efficient @@ -2665,17 +2667,17 @@ TEST(QueryPlanner, TextLimit) { entityScan(Var{"?text1"}, Var{"?scientist1"}, "test*")), Var{"?text1"}, vector{Var{"?scientist1"}}, vector{ - Var{"?text1"}.getScoreVariable(Var{"?scientist1"})}), - h::TextLimit(5, - h::UnorderedJoins( - wordScan(Var{"?text2"}, "test*"), - entityScan(Var{"?text2"}, Var{"?author1"}, "test*"), - entityScan(Var{"?text2"}, Var{"?author2"}, "test*")), - Var{"?text2"}, - vector{Var{"?author1"}, Var{"?author2"}}, - vector{ - Var{"?text2"}.getScoreVariable(Var{"?author1"}), - Var{"?text2"}.getScoreVariable(Var{"?author2"})})), + Var{"?text1"}.getEntityScoreVariable(Var{"?scientist1"})}), + h::TextLimit( + 5, + h::UnorderedJoins( + wordScan(Var{"?text2"}, "test*"), + entityScan(Var{"?text2"}, Var{"?author1"}, "test*"), + entityScan(Var{"?text2"}, Var{"?author2"}, "test*")), + Var{"?text2"}, vector{Var{"?author1"}, Var{"?author2"}}, + vector{ + Var{"?text2"}.getEntityScoreVariable(Var{"?author1"}), + Var{"?text2"}.getEntityScoreVariable(Var{"?author2"})})), qec); } diff --git a/test/QueryPlannerTestHelpers.h b/test/QueryPlannerTestHelpers.h index dfc037645b..c300bf0d5f 100644 --- a/test/QueryPlannerTestHelpers.h +++ b/test/QueryPlannerTestHelpers.h @@ -131,7 +131,7 @@ constexpr auto TextIndexScanForWord = [](Variable textRecordVar, string word) -> QetMatcher { return RootOperation<::TextIndexScanForWord>(AllOf( AD_PROPERTY(::TextIndexScanForWord, getResultWidth, - Eq(1 + word.ends_with('*'))), + Eq(2 + word.ends_with('*'))), AD_PROPERTY(::TextIndexScanForWord, textRecordVar, Eq(textRecordVar)), AD_PROPERTY(::TextIndexScanForWord, word, word))); }; diff --git a/test/engine/TextIndexScanForWordTest.cpp b/test/engine/TextIndexScanForWordTest.cpp index 57f4b46e02..eac3cb0d2f 100644 --- a/test/engine/TextIndexScanForWordTest.cpp +++ b/test/engine/TextIndexScanForWordTest.cpp @@ -5,6 +5,7 @@ #include #include +#include "../printers/VariablePrinters.h" #include "../util/GTestHelpers.h" #include "../util/IdTableHelpers.h" #include "../util/IndexTestHelpers.h" @@ -18,52 +19,170 @@ using ad_utility::source_location; namespace h = textIndexScanTestHelpers; namespace { + std::string kg = "

\"he failed the test\" .

\"testing can help\" .

" "\"some other sentence\" .

\"the test on friday was really hard\" " - ". . ."; + ". . . ."; + +std::string wordsFileContent = + h::createWordsFileLine("astronomer", false, 1, 1) + + h::createWordsFileLine("", true, 1, 0) + + h::createWordsFileLine("scientist", false, 1, 1) + + h::createWordsFileLine("field", false, 1, 1) + + h::createWordsFileLine("astronomy", false, 1, 1) + + h::createWordsFileLine("astronomer", false, 2, 0) + + h::createWordsFileLine("", true, 2, 0) + + h::createWordsFileLine(":s:firstsentence", false, 2, 0) + + h::createWordsFileLine("scientist", false, 2, 0) + + h::createWordsFileLine("field", false, 2, 0) + + h::createWordsFileLine("astronomy", false, 2, 0) + + h::createWordsFileLine("astronomy", false, 3, 1) + + h::createWordsFileLine("concentrates", false, 3, 1) + + h::createWordsFileLine("studies", false, 3, 1) + + h::createWordsFileLine("specific", false, 3, 1) + + h::createWordsFileLine("question", false, 3, 1) + + h::createWordsFileLine("outside", false, 3, 1) + + h::createWordsFileLine("scope", false, 3, 1) + + h::createWordsFileLine("earth", false, 3, 1) + + h::createWordsFileLine("astronomy", false, 4, 1) + + h::createWordsFileLine("concentrates", false, 4, 1) + + h::createWordsFileLine("studies", false, 4, 1) + + h::createWordsFileLine("field", false, 4, 1) + + h::createWordsFileLine("outside", false, 4, 1) + + h::createWordsFileLine("scope", false, 4, 1) + + h::createWordsFileLine("earth", false, 4, 1) + + h::createWordsFileLine("tester", false, 5, 1) + + h::createWordsFileLine("rockets", false, 5, 1) + + h::createWordsFileLine("astronomer", false, 5, 1) + + h::createWordsFileLine("", true, 5, 0) + + h::createWordsFileLine("although", false, 5, 1) + + h::createWordsFileLine("astronomer", false, 6, 0) + + h::createWordsFileLine("", true, 6, 0) + + h::createWordsFileLine("although", false, 6, 0) + + h::createWordsFileLine("", true, 6, 0) + + h::createWordsFileLine("space", false, 6, 1) + + h::createWordsFileLine("", true, 7, 0) + + h::createWordsFileLine("space", false, 7, 0) + + h::createWordsFileLine("earth", false, 7, 1); + +std::string firstDocText = + "An astronomer is a scientist in the field of " + "astronomy who concentrates their studies on a " + "specific question or field outside of the scope of " + "Earth."; + +std::string secondDocText = + "The Tester of the rockets can be an astronomer " + "too although they might not be in space but on " + "earth."; + +std::string docsFileContent = h::createDocsFileLine(4, firstDocText) + + h::createDocsFileLine(7, secondDocText); + +std::pair contentsOfWordsFileAndDocsFile = { + wordsFileContent, docsFileContent}; TEST(TextIndexScanForWord, WordScanPrefix) { - auto qec = getQec(kg, true, true, true, 16_B, true); + auto qec = getQec(kg, true, true, true, 16_B, true, true, + contentsOfWordsFileAndDocsFile); TextIndexScanForWord s1{qec, Variable{"?text1"}, "test*"}; TextIndexScanForWord s2{qec, Variable{"?text2"}, "test*"}; - ASSERT_EQ(s1.getResultWidth(), 2); + // Test if size calculations are right + ASSERT_EQ(s1.getResultWidth(), 3); auto result = s1.computeResultOnlyForTesting(); - ASSERT_EQ(result.idTable().numColumns(), 2); - ASSERT_EQ(result.idTable().size(), 3); + ASSERT_EQ(result.idTable().numColumns(), 3); + ASSERT_EQ(result.idTable().size(), 4); s2.getExternallyVisibleVariableColumns(); + // Test if all columns are there and correct using enum ColumnIndexAndTypeInfo::UndefStatus; VariableToColumnMap expectedVariables{ {Variable{"?text2"}, {0, AlwaysDefined}}, - {Variable{"?ql_matchingword_text2_test"}, {1, AlwaysDefined}}}; + {Variable{"?ql_matchingword_text2_test"}, {1, AlwaysDefined}}, + {Variable{"?ql_score_prefix_text2_test"}, {2, AlwaysDefined}}}; EXPECT_THAT(s2.getExternallyVisibleVariableColumns(), ::testing::UnorderedElementsAreArray(expectedVariables)); - ASSERT_EQ(h::combineToString("\"he failed the test\"", "test"), + // Tests if the correct texts are retrieved from a mix of non literal and + // literal texts + ASSERT_EQ(h::combineToString(secondDocText, "tester"), h::combineToString(h::getTextRecordFromResultTable(qec, result, 0), h::getWordFromResultTable(qec, result, 0))); - ASSERT_EQ(h::combineToString("\"testing can help\"", "testing"), + + ASSERT_EQ(h::combineToString("\"he failed the test\"", "test"), h::combineToString(h::getTextRecordFromResultTable(qec, result, 1), h::getWordFromResultTable(qec, result, 1))); + ASSERT_EQ(h::combineToString("\"testing can help\"", "testing"), + h::combineToString(h::getTextRecordFromResultTable(qec, result, 2), + h::getWordFromResultTable(qec, result, 2))); ASSERT_EQ( h::combineToString("\"the test on friday was really hard\"", "test"), - h::combineToString(h::getTextRecordFromResultTable(qec, result, 2), - h::getWordFromResultTable(qec, result, 2))); + h::combineToString(h::getTextRecordFromResultTable(qec, result, 3), + h::getWordFromResultTable(qec, result, 3))); + + // Tests if the correct texts are retrieved from the non literal texts + TextIndexScanForWord t1{qec, Variable{"?t1"}, "astronom*"}; + auto tresult = t1.computeResultOnlyForTesting(); + ASSERT_EQ(TextRecordIndex::make(1), + h::getTextRecordIdFromResultTable(qec, tresult, 0)); + ASSERT_EQ(firstDocText, h::getTextRecordFromResultTable(qec, tresult, 0)); + ASSERT_EQ(TextRecordIndex::make(1), + h::getTextRecordIdFromResultTable(qec, tresult, 1)); + ASSERT_EQ(firstDocText, h::getTextRecordFromResultTable(qec, tresult, 1)); + ASSERT_EQ(TextRecordIndex::make(2), + h::getTextRecordIdFromResultTable(qec, tresult, 2)); + ASSERT_EQ(firstDocText, h::getTextRecordFromResultTable(qec, tresult, 2)); + ASSERT_EQ(TextRecordIndex::make(2), + h::getTextRecordIdFromResultTable(qec, tresult, 3)); + ASSERT_EQ(firstDocText, h::getTextRecordFromResultTable(qec, tresult, 3)); + ASSERT_EQ(TextRecordIndex::make(3), + h::getTextRecordIdFromResultTable(qec, tresult, 4)); + ASSERT_EQ(firstDocText, h::getTextRecordFromResultTable(qec, tresult, 4)); + ASSERT_EQ(TextRecordIndex::make(4), + h::getTextRecordIdFromResultTable(qec, tresult, 5)); + ASSERT_EQ(firstDocText, h::getTextRecordFromResultTable(qec, tresult, 5)); + ASSERT_EQ(TextRecordIndex::make(5), + h::getTextRecordIdFromResultTable(qec, tresult, 6)); + ASSERT_EQ(secondDocText, h::getTextRecordFromResultTable(qec, tresult, 6)); + ASSERT_EQ(TextRecordIndex::make(6), + h::getTextRecordIdFromResultTable(qec, tresult, 7)); + ASSERT_EQ(secondDocText, h::getTextRecordFromResultTable(qec, tresult, 7)); + + // Tests if correct words are deducted from prefix + ASSERT_EQ("astronomer", h::getWordFromResultTable(qec, tresult, 0)); + ASSERT_EQ("astronomy", h::getWordFromResultTable(qec, tresult, 1)); + ASSERT_EQ("astronomer", h::getWordFromResultTable(qec, tresult, 2)); + ASSERT_EQ("astronomy", h::getWordFromResultTable(qec, tresult, 3)); + ASSERT_EQ("astronomy", h::getWordFromResultTable(qec, tresult, 4)); + ASSERT_EQ("astronomy", h::getWordFromResultTable(qec, tresult, 5)); + ASSERT_EQ("astronomer", h::getWordFromResultTable(qec, tresult, 6)); + ASSERT_EQ("astronomer", h::getWordFromResultTable(qec, tresult, 7)); + + // Tests if the correct scores are retrieved from the non literal texts + ASSERT_EQ(1, h::getScoreFromResultTable(qec, tresult, 0, true)); + ASSERT_EQ(1, h::getScoreFromResultTable(qec, tresult, 1, true)); + ASSERT_EQ(0, h::getScoreFromResultTable(qec, tresult, 2, true)); + ASSERT_EQ(0, h::getScoreFromResultTable(qec, tresult, 3, true)); + ASSERT_EQ(1, h::getScoreFromResultTable(qec, tresult, 4, true)); + ASSERT_EQ(1, h::getScoreFromResultTable(qec, tresult, 5, true)); + ASSERT_EQ(1, h::getScoreFromResultTable(qec, tresult, 6, true)); + ASSERT_EQ(0, h::getScoreFromResultTable(qec, tresult, 7, true)); } TEST(TextIndexScanForWord, WordScanBasic) { - auto qec = getQec(kg, true, true, true, 16_B, true); + auto qec = getQec(kg, true, true, true, 16_B, true, true, + contentsOfWordsFileAndDocsFile); TextIndexScanForWord s1{qec, Variable{"?text1"}, "test"}; - ASSERT_EQ(s1.getResultWidth(), 1); + ASSERT_EQ(s1.getResultWidth(), 2); auto result = s1.computeResultOnlyForTesting(); - ASSERT_EQ(result.idTable().numColumns(), 1); + ASSERT_EQ(result.idTable().numColumns(), 2); ASSERT_EQ(result.idTable().size(), 2); ASSERT_EQ("\"he failed the test\"", @@ -73,18 +192,29 @@ TEST(TextIndexScanForWord, WordScanBasic) { TextIndexScanForWord s2{qec, Variable{"?text1"}, "testing"}; - ASSERT_EQ(s2.getResultWidth(), 1); + ASSERT_EQ(s2.getResultWidth(), 2); result = s2.computeResultOnlyForTesting(); - ASSERT_EQ(result.idTable().numColumns(), 1); + ASSERT_EQ(result.idTable().numColumns(), 2); ASSERT_EQ(result.idTable().size(), 1); ASSERT_EQ("\"testing can help\"", h::getTextRecordFromResultTable(qec, result, 0)); + + TextIndexScanForWord s3{qec, Variable{"?text1"}, "tester"}; + + ASSERT_EQ(s3.getResultWidth(), 2); + + result = s3.computeResultOnlyForTesting(); + ASSERT_EQ(result.idTable().numColumns(), 2); + ASSERT_EQ(result.idTable().size(), 1); + + ASSERT_EQ(secondDocText, h::getTextRecordFromResultTable(qec, result, 0)); } TEST(TextIndexScanForWord, CacheKey) { - auto qec = getQec(kg, true, true, true, 16_B, true); + auto qec = getQec(kg, true, true, true, 16_B, true, true, + contentsOfWordsFileAndDocsFile); TextIndexScanForWord s1{qec, Variable{"?text1"}, "test*"}; TextIndexScanForWord s2{qec, Variable{"?text2"}, "test*"}; @@ -107,7 +237,8 @@ TEST(TextIndexScanForWord, CacheKey) { } TEST(TextIndexScanForWord, KnownEmpty) { - auto qec = getQec(kg, true, true, true, 16_B, true); + auto qec = getQec(kg, true, true, true, 16_B, true, true, + contentsOfWordsFileAndDocsFile); TextIndexScanForWord s1{qec, Variable{"?text1"}, "nonExistentWord*"}; ASSERT_TRUE(s1.knownEmptyResult()); diff --git a/test/engine/TextIndexScanTestHelpers.h b/test/engine/TextIndexScanTestHelpers.h index 597344ad9e..83a72ddea4 100644 --- a/test/engine/TextIndexScanTestHelpers.h +++ b/test/engine/TextIndexScanTestHelpers.h @@ -4,17 +4,40 @@ #pragma once +#include "global/IndexTypes.h" namespace textIndexScanTestHelpers { // NOTE: this function exploits a "lucky accident" that allows us to // obtain the textRecord using indexToString. // TODO: Implement a more elegant/stable version +// Idea for a more stable version is to add the literals to the docsfile +// which is later parsed and written to the docsDB. This would lead to a +// possible retrieval of the literals text with the getTextExcerpt function. +// The only problem is the increased size of the docsDB and the double saving +// of the literals. inline string getTextRecordFromResultTable(const QueryExecutionContext* qec, const ProtoResult& result, const size_t& rowIndex) { - return qec->getIndex().indexToString( - result.idTable().getColumn(0)[rowIndex].getVocabIndex()); + size_t nofNonLiterals = qec->getIndex().getNofNonLiteralsInTextIndex(); + uint64_t textRecordIdFromTable = + result.idTable().getColumn(0)[rowIndex].getTextRecordIndex().get(); + if (nofNonLiterals <= textRecordIdFromTable) { + // Return when from Literals + return qec->getIndex().indexToString( + VocabIndex::make(textRecordIdFromTable - nofNonLiterals)); + } else { + // Return when from DocsDB + return qec->getIndex().getTextExcerpt( + result.idTable().getColumn(0)[rowIndex].getTextRecordIndex()); + } +} + +inline const TextRecordIndex getTextRecordIdFromResultTable( + [[maybe_unused]] const QueryExecutionContext* qec, + const ProtoResult& result, const size_t& rowIndex) { + return result.idTable().getColumn(0)[rowIndex].getTextRecordIndex(); } +// Only use on prefix search results inline string getEntityFromResultTable(const QueryExecutionContext* qec, const ProtoResult& result, const size_t& rowIndex) { @@ -22,6 +45,7 @@ inline string getEntityFromResultTable(const QueryExecutionContext* qec, result.idTable().getColumn(1)[rowIndex].getVocabIndex()); } +// Only use on prefix search results inline string getWordFromResultTable(const QueryExecutionContext* qec, const ProtoResult& result, const size_t& rowIndex) { @@ -29,9 +53,31 @@ inline string getWordFromResultTable(const QueryExecutionContext* qec, result.idTable().getColumn(1)[rowIndex].getWordVocabIndex())}; } +inline size_t getScoreFromResultTable( + [[maybe_unused]] const QueryExecutionContext* qec, + const ProtoResult& result, const size_t& rowIndex, bool wasPrefixSearch) { + size_t colToRetrieve = wasPrefixSearch ? 2 : 1; + return static_cast( + result.idTable().getColumn(colToRetrieve)[rowIndex].getInt()); +} + inline string combineToString(const string& text, const string& word) { std::stringstream ss; ss << "Text: " << text << ", Word: " << word << std::endl; return ss.str(); } + +inline std::string inlineSeparator = "\t"; +inline std::string lineSeparator = "\n"; + +inline string createWordsFileLine(std::string word, bool isEntity, + size_t contextId, size_t score) { + return word + inlineSeparator + (isEntity ? "1" : "0") + inlineSeparator + + std::to_string(contextId) + inlineSeparator + std::to_string(score) + + lineSeparator; +}; + +inline string createDocsFileLine(size_t docId, std::string docContent) { + return std::to_string(docId) + inlineSeparator + docContent + lineSeparator; +}; } // namespace textIndexScanTestHelpers diff --git a/test/util/IndexTestHelpers.cpp b/test/util/IndexTestHelpers.cpp index 92d665a491..0dcfd334a6 100644 --- a/test/util/IndexTestHelpers.cpp +++ b/test/util/IndexTestHelpers.cpp @@ -46,7 +46,12 @@ std::vector getAllIndexFilenames( indexBasename + ".prefixes", indexBasename + ".vocabulary.internal", indexBasename + ".vocabulary.external", - indexBasename + ".vocabulary.external.offsets"}; + indexBasename + ".vocabulary.external.offsets", + indexBasename + ".wordsfile", + indexBasename + ".docsfile", + indexBasename + ".text.index", + indexBasename + ".text.vocabulary", + indexBasename + ".text.docsDB"}; } namespace { @@ -134,7 +139,9 @@ Index makeTestIndex(const std::string& indexBasename, bool loadAllPermutations, bool usePatterns, [[maybe_unused]] bool usePrefixCompression, ad_utility::MemorySize blocksizePermutations, - bool createTextIndex) { + bool createTextIndex, bool addWordsFromLiterals, + std::optional> + contentsOfWordsFileAndDocsFile) { // Ignore the (irrelevant) log output of the index building and loading during // these tests. static std::ostringstream ignoreLogStream; @@ -181,7 +188,29 @@ Index makeTestIndex(const std::string& indexBasename, std::nullopt}; index.createFromFiles({spec}); if (createTextIndex) { - index.addTextFromContextFile("", true); + if (contentsOfWordsFileAndDocsFile.has_value()) { + // Create and write to words- and docsfile to later build a full text + // index from them + ad_utility::File wordsFile(indexBasename + ".wordsfile", "w"); + ad_utility::File docsFile(indexBasename + ".docsfile", "w"); + wordsFile.write(contentsOfWordsFileAndDocsFile.value().first.c_str(), + contentsOfWordsFileAndDocsFile.value().first.size()); + docsFile.write(contentsOfWordsFileAndDocsFile.value().second.c_str(), + contentsOfWordsFileAndDocsFile.value().second.size()); + wordsFile.close(); + docsFile.close(); + index.setKbName(indexBasename); + index.setTextName(indexBasename); + index.setOnDiskBase(indexBasename); + if (addWordsFromLiterals) { + index.addTextFromContextFile(indexBasename + ".wordsfile", true); + } else { + index.addTextFromContextFile(indexBasename + ".wordsfile", false); + } + index.buildDocsDB(indexBasename + ".docsfile"); + } else if (addWordsFromLiterals) { + index.addTextFromContextFile("", true); + } } } if (!usePatterns || !loadAllPermutations) { @@ -216,7 +245,9 @@ QueryExecutionContext* getQec(std::optional turtleInput, bool loadAllPermutations, bool usePatterns, bool usePrefixCompression, ad_utility::MemorySize blocksizePermutations, - bool createTextIndex) { + bool createTextIndex, bool addWordsFromLiterals, + std::optional> + contentsOfWordsFileAndDocsFile) { // Similar to `absl::Cleanup`. Calls the `callback_` in the destructor, but // the callback is stored as a `std::function`, which allows to store // different types of callbacks in the same wrapper type. @@ -275,7 +306,8 @@ QueryExecutionContext* getQec(std::optional turtleInput, std::make_unique(makeTestIndex( testIndexBasename, turtleInput, loadAllPermutations, usePatterns, usePrefixCompression, - blocksizePermutations, createTextIndex)), + blocksizePermutations, createTextIndex, + addWordsFromLiterals, contentsOfWordsFileAndDocsFile)), std::make_unique()}); } auto* qec = contextMap.at(key).qec_.get(); diff --git a/test/util/IndexTestHelpers.h b/test/util/IndexTestHelpers.h index 3e09604613..cbbd5ea486 100644 --- a/test/util/IndexTestHelpers.h +++ b/test/util/IndexTestHelpers.h @@ -44,7 +44,10 @@ Index makeTestIndex(const std::string& indexBasename, bool loadAllPermutations = true, bool usePatterns = true, bool usePrefixCompression = true, ad_utility::MemorySize blocksizePermutations = 16_B, - bool createTextIndex = false); + bool createTextIndex = false, + bool addWordsFromLiterals = true, + std::optional> + contentsOfWordsFileAndDocsfile = std::nullopt); // Return a static `QueryExecutionContext` that refers to an index that was // build using `makeTestIndex` (see above). The index (most notably its @@ -55,7 +58,9 @@ QueryExecutionContext* getQec( bool loadAllPermutations = true, bool usePatterns = true, bool usePrefixCompression = true, ad_utility::MemorySize blocksizePermutations = 16_B, - bool createTextIndex = false); + bool createTextIndex = false, bool addWordsFromLiterals = true, + std::optional> + contentsOfWordsFileAndDocsfile = std::nullopt); // Return a lambda that takes a string and converts it into an ID by looking // it up in the vocabulary of `index`. An `AD_CONTRACT_CHECK` will fail if the From 44c2b4fb8061abd35bd2653010c10f612d64a94b Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Mon, 16 Dec 2024 20:46:13 +0100 Subject: [PATCH 06/14] Fix bug for UNION with SUBQUERY with invisible variables (#1682) There was a bug in the following scenario: a UNION that is lazily computed and at least one of the inputs of the UNION contains a subquery that does not SELECT all possible variables. Then the UNION threw an assertion, because there was some confusion between the number of visible variables in the input (only the ones selected by the subquery) and the actual number of columns in the result (which includes the not-selected variables of the subquery). Fixes #1681 --- src/engine/Union.cpp | 5 ++++- test/UnionTest.cpp | 35 +++++++++++++++++++++++++++-------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/engine/Union.cpp b/src/engine/Union.cpp index 46e5d06f4a..ec1afcba05 100644 --- a/src/engine/Union.cpp +++ b/src/engine/Union.cpp @@ -242,7 +242,10 @@ std::vector Union::computePermutation() const { // _____________________________________________________________________________ IdTable Union::transformToCorrectColumnFormat( IdTable idTable, const std::vector& permutation) const { - while (idTable.numColumns() < getResultWidth()) { + // NOTE: previously the check was for `getResultWidth()`, but that is wrong if + // some variables in the subtree are invisible because of a subquery. + auto maxNumRequiredColumns = ql::ranges::max(permutation) + 1; + while (idTable.numColumns() < maxNumRequiredColumns) { idTable.addEmptyColumn(); ad_utility::chunkedFill(idTable.getColumn(idTable.numColumns() - 1), Id::makeUndefined(), chunkSize, diff --git a/test/UnionTest.cpp b/test/UnionTest.cpp index b0f52f0420..265aa92ec8 100644 --- a/test/UnionTest.cpp +++ b/test/UnionTest.cpp @@ -74,17 +74,34 @@ TEST(Union, computeUnionLarge) { // _____________________________________________________________________________ TEST(Union, computeUnionLazy) { - auto runTest = [](bool nonLazyChildren, + auto runTest = [](bool nonLazyChildren, bool invisibleSubtreeColumns, ad_utility::source_location loc = ad_utility::source_location::current()) { auto l = generateLocationTrace(loc); auto* qec = ad_utility::testing::getQec(); qec->getQueryTreeCache().clearAll(); - IdTable left = makeIdTableFromVector({{V(1)}, {V(2)}, {V(3)}}); - auto leftT = ad_utility::makeExecutionTree( - qec, std::move(left), Vars{Variable{"?x"}}, false, - std::vector{}, LocalVocab{}, std::nullopt, - nonLazyChildren); + auto leftT = [&]() { + if (!invisibleSubtreeColumns) { + IdTable left = makeIdTableFromVector({{V(1)}, {V(2)}, {V(3)}}); + return ad_utility::makeExecutionTree( + qec, std::move(left), Vars{Variable{"?x"}}, false, + std::vector{}, LocalVocab{}, std::nullopt, + nonLazyChildren); + } else { + // With the `invisibleSubtreeColumns==true` case we test the case, that + // the input contains variables that are not visible because of a + // subquery. This case was previously buggy and triggered an assertion. + IdTable left = makeIdTableFromVector( + {{V(1), V(3)}, {V(2), V(27)}, {V(3), V(123)}}); + auto tree = ad_utility::makeExecutionTree( + qec, std::move(left), Vars{Variable{"?x"}, Variable{"?invisible"}}, + false, std::vector{}, LocalVocab{}, std::nullopt, + nonLazyChildren); + tree->getRootOperation()->setSelectedVariablesForSubquery( + {Variable{"?x"}}); + return tree; + } + }(); IdTable right = makeIdTableFromVector({{V(4), V(5)}, {V(6), V(7)}}); auto rightT = ad_utility::makeExecutionTree( @@ -112,8 +129,10 @@ TEST(Union, computeUnionLazy) { ASSERT_EQ(++iterator, result.end()); }; - runTest(false); - runTest(true); + runTest(false, false); + runTest(false, true); + runTest(true, false); + runTest(true, true); } // _____________________________________________________________________________ From 6429061bc09aeb95f42be2d3850def99ef8a969c Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Mon, 16 Dec 2024 21:34:04 +0100 Subject: [PATCH 07/14] Allow GROUP BY variables with parentheses or duplicates (#1685) With this change, grouped variables can be put in parentheses and still be used in the SELECT clause. For example, `GROUP BY (?x)` is now handled exactly like `GROUP BY ?x`. Additionally, we now deduplicate the grouped variables. For example, `GROUP BY ?x ?x` is now equivalent to `GROUP BY ?x`. --- src/parser/ParsedQuery.cpp | 22 ++++++++++++++++------ test/QueryPlannerTest.cpp | 9 +++++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/parser/ParsedQuery.cpp b/src/parser/ParsedQuery.cpp index 6bee76da2c..6cb844071b 100644 --- a/src/parser/ParsedQuery.cpp +++ b/src/parser/ParsedQuery.cpp @@ -356,17 +356,27 @@ void ParsedQuery::checkUsedVariablesAreVisible( // ____________________________________________________________________________ void ParsedQuery::addGroupByClause(std::vector groupKeys) { - // Process groupClause - auto processVariable = [this](const Variable& groupKey) { + // Deduplicate the group by variables to support e.g. `GROUP BY ?x ?x ?x`. + // Note: The `GroupBy` class expects the grouped variables to be unique. + ad_utility::HashSet deduplicatedGroupByVars; + auto processVariable = [this, + &deduplicatedGroupByVars](const Variable& groupKey) { checkVariableIsVisible(groupKey, "GROUP BY"); - - _groupByVariables.push_back(groupKey); + if (deduplicatedGroupByVars.insert(groupKey).second) { + _groupByVariables.push_back(groupKey); + } }; ad_utility::HashSet variablesDefinedInGroupBy; auto processExpression = - [this, &variablesDefinedInGroupBy]( - sparqlExpression::SparqlExpressionPimpl groupKey) { + [this, &variablesDefinedInGroupBy, + &processVariable](sparqlExpression::SparqlExpressionPimpl groupKey) { + // Handle the case of redundant braces around a variable, e.g. `GROUP BY + // (?x)`, which is parsed as an expression by the parser. + if (auto var = groupKey.getVariableOrNullopt(); var.has_value()) { + processVariable(var.value()); + return; + } checkUsedVariablesAreVisible(groupKey, "GROUP BY", variablesDefinedInGroupBy, " or previously in the same GROUP BY"); diff --git a/test/QueryPlannerTest.cpp b/test/QueryPlannerTest.cpp index 92ae4a32c0..90462f3cc3 100644 --- a/test/QueryPlannerTest.cpp +++ b/test/QueryPlannerTest.cpp @@ -2904,3 +2904,12 @@ TEST(QueryPlanner, Describe) { "?y", "

", "", {}, ad_utility::HashSet{""}))); } + +// ____________________________________________________________________________ +TEST(QueryPlanner, GroupByRedundanteParensAndVariables) { + auto matcher = h::GroupBy({Variable{"?x"}}, {}, + h::IndexScanFromStrings("?x", "?y", "?z")); + h::expect("SELECT ?x { ?x ?y ?z} GROUP BY (?x)", matcher); + h::expect("SELECT ?x { ?x ?y ?z} GROUP BY ?x ?x", matcher); + h::expect("SELECT ?x { ?x ?y ?z} GROUP BY ?x ?x (?x)", matcher); +} From e422852b94cd1ecf97f9b70d5b30cf4a938a9dc1 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Tue, 17 Dec 2024 00:18:31 +0100 Subject: [PATCH 08/14] Fix bug for materialized join results with lazy input with local vocabs (#1684) This fixes a subtle bug which led to a segmentation fault in the following scenario: 1. The result of a join is fully materialized, but at least one of its inputs is lazy 2. The blocks of the unique input contain local vocabs with unique entries (not shared between the blocks) 3. There is at least one block in the lazy input that has no matching row in the other input, and therefore its local vocab is not used 4. There is at least one block before the block from item 3 that is needed (including its local vocab) Previously, the local vocabs from all blocks that fulfill item 3 were not part of the input and caused segfaults, when the corresponding local vocab pointer was dereferenced. Fixes #1679 --- src/engine/AddCombinedRowToTable.h | 9 ++++++-- test/AddCombinedRowToTableTest.cpp | 36 ++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/engine/AddCombinedRowToTable.h b/src/engine/AddCombinedRowToTable.h index c43e298a88..45f51fb527 100644 --- a/src/engine/AddCombinedRowToTable.h +++ b/src/engine/AddCombinedRowToTable.h @@ -155,8 +155,13 @@ class AddCombinedRowToIdTable { if (nextIndex_ != 0) { AD_CORRECTNESS_CHECK(inputLeftAndRight_.has_value()); flush(); - } else { - // Clear vocab when no rows were written. + } else if (resultTable_.empty()) { + // Clear local vocab when no rows were written. + // + // TODO This is a conservative approach. We could + // optimize this case (clear the local vocab more often, but still + // correctly) by considering the situation after all the relevant inputs + // have been processed. mergedVocab_ = LocalVocab{}; } } diff --git a/test/AddCombinedRowToTableTest.cpp b/test/AddCombinedRowToTableTest.cpp index 2c409a830b..4bf72adcf9 100644 --- a/test/AddCombinedRowToTableTest.cpp +++ b/test/AddCombinedRowToTableTest.cpp @@ -323,3 +323,39 @@ TEST(AddCombinedRowToTable, verifyLocalVocabIsRetainedWhenNotMoving) { EXPECT_TRUE(vocabContainsString(localVocab, "b")); EXPECT_THAT(localVocab.getAllWordsForTesting(), ::testing::SizeIs(2)); } + +// _____________________________________________________________________________ +TEST(AddCombinedRowToTable, localVocabIsOnlyClearedWhenLegal) { + auto outputTable = makeIdTableFromVector({}); + outputTable.setNumColumns(3); + ad_utility::AddCombinedRowToIdTable adder{ + 1, std::move(outputTable), + std::make_shared>(), 1}; + + IdTableWithVocab input1{makeIdTableFromVector({{0, 1}}), + createVocabWithSingleString("a")}; + IdTableWithVocab input2{makeIdTableFromVector({{0, 2}}), + createVocabWithSingleString("b")}; + + adder.setInput(input1, input2); + adder.addRow(0, 0); + IdTableWithVocab input3{makeIdTableFromVector({{3, 1}}), + createVocabWithSingleString("c")}; + IdTableWithVocab input4{makeIdTableFromVector({{3, 2}}), + createVocabWithSingleString("d")}; + // NOTE: This seemingly redundant call to `setInput` is important, as it tests + // a previous bug: Each call to `setInput` implicitly also calls `flush` and + // also possibly clears the local vocab if it is not used anymore. In this + // case however we may not clear the local vocab, as the result of the + // previous calls to `addRow` has not yet been extracted. + adder.setInput(input1, input2); + adder.setInput(input3, input4); + adder.addRow(0, 0); + auto localVocab = adder.localVocab().clone(); + + EXPECT_TRUE(vocabContainsString(localVocab, "a")); + EXPECT_TRUE(vocabContainsString(localVocab, "b")); + EXPECT_TRUE(vocabContainsString(localVocab, "c")); + EXPECT_TRUE(vocabContainsString(localVocab, "d")); + EXPECT_THAT(localVocab.getAllWordsForTesting(), ::testing::SizeIs(4)); +} From 332d8e5be25ab6b7cbd03b30e3f3a0d198056025 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Tue, 17 Dec 2024 00:36:03 +0100 Subject: [PATCH 09/14] Add `InputRange...` classes as a replacement for `cppcoro::generator` (#1678) The new classes (called `InputRange...`) and their requirements can be found in `src/util/iterators.h`. They can be used to replace our many uses of `cppcoro::generator`. For an example, see `src/engine/Result.h`, where `using LazyResult = ad_utility::InputRangeTypeErased` is defined as a replacement for `using Generator = cppcoro::generator`, and the many places where now `LazyResult` is used instead of `Generator`. Other replacements will take a little more work. --- src/engine/CMakeLists.txt | 2 +- src/engine/Distinct.cpp | 2 +- src/engine/Distinct.h | 3 +- src/engine/GroupBy.cpp | 2 - src/engine/IndexScan.cpp | 6 +- src/engine/IndexScan.h | 2 +- src/engine/Join.cpp | 6 +- src/engine/Result.cpp | 10 +- src/engine/Result.h | 13 +- src/engine/Service.cpp | 4 +- src/parser/ParsedQuery.cpp | 2 +- src/util/BackgroundStxxlSorter.h | 2 +- src/util/Generators.h | 5 +- src/util/Iterators.h | 218 ++++++++++++++++++++++++++++++- src/util/TypeTraits.h | 3 +- src/util/Views.h | 108 +++++++++------ test/CMakeLists.txt | 39 ++++-- test/FilterTest.cpp | 2 +- test/IteratorTest.cpp | 110 ++++++++++++++++ test/OperationTest.cpp | 5 +- test/ResultTest.cpp | 2 +- test/ViewsTest.cpp | 2 +- test/engine/DistinctTest.cpp | 2 +- test/engine/IndexScanTest.cpp | 23 ++-- test/util/IdTableHelpers.cpp | 2 +- test/util/IdTableHelpers.h | 2 +- 26 files changed, 478 insertions(+), 99 deletions(-) diff --git a/src/engine/CMakeLists.txt b/src/engine/CMakeLists.txt index 5789f50023..be22a64d5d 100644 --- a/src/engine/CMakeLists.txt +++ b/src/engine/CMakeLists.txt @@ -1,6 +1,6 @@ add_subdirectory(sparqlExpressions) add_library(SortPerformanceEstimator SortPerformanceEstimator.cpp) -qlever_target_link_libraries(SortPerformanceEstimator) +qlever_target_link_libraries(SortPerformanceEstimator parser) add_library(engine Engine.cpp QueryExecutionTree.cpp Operation.cpp Result.cpp LocalVocab.cpp IndexScan.cpp Join.cpp Sort.cpp diff --git a/src/engine/Distinct.cpp b/src/engine/Distinct.cpp index a3047569a1..2938bc30e6 100644 --- a/src/engine/Distinct.cpp +++ b/src/engine/Distinct.cpp @@ -35,7 +35,7 @@ VariableToColumnMap Distinct::computeVariableToColumnMap() const { // _____________________________________________________________________________ template -Result::Generator Distinct::lazyDistinct(Result::Generator input, +Result::Generator Distinct::lazyDistinct(Result::LazyResult input, bool yieldOnce) const { IdTable aggregateTable{subtree_->getResultWidth(), allocator()}; LocalVocab aggregateVocab{}; diff --git a/src/engine/Distinct.h b/src/engine/Distinct.h index 0fda43fd50..dba3e60b15 100644 --- a/src/engine/Distinct.h +++ b/src/engine/Distinct.h @@ -64,7 +64,8 @@ class Distinct : public Operation { // if every `IdTable` from `input` should yield it's own `IdTable` or if all // of them should get aggregated into a single big `IdTable`. template - Result::Generator lazyDistinct(Result::Generator input, bool yieldOnce) const; + Result::Generator lazyDistinct(Result::LazyResult input, + bool yieldOnce) const; // Removes all duplicates from input with regards to the columns // in keepIndices. The input needs to be sorted on the keep indices, diff --git a/src/engine/GroupBy.cpp b/src/engine/GroupBy.cpp index 1ae50b1b79..9dde7353a3 100644 --- a/src/engine/GroupBy.cpp +++ b/src/engine/GroupBy.cpp @@ -177,8 +177,6 @@ uint64_t GroupBy::getSizeEstimateBeforeLimit() { return _subtree->getMultiplicity(_subtree->getVariableColumn(var)); }; - // TODO Once we can use `std::views` this can be solved - // more elegantly. float minMultiplicity = ql::ranges::min( _groupByVariables | ql::views::transform(varToMultiplicity)); return _subtree->getSizeEstimate() / minMultiplicity; diff --git a/src/engine/IndexScan.cpp b/src/engine/IndexScan.cpp index 0cd735863d..faa9cda3c5 100644 --- a/src/engine/IndexScan.cpp +++ b/src/engine/IndexScan.cpp @@ -489,13 +489,13 @@ void IndexScan::updateRuntimeInfoForLazyScan(const LazyScanMetadata& metadata) { // resulting from the generator. struct IndexScan::SharedGeneratorState { // The generator that yields the tables to be joined with the index scan. - Result::Generator generator_; + Result::LazyResult generator_; // The column index of the join column in the tables yielded by the generator. ColumnIndex joinColumn_; // Metadata and blocks of this index scan. Permutation::MetadataAndBlocks metaBlocks_; // The iterator of the generator that is currently being consumed. - std::optional iterator_ = std::nullopt; + std::optional iterator_ = std::nullopt; // Values returned by the generator that have not been re-yielded yet. // Typically we expect only 3 or less values to be prefetched (this is an // implementation detail of `BlockZipperJoinImpl`). @@ -648,7 +648,7 @@ Result::Generator IndexScan::createPrefilteredIndexScanSide( // _____________________________________________________________________________ std::pair IndexScan::prefilterTables( - Result::Generator input, ColumnIndex joinColumn) { + Result::LazyResult input, ColumnIndex joinColumn) { AD_CORRECTNESS_CHECK(numVariables_ <= 3 && numVariables_ > 0); auto metaBlocks = getMetadataForScan(); diff --git a/src/engine/IndexScan.h b/src/engine/IndexScan.h index d778260efe..72d377cfc3 100644 --- a/src/engine/IndexScan.h +++ b/src/engine/IndexScan.h @@ -106,7 +106,7 @@ class IndexScan final : public Operation { // there are undef values, the second generator represents the full index // scan. std::pair prefilterTables( - Result::Generator input, ColumnIndex joinColumn); + Result::LazyResult input, ColumnIndex joinColumn); private: // Implementation detail that allows to consume a generator from two other diff --git a/src/engine/Join.cpp b/src/engine/Join.cpp index b7b25a8e74..9956ff3e09 100644 --- a/src/engine/Join.cpp +++ b/src/engine/Join.cpp @@ -40,8 +40,10 @@ using LazyInputView = // Convert a `generator` to a `generator` for // more efficient access in the join columns below and apply the given // permutation to each table. -LazyInputView convertGenerator(Result::Generator gen, - OptionalPermutation permutation = {}) { +CPP_template(typename Input)( + requires ad_utility::SameAsAny) LazyInputView + convertGenerator(Input gen, OptionalPermutation permutation = {}) { for (auto& [table, localVocab] : gen) { applyPermutation(table, permutation); // Make sure to actually move the table into the wrapper so that the tables diff --git a/src/engine/Result.cpp b/src/engine/Result.cpp index a1cfa10583..3b476777bb 100644 --- a/src/engine/Result.cpp +++ b/src/engine/Result.cpp @@ -124,7 +124,7 @@ void Result::applyLimitOffset( limitOffset); limitTimeCallback(limitTimer.msecs(), idTable()); } else { - auto generator = [](Generator original, LimitOffsetClause limitOffset, + auto generator = [](LazyResult original, LimitOffsetClause limitOffset, auto limitTimeCallback) -> Generator { if (limitOffset._limit.value_or(1) == 0) { co_return; @@ -160,7 +160,7 @@ void Result::assertThatLimitWasRespected(const LimitOffsetClause& limitOffset) { auto limit = limitOffset._limit; AD_CONTRACT_CHECK(!limit.has_value() || numRows <= limit.value()); } else { - auto generator = [](Generator original, + auto generator = [](LazyResult original, LimitOffsetClause limitOffset) -> Generator { auto limit = limitOffset._limit; uint64_t elementCount = 0; @@ -192,7 +192,7 @@ void Result::checkDefinedness(const VariableToColumnMap& varColMap) { AD_EXPENSIVE_CHECK(performCheck( varColMap, std::get(data_).idTable_)); } else { - auto generator = [](Generator original, + auto generator = [](LazyResult original, [[maybe_unused]] VariableToColumnMap varColMap, [[maybe_unused]] auto performCheck) -> Generator { for (IdTableVocabPair& pair : original) { @@ -212,7 +212,7 @@ void Result::runOnNewChunkComputed( onNewChunk, std::function onGeneratorFinished) { AD_CONTRACT_CHECK(!isFullyMaterialized()); - auto generator = [](Generator original, auto onNewChunk, + auto generator = [](LazyResult original, auto onNewChunk, auto onGeneratorFinished) -> Generator { // Call this within destructor to make sure it is also called when an // operation stops iterating before reaching the end. @@ -254,7 +254,7 @@ const IdTable& Result::idTable() const { } // _____________________________________________________________________________ -Result::Generator& Result::idTables() const { +Result::LazyResult& Result::idTables() const { AD_CONTRACT_CHECK(!isFullyMaterialized()); const auto& container = std::get(data_); AD_CONTRACT_CHECK(!container.consumed_->exchange(true)); diff --git a/src/engine/Result.h b/src/engine/Result.h index af23432961..10b7364a3e 100644 --- a/src/engine/Result.h +++ b/src/engine/Result.h @@ -33,16 +33,23 @@ class Result { : idTable_{std::move(idTable)}, localVocab_{std::move(localVocab)} {} }; + // The current implementation of (most of the) lazy results. Will be replaced + // in the future to make QLever compatible with C++17 again. using Generator = cppcoro::generator; + // The lazy result type that is actually stored. It is type-erased and allows + // explicit conversion from the `Generator` above. + using LazyResult = ad_utility::InputRangeTypeErased; private: // Needs to be mutable in order to be consumable from a const result. struct GenContainer { - mutable Generator generator_; + mutable LazyResult generator_; mutable std::unique_ptr consumed_ = std::make_unique(false); - explicit GenContainer(Generator generator) + explicit GenContainer(LazyResult generator) : generator_{std::move(generator)} {} + explicit GenContainer(Generator generator) + : generator_{Generator{std::move(generator)}} {} }; using LocalVocabPtr = std::shared_ptr; @@ -155,7 +162,7 @@ class Result { // Access to the underlying `IdTable`s. Throw an `ad_utility::Exception` // if the underlying `data_` member holds the wrong variant. - Generator& idTables() const; + LazyResult& idTables() const; // Const access to the columns by which the `idTable()` is sorted. const std::vector& sortedBy() const { return sortedBy_; } diff --git a/src/engine/Service.cpp b/src/engine/Service.cpp index 81df6be64c..21338be71b 100644 --- a/src/engine/Service.cpp +++ b/src/engine/Service.cpp @@ -564,8 +564,8 @@ void Service::precomputeSiblingResult(std::shared_ptr left, // Creates a `Result::Generator` from partially materialized result data. auto partialResultGenerator = [](std::vector pairs, - Result::Generator prevGenerator, - Result::Generator::iterator it) -> Result::Generator { + Result::LazyResult prevGenerator, + std::ranges::iterator_t it) -> Result::Generator { for (auto& pair : pairs) { co_yield pair; } diff --git a/src/parser/ParsedQuery.cpp b/src/parser/ParsedQuery.cpp index 6cb844071b..c4326dfc92 100644 --- a/src/parser/ParsedQuery.cpp +++ b/src/parser/ParsedQuery.cpp @@ -265,7 +265,7 @@ void ParsedQuery::GraphPattern::addLanguageFilter(const Variable& variable, std::vector matchingTriples; using BasicPattern = parsedQuery::BasicGraphPattern; namespace ad = ad_utility; - namespace stdv = std::views; + namespace stdv = ql::views; for (BasicPattern* basicPattern : _graphPatterns | stdv::transform(ad::getIf) | stdv::filter(ad::toBool)) { diff --git a/src/util/BackgroundStxxlSorter.h b/src/util/BackgroundStxxlSorter.h index d142f81a3c..8ab866bfff 100644 --- a/src/util/BackgroundStxxlSorter.h +++ b/src/util/BackgroundStxxlSorter.h @@ -102,7 +102,7 @@ class BackgroundStxxlSorter { /// Transition from the input phase, where `push()` may be called, to the /// output phase and return a generator that yields the sorted elements. This /// function may be called exactly once. - [[nodiscard]] cppcoro::generator sortedView() { + [[nodiscard]] auto sortedView() { setupSort(); return bufferedAsyncView(outputGeneratorImpl(), _numElementsInRun); } diff --git a/src/util/Generators.h b/src/util/Generators.h index 9157c48359..db232d0188 100644 --- a/src/util/Generators.h +++ b/src/util/Generators.h @@ -19,9 +19,10 @@ namespace ad_utility { // returns false. If the `aggregator` returns false, the cached value is // discarded. If the cached value is still present once the generator is fully // consumed, `onFullyCached` is called with the cached value. -template +template > cppcoro::generator wrapGeneratorWithCache( - cppcoro::generator generator, + InputRange generator, InvocableWithExactReturnType&, const T&> auto aggregator, InvocableWithExactReturnType auto onFullyCached) { diff --git a/src/util/Iterators.h b/src/util/Iterators.h index a837d6e39a..1337b63e27 100644 --- a/src/util/Iterators.h +++ b/src/util/Iterators.h @@ -10,6 +10,7 @@ #include #include "util/Enums.h" +#include "util/TypeTraits.h" namespace ad_utility { @@ -49,7 +50,7 @@ class IteratorForAccessOperator { using iterator_category = std::random_access_iterator_tag; using difference_type = int64_t; using index_type = uint64_t; - // It is possible to explicitly specify the `value_type` and `reference_type` + // It is possible to explicitly specify the `value_type` and `reference` // if they differ from the defaults. For an example, see the `IdTable` class // which uses a proxy type as its `reference`. using value_type = std::conditional_t< @@ -182,6 +183,221 @@ auto makeForwardingIterator(It iterator) { } } +// This CRTP-Mixin can be used to add iterators to a simple state-machine like +// class, s.t. it behaves like an `InputRange`. The derived class needs the +// following functions: `start()`, `isFinished()`, `get()` , `next()`. +// * `void start()` -> called when `begin()` is called to allow for deferred +// initialization. After calling `start()` either `get()` must return the first +// element, or `isFinished()` must return true ( for an empty range). +// * `bool isFinished()` -> has to return true if there are no more values, and +// calls to `get()` are thus impossible. +// * `reference get()` -> get the current value (typically as a reference). +// * `void next()` advance to the next value. After calling `next()` either +// `isFinished()` must be true, or `get()` must return the next value. +template +class InputRangeMixin { + public: + // Cast `this` to the derived class for easier access. + Derived& derived() { return static_cast(*this); } + const Derived& derived() const { return static_cast(*this); } + + // A simple sentinel which is returned by the call to `end()`. + struct Sentinel {}; + + // The iterator class. + class Iterator { + public: + using iterator_category = std::input_iterator_tag; + using difference_type = std::int64_t; + using reference = decltype(std::declval().get()); + using value_type = std::remove_reference_t; + using pointer = value_type*; + InputRangeMixin* mixin_ = nullptr; + + public: + Iterator() = default; + explicit Iterator(InputRangeMixin* mixin) : mixin_{mixin} {} + Iterator& operator++() { + mixin_->derived().next(); + return *this; + } + + // Needed for the `range` concept. + void operator++(int) { (void)operator++(); } + + decltype(auto) operator*() { return mixin_->derived().get(); } + decltype(auto) operator*() const { return mixin_->derived().get(); } + decltype(auto) operator->() { return std::addressof(operator*()); } + decltype(auto) operator->() const { return std::addressof(operator*()); } + + // The comparison `it == end()` just queries `isFinished()` , so an empty + // `Sentinel` suffices. + friend bool operator==(const Iterator& it, Sentinel) { + return it.mixin_->derived().isFinished(); + } + friend bool operator==(Sentinel s, const Iterator& it) { return it == s; } + friend bool operator!=(const Iterator& it, Sentinel s) { + return !(it == s); + } + friend bool operator!=(Sentinel s, const Iterator& it) { + return !(it == s); + } + }; + + public: + // The only functions needed to make this a proper range: `begin()` and + // `end()`. + Iterator begin() { + derived().start(); + return Iterator{this}; + } + Sentinel end() const { return {}; }; +}; + +// A similar mixin to the above, with slightly different characteristics: +// 1. It only requires a single function `std::optional get() +// override` +// 2. It uses simple inheritance with virtual functions, which allows for type +// erasure of different ranges with the same `ValueType`. +// 3. While the interface is simpler (see 1.+2.) each step in iterating is a +// little bit more complex, as the mixin has to store the value. This might be +// less efficient for very simple generators, because the compiler might be able +// to optimize this mixin as well as the one above. +template +class InputRangeFromGet { + public: + using Storage = std::optional; + Storage storage_ = std::nullopt; + + private: + // The single virtual function which has to be overloaded. `std::nullopt` + // means that there will be no more values. + virtual Storage get() = 0; + + public: + virtual ~InputRangeFromGet() = default; + + // Get the next value and store it. + void getNextAndStore() { storage_ = get(); } + + struct Sentinel {}; + class Iterator { + public: + using iterator_category = std::input_iterator_tag; + using difference_type = std::int64_t; + using value_type = typename InputRangeFromGet::Storage::value_type; + using pointer = value_type*; + using reference = std::add_lvalue_reference_t; + using const_reference = std::add_const_t; + InputRangeFromGet* mixin_ = nullptr; + + public: + Iterator() = default; + explicit Iterator(InputRangeFromGet* mixin) : mixin_{mixin} {} + Iterator& operator++() { + mixin_->getNextAndStore(); + return *this; + } + + // Needed for the `range` concept. + void operator++(int) { (void)operator++(); } + + reference operator*() { return mixin_->storage_.value(); } + const_reference operator*() const { return mixin_->storage_.value(); } + decltype(auto) operator->() { return std::addressof(operator*()); } + decltype(auto) operator->() const { return std::addressof(operator*()); } + + friend bool operator==(const Iterator& it, Sentinel) { + return !it.mixin_->storage_.has_value(); + } + friend bool operator==(Sentinel s, const Iterator& it) { return it == s; } + friend bool operator!=(const Iterator& it, Sentinel s) { + return !(it == s); + } + friend bool operator!=(Sentinel s, const Iterator& it) { + return !(it == s); + } + }; + + Iterator begin() { + getNextAndStore(); + return Iterator{this}; + } + Sentinel end() const { return {}; }; +}; + +// This class takes an arbitrary input range, and turns it into a class that +// inherits from `InputRangeFromGet` (see above). While this adds a layer of +// indirection, it makes type erasure between input ranges with the same value +// type very simple. +template +class RangeToInputRangeFromGet + : public InputRangeFromGet> { + Range range_; + using Iterator = ql::ranges::iterator_t; + std::optional iterator_ = std::nullopt; + bool isDone() { return iterator_ == ql::ranges::end(range_); } + + public: + explicit RangeToInputRangeFromGet(Range range) : range_{std::move(range)} {} + + // As we use the `InputRangeOptionalMixin`, we only have to override the + // single `get()` method. + std::optional> get() override { + if (!iterator_.has_value()) { + // For the very first value we have to call `begin()`. + iterator_ = ql::ranges::begin(range_); + if (isDone()) { + return std::nullopt; + } + } else { + // Not the first value, so we have to advance the iterator. + if (isDone()) { + return std::nullopt; + } + ++iterator_.value(); + } + + // We now have advanced the iterator to the next value, so we can return it + // if existing. + if (isDone()) { + return std::nullopt; + } + return std::move(*iterator_.value()); + } +}; + +// A simple type-erased input range (that is, one class for *any* input range +// with the given `ValueType`). It internally uses the `InputRangeOptionalMixin` +// from above as an implementation detail. +template +class InputRangeTypeErased { + // Unique (and therefore owning) pointer to the virtual base class. + std::unique_ptr> impl_; + + public: + // Constructor for ranges that directly inherit from + // `InputRangeOptionalMixin`. + template + requires std::is_base_of_v, Range> + explicit InputRangeTypeErased(Range range) + : impl_{std::make_unique(std::move(range))} {} + + // Constructor for all other ranges. We first pass them through the + // `InputRangeToOptional` class from above to make it compatible with the base + // class. + template + requires(!std::is_base_of_v, Range> && + ql::ranges::range && + std::same_as, ValueType>) + explicit InputRangeTypeErased(Range range) + : impl_{std::make_unique>( + std::move(range))} {} + + decltype(auto) begin() { return impl_->begin(); } + decltype(auto) end() { return impl_->end(); } + using iterator = typename InputRangeFromGet::Iterator; +}; } // namespace ad_utility #endif // QLEVER_ITERATORS_H diff --git a/src/util/TypeTraits.h b/src/util/TypeTraits.h index 07703494f7..7d39e4aa70 100644 --- a/src/util/TypeTraits.h +++ b/src/util/TypeTraits.h @@ -12,6 +12,7 @@ #include #include +#include "backports/algorithm.h" #include "util/Forward.h" namespace ad_utility { @@ -135,7 +136,7 @@ concept SimilarToAny = (... || isSimilar); /// True iff `T` is the same as any of the `Ts...`. template -concept SameAsAny = (... || std::same_as); +concept SameAsAny = (... || ql::concepts::same_as); /* The implementation for `SimilarToAnyTypeIn` and `SameAsAnyTypeIn` (see below diff --git a/src/util/Views.h b/src/util/Views.h index 4666a28228..74e6eec14c 100644 --- a/src/util/Views.h +++ b/src/util/Views.h @@ -5,55 +5,20 @@ #pragma once #include -#include #include #include "backports/algorithm.h" #include "backports/concepts.h" #include "util/Generator.h" +#include "util/Iterators.h" #include "util/Log.h" namespace ad_utility { -/// Takes a input-iterable and yields the elements of that view (no visible -/// effect). The iteration over the input view is done on a separate thread with -/// a buffer size of `blockSize`. This might speed up the computation when the -/// values of the input view are expensive to compute. -template -cppcoro::generator bufferedAsyncView( - View view, uint64_t blockSize) { - using value_type = typename View::value_type; - auto it = view.begin(); - auto end = view.end(); - auto getNextBlock = [&it, &end, blockSize] { - std::vector buffer; - buffer.reserve(blockSize); - size_t i = 0; - while (i < blockSize && it != end) { - buffer.push_back(*it); - ++it; - ++i; - } - return buffer; - }; - - auto block = getNextBlock(); - auto future = std::async(std::launch::async, getNextBlock); - while (true) { - for (auto& element : block) { - co_yield element; - } - block = future.get(); - if (block.empty()) { - co_return; - } - future = std::async(std::launch::async, getNextBlock); - } -} - /// Takes a view and yields the elements of the same view, but skips over /// consecutive duplicates. -template +template > cppcoro::generator uniqueView(SortedView view) { size_t numInputs = 0; size_t numUnique = 0; @@ -115,7 +80,7 @@ cppcoro::generator uniqueBlockView( } // A view that owns its underlying storage. It is a replacement for -// `std::ranges::owning_view` which is not yet supported by `GCC 11` and +// `ranges::owning_view` which is not yet supported by `GCC 11` and // `range-v3`. The implementation is taken from libstdc++-13. The additional // optional `supportsConst` argument explicitly disables const iteration for // this view when set to false, see `OwningViewNoConst` below for details. @@ -225,7 +190,7 @@ CPP_concept can_ref_view = CPP_requires_ref(can_ref_view, Range); // implementations. template constexpr auto allView(Range&& range) { - if constexpr (std::ranges::view>) { + if constexpr (ql::ranges::view>) { return AD_FWD(range); } else if constexpr (detail::can_ref_view) { return ql::ranges::ref_view{AD_FWD(range)}; @@ -234,6 +199,69 @@ constexpr auto allView(Range&& range) { } } +namespace detail { +// The implementation of `bufferedAsyncView` (see below). It yields its result +// in blocks. +template +struct BufferedAsyncView : InputRangeMixin> { + View view_; + uint64_t blockSize_; + bool finished_ = false; + + explicit BufferedAsyncView(View view, uint64_t blockSize) + : view_{std::move(view)}, blockSize_{blockSize} { + AD_CONTRACT_CHECK(blockSize_ > 0); + } + + ql::ranges::iterator_t it_; + ql::ranges::sentinel_t end_ = ql::ranges::end(view_); + using value_type = ql::ranges::range_value_t; + std::future> future_; + + std::vector buffer_; + std::vector getNextBlock() { + std::vector buffer; + buffer.reserve(blockSize_); + size_t i = 0; + while (i < blockSize_ && it_ != end_) { + buffer.push_back(*it_); + ++it_; + ++i; + } + return buffer; + }; + + void start() { + it_ = view_.begin(); + buffer_ = getNextBlock(); + finished_ = buffer_.empty(); + future_ = + std::async(std::launch::async, [this]() { return getNextBlock(); }); + } + bool isFinished() { return finished_; } + auto& get() { return buffer_; } + const auto& get() const { return buffer_; } + + void next() { + buffer_ = future_.get(); + finished_ = buffer_.empty(); + future_ = + std::async(std::launch::async, [this]() { return getNextBlock(); }); + } +}; +} // namespace detail + +/// Takes a input-iterable and yields the elements of that view (no visible +/// effect). The iteration over the input view is done on a separate thread with +/// a buffer size of `blockSize`. This might speed up the computation when the +/// values of the input view are expensive to compute. +/// +template +auto bufferedAsyncView(View view, uint64_t blockSize) { + return ql::views::join( + allView(detail::BufferedAsyncView{std::move(view), blockSize})); +} + // Returns a view that contains all the values in `[0, upperBound)`, similar to // Python's `range` function. Avoids the common pitfall in `ql::views::iota` // that the count variable is only derived from the first argument. For example, diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c260f34d4e..46a83d5b7a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -7,7 +7,7 @@ add_subdirectory(util) # general test utilities and all libraries that are specified as additional # arguments. function(linkTest basename) - qlever_target_link_libraries(${basename} ${ARGN} GTest::gtest GTest::gmock_main testUtil ${CMAKE_THREAD_LIBS_INIT}) + qlever_target_link_libraries(${basename} ${ARGN} GTest::gtest GTest::gmock_main ${CMAKE_THREAD_LIBS_INIT}) endfunction() # Add the executable ${basename} that is compiled from the source file @@ -47,15 +47,12 @@ else () message(STATUS "The tests are split over multiple binaries") endif () -# Usage: `addAndLinkTest(basename, [additionalLibraries...]` -# Add a GTest/GMock test case that is called `basename` and compiled from a file called -# `basename.cpp`. All tests are linked against `gmock_main` and the threading library. -# additional libraries against which the test case has to be linked can be specified as -# additional arguments after the `basename` -function(addLinkAndDiscoverTest basename) + +# The implementation of `addLinkAndDiscoverTest` and `addLinkandDiscoverTestNoLibs` below. +function(addLinkAndDiscoverTestImpl basename) if (SINGLE_TEST_BINARY) target_sources(QLeverAllUnitTestsMain PUBLIC ${basename}.cpp) - qlever_target_link_libraries(QLeverAllUnitTestsMain ${ARGN}) + qlever_target_link_libraries(QLeverAllUnitTestsMain ${ARGN} ) else () addTest(${basename}) linkAndDiscoverTest(${basename} ${ARGN}) @@ -63,6 +60,24 @@ function(addLinkAndDiscoverTest basename) endfunction() +# Usage: `addAndLinkTest[NoLibs](basename, [additionalLibraries...]` +# Add a GTest/GMock test case that is called `basename` and compiled from a file called +# `basename.cpp`. All tests are linked against `gmock_main` and the threading library. +# additional libraries against which the test case has to be linked can be specified as +# additional arguments after the `basename` + +# This function links the test against `testUtil` (basically all of QLever). +function(addLinkAndDiscoverTest basename) + addLinkAndDiscoverTestImpl(${basename} ${ARGN} testUtil) +endfunction() + +# This function links only against Gtest + the explicitly specified libraries. +# It can be used for tests of standalone utils that don't require the rest of QLever. +function(addLinkAndDiscoverTestNoLibs basename) + addLinkAndDiscoverTestImpl(${basename} ${ARGN}) +endfunction() + + # Add a GTest/GMock test case that is called `basename` and compiled from a file called # `basename.cpp`. All tests are linked against `gmock_main` and the threading library. # In contrast to `addLinkAndDiscoverTest` this doesn't let ctest run all subtests individually, @@ -73,7 +88,7 @@ function(addLinkAndRunAsSingleTest basename) qlever_target_link_libraries(QLeverAllUnitTestsMain ${ARGN}) else () addTest(${basename}) - linkTest(${basename} ${ARGN}) + linkTest(${basename} testUtil ${ARGN}) add_test(NAME ${basename} COMMAND ${basename}) endif () @@ -89,7 +104,7 @@ function(addLinkAndDiscoverTestSerial basename) qlever_target_link_libraries(QLeverAllUnitTestsMain ${ARGN}) else () addTest(${basename}) - linkAndDiscoverTestSerial(${basename} ${ARGN}) + linkAndDiscoverTestSerial(${basename} testUtil ${ARGN}) endif () endfunction() @@ -241,14 +256,14 @@ addLinkAndDiscoverTest(MilestoneIdTest) addLinkAndDiscoverTest(VocabularyTest index) -addLinkAndDiscoverTest(IteratorTest) +addLinkAndDiscoverTestNoLibs(IteratorTest) # Stxxl currently always uses a file ./-stxxl.disk for all indices, which # makes it impossible to run the test cases for the Index class in parallel. # TODO fix this addLinkAndDiscoverTestSerial(BackgroundStxxlSorterTest ${STXXL_LIBRARIES}) -addLinkAndDiscoverTest(ViewsTest) +addLinkAndDiscoverTestNoLibs(ViewsTest) addLinkAndDiscoverTest(ForwardTest) diff --git a/test/FilterTest.cpp b/test/FilterTest.cpp index e6586061ea..7df2617c3f 100644 --- a/test/FilterTest.cpp +++ b/test/FilterTest.cpp @@ -21,7 +21,7 @@ namespace { ValueId asBool(bool value) { return Id::makeFromBool(value); } // Convert a generator to a vector for easier comparison in assertions -std::vector toVector(Result::Generator generator) { +std::vector toVector(Result::LazyResult generator) { std::vector result; for (auto& pair : generator) { // IMPORTANT: The `LocalVocab` contained in the pair will be destroyed at diff --git a/test/IteratorTest.cpp b/test/IteratorTest.cpp index af7de3f7ea..9ee5118e96 100644 --- a/test/IteratorTest.cpp +++ b/test/IteratorTest.cpp @@ -8,6 +8,7 @@ #include #include "../src/util/Iterators.h" +#include "backports/algorithm.h" auto testIterator = [](const auto& input, auto begin, auto end) { auto it = begin; @@ -95,3 +96,112 @@ TEST(Iterator, makeForwardingIterator) { ASSERT_EQ(1u, vector.size()); ASSERT_TRUE(vector[0].empty()); } + +namespace { +template +// This function tests a view that behaves like `ql::views::iota`. +// The argument `makeIotaRange` is given a lower bound (size_t, `0` if not +// specified) and an upper bound (`optional`, unlimited (nullopt) if not +// specified) and must return a `ql::ranges::input_range` that yields the +// elements in the range `(lower, upper]`. +void testIota(MakeIotaRange makeIotaRange) { + size_t sum = 0; + // Test manual iteration. + for (auto s : makeIotaRange(0, 5)) { + sum += s; + } + EXPECT_EQ(sum, 10); + + // Check that the range is an input range, but fulfills none of the stricter + // categories. + auto iota = makeIotaRange(); + using Iota = decltype(iota); + static_assert(ql::ranges::input_range); + static_assert(!ql::ranges::forward_range); + + // Test the interaction with the `ql::views` and `ql::ranges` machinery. + auto view = iota | ql::views::drop(3) | ql::views::take(7); + static_assert(ql::ranges::input_range); + sum = 0; + auto add = [&sum](auto val) { sum += val; }; + ql::ranges::for_each(view, add); + + // 42 == 3 + 4 + ... + 9 + EXPECT_EQ(sum, 42); +} +} // namespace + +// _____________________________________________________________________________ +TEST(Iterator, InputRangeMixin) { + using namespace ad_utility; + struct Iota : InputRangeMixin { + size_t value_ = 0; + std::optional upper_; + explicit Iota(size_t lower = 0, std::optional upper = {}) + : value_{lower}, upper_{upper} {} + void start() {} + bool isFinished() const { return value_ == upper_; } + size_t get() const { return value_; } + void next() { ++value_; } + }; + + auto makeIota = [](size_t lower = 0, std::optional upper = {}) { + return Iota{lower, upper}; + }; + testIota(makeIota); +} + +//_____________________________________________________________________________ +TEST(Iterator, InputRangeFromGet) { + using namespace ad_utility; + struct Iota : InputRangeFromGet { + size_t value_ = 0; + std::optional upper_; + explicit Iota(size_t lower = 0, std::optional upper = {}) + : value_{lower}, upper_{upper} {} + std::optional get() override { + if (value_ == upper_) { + return std::nullopt; + } + return value_++; + } + }; + auto makeIota = [](size_t lower = 0, std::optional upper = {}) { + return Iota{lower, upper}; + }; + testIota(makeIota); +} +//_____________________________________________________________________________ +TEST(Iterator, InputRangeTypeErased) { + using namespace ad_utility; + struct IotaImpl : InputRangeFromGet { + size_t value_ = 0; + std::optional upper_; + explicit IotaImpl(size_t lower = 0, std::optional upper = {}) + : value_{lower}, upper_{upper} {} + std::optional get() override { + if (value_ == upper_) { + return std::nullopt; + } + return value_++; + } + }; + + using Iota = InputRangeTypeErased; + auto makeIota = [](size_t lower = 0, std::optional upper = {}) { + return Iota{IotaImpl{lower, upper}}; + }; + testIota(makeIota); + + // We can also type-erase any input range with the correct value type, in + // particular ranges and views from the standard library. + auto makeIotaFromStdIota = [](size_t lower = 0, + std::optional upper = {}) { + if (!upper.has_value()) { + return Iota{ql::views::iota(lower)}; + } else { + return Iota{ql::views::iota(lower, upper.value())}; + } + }; + testIota(makeIotaFromStdIota); +} diff --git a/test/OperationTest.cpp b/test/OperationTest.cpp index 4ad1f1313c..ec9ab35c7f 100644 --- a/test/OperationTest.cpp +++ b/test/OperationTest.cpp @@ -24,10 +24,9 @@ using Status = RuntimeInformation::Status; namespace { // Helper function to perform actions at various stages of a generator -template +template > auto expectAtEachStageOfGenerator( - cppcoro::generator generator, - std::vector> functions, + Range generator, std::vector> functions, ad_utility::source_location l = ad_utility::source_location::current()) { auto locationTrace = generateLocationTrace(l); size_t index = 0; diff --git a/test/ResultTest.cpp b/test/ResultTest.cpp index b6fd694d94..4e0deb2a53 100644 --- a/test/ResultTest.cpp +++ b/test/ResultTest.cpp @@ -55,7 +55,7 @@ std::vector getAllSubSplits(const IdTable& idTable) { } // _____________________________________________________________________________ -void consumeGenerator(Result::Generator& generator) { +void consumeGenerator(Result::LazyResult& generator) { for ([[maybe_unused]] IdTableVocabPair& _ : generator) { } } diff --git a/test/ViewsTest.cpp b/test/ViewsTest.cpp index 24fc744afb..4c1a569b96 100644 --- a/test/ViewsTest.cpp +++ b/test/ViewsTest.cpp @@ -19,7 +19,7 @@ TEST(Views, BufferedAsyncView) { for (const auto& element : view) { result.push_back(element); } - ASSERT_EQ(result, inputVector); + EXPECT_THAT(result, ::testing::ContainerEq(inputVector)); }; uint64_t numElements = 1000; diff --git a/test/engine/DistinctTest.cpp b/test/engine/DistinctTest.cpp index c20d0ba5c6..5ce219b60f 100644 --- a/test/engine/DistinctTest.cpp +++ b/test/engine/DistinctTest.cpp @@ -14,7 +14,7 @@ using V = Variable; namespace { // Convert a generator to a vector for easier comparison in assertions -std::vector toVector(Result::Generator generator) { +std::vector toVector(Result::LazyResult generator) { std::vector result; for (auto& [table, vocab] : generator) { // IMPORTANT: The `vocab` will go out of scope here, but the tests don't use diff --git a/test/engine/IndexScanTest.cpp b/test/engine/IndexScanTest.cpp index 2c526787a3..62f01647a0 100644 --- a/test/engine/IndexScanTest.cpp +++ b/test/engine/IndexScanTest.cpp @@ -21,6 +21,7 @@ using ad_utility::source_location; namespace { using Tc = TripleComponent; using Var = Variable; +using LazyResult = Result::LazyResult; using IndexPair = std::pair; @@ -866,8 +867,8 @@ TEST_P(IndexScanWithLazyJoin, prefilterTablesDoesFilterCorrectly) { co_yield p3; }; - auto [joinSideResults, scanResults] = - consumeGenerators(scan.prefilterTables(makeJoinSide(this), 0)); + auto [joinSideResults, scanResults] = consumeGenerators( + scan.prefilterTables(LazyResult{makeJoinSide(this)}, 0)); ASSERT_EQ(scanResults.size(), 2); ASSERT_EQ(joinSideResults.size(), 3); @@ -910,8 +911,8 @@ TEST_P(IndexScanWithLazyJoin, co_yield p2; }; - auto [joinSideResults, scanResults] = - consumeGenerators(scan.prefilterTables(makeJoinSide(this), 0)); + auto [joinSideResults, scanResults] = consumeGenerators( + scan.prefilterTables(LazyResult{makeJoinSide(this)}, 0)); ASSERT_EQ(scanResults.size(), 1); ASSERT_EQ(joinSideResults.size(), 2); @@ -944,7 +945,7 @@ TEST_P(IndexScanWithLazyJoin, }; auto [joinSideResults, scanResults] = - consumeGenerators(scan.prefilterTables(makeJoinSide(), 0)); + consumeGenerators(scan.prefilterTables(LazyResult{makeJoinSide()}, 0)); ASSERT_EQ(scanResults.size(), 0); ASSERT_EQ(joinSideResults.size(), 0); @@ -973,8 +974,8 @@ TEST_P(IndexScanWithLazyJoin, prefilterTablesDoesNotFilterOnUndefined) { co_yield p7; }; - auto [_, scanResults] = - consumeGenerators(scan.prefilterTables(makeJoinSide(this), 0)); + auto [_, scanResults] = consumeGenerators( + scan.prefilterTables(LazyResult{makeJoinSide(this)}, 0)); ASSERT_EQ(scanResults.size(), 3); EXPECT_TRUE(scanResults.at(0).localVocab_.empty()); @@ -1005,7 +1006,7 @@ TEST_P(IndexScanWithLazyJoin, prefilterTablesDoesNotFilterWithSingleUndefined) { }; auto [_, scanResults] = - consumeGenerators(scan.prefilterTables(makeJoinSide(), 0)); + consumeGenerators(scan.prefilterTables(LazyResult{makeJoinSide()}, 0)); ASSERT_EQ(scanResults.size(), 3); EXPECT_TRUE(scanResults.at(0).localVocab_.empty()); @@ -1036,7 +1037,7 @@ TEST_P(IndexScanWithLazyJoin, prefilterTablesWorksWithSingleEmptyTable) { }; auto [_, scanResults] = - consumeGenerators(scan.prefilterTables(makeJoinSide(), 0)); + consumeGenerators(scan.prefilterTables(LazyResult{makeJoinSide()}, 0)); ASSERT_EQ(scanResults.size(), 0); } @@ -1048,7 +1049,7 @@ TEST_P(IndexScanWithLazyJoin, prefilterTablesWorksWithEmptyGenerator) { auto makeJoinSide = []() -> Result::Generator { co_return; }; auto [_, scanResults] = - consumeGenerators(scan.prefilterTables(makeJoinSide(), 0)); + consumeGenerators(scan.prefilterTables(LazyResult{makeJoinSide()}, 0)); ASSERT_EQ(scanResults.size(), 0); } @@ -1075,7 +1076,7 @@ TEST(IndexScan, prefilterTablesWithEmptyIndexScanReturnsEmptyGenerators) { }; auto [leftGenerator, rightGenerator] = - scan.prefilterTables(makeJoinSide(), 0); + scan.prefilterTables(Result::LazyResult{makeJoinSide()}, 0); EXPECT_EQ(leftGenerator.begin(), leftGenerator.end()); EXPECT_EQ(rightGenerator.begin(), rightGenerator.end()); diff --git a/test/util/IdTableHelpers.cpp b/test/util/IdTableHelpers.cpp index 55ac6209f9..34ad9414e7 100644 --- a/test/util/IdTableHelpers.cpp +++ b/test/util/IdTableHelpers.cpp @@ -251,7 +251,7 @@ std::shared_ptr idTableToExecutionTree( // _____________________________________________________________________________ std::pair> aggregateTables( - Result::Generator generator, size_t numColumns) { + Result::LazyResult generator, size_t numColumns) { IdTable aggregateTable{numColumns, ad_utility::makeUnlimitedAllocator()}; std::vector localVocabs; for (auto& [idTable, localVocab] : generator) { diff --git a/test/util/IdTableHelpers.h b/test/util/IdTableHelpers.h index bc7035cd2f..40e2fe8213 100644 --- a/test/util/IdTableHelpers.h +++ b/test/util/IdTableHelpers.h @@ -260,4 +260,4 @@ std::shared_ptr idTableToExecutionTree( // Fully consume a given generator and store it in an `IdTable` and store the // local vocabs in a vector. std::pair> aggregateTables( - Result::Generator generator, size_t numColumns); + Result::LazyResult generator, size_t numColumns); From 97c195aaf9e249f43a6417f99ea93f7e94831f39 Mon Sep 17 00:00:00 2001 From: Johannes Kalmbach Date: Wed, 18 Dec 2024 01:29:35 +0100 Subject: [PATCH 10/14] Add macros `QL_CONCEPT_OR_NOTHING` and `QL_CONCEPT_OR_TEMPLATE` (#1686) The two macros can be used for concepts that improve the semantics and safety of the code, but are not necessary for compilation. They can be dropped or replaced by something simpler when compiling with C++17. The macro `QL_CONCEPT_OR_NOTHING` can be used for concepts that can be omitted when compiling with C++17. For example, `QL_CONCEPT_OR_NOTHING(std::view) auto x = someFunction()`. The macro `QL_CONCEPT_OR_TEMPLATE` can be used for concepts that can be replaced by `typename` when compiling with C++17. For example, `template ) T> void f(){...}`. --- benchmark/JoinAlgorithmBenchmark.cpp | 3 +- benchmark/infrastructure/Benchmark.h | 5 +- .../BenchmarkMeasurementContainer.cpp | 2 +- .../BenchmarkMeasurementContainer.h | 2 +- src/backports/algorithm.h | 13 +---- src/backports/concepts.h | 58 +++++++++++++++---- .../sparqlExpressions/LiteralExpression.h | 9 ++- src/util/ConfigManager/ConfigManager.cpp | 8 ++- src/util/ConfigManager/ConfigManager.h | 14 +++-- src/util/ConfigManager/ConfigOptionProxy.h | 8 +-- src/util/FsstCompressor.h | 17 ++++-- src/util/Iterators.h | 6 +- src/util/JoinAlgorithms/JoinAlgorithms.h | 5 +- src/util/TypeTraits.h | 4 +- src/util/Views.h | 3 +- test/ConfigOptionProxyTest.cpp | 6 +- 16 files changed, 101 insertions(+), 62 deletions(-) diff --git a/benchmark/JoinAlgorithmBenchmark.cpp b/benchmark/JoinAlgorithmBenchmark.cpp index 18945aa41c..b06202dbf4 100644 --- a/benchmark/JoinAlgorithmBenchmark.cpp +++ b/benchmark/JoinAlgorithmBenchmark.cpp @@ -1328,7 +1328,8 @@ class GeneralInterfaceImplementation : public BenchmarkInterface { */ bool addNewRowToBenchmarkTable( ResultTable* table, - const ad_utility::SameAsAny auto changingParameterValue, + const QL_CONCEPT_OR_NOTHING( + ad_utility::SameAsAny) auto changingParameterValue, ad_utility::InvocableWithExactReturnType auto stopFunction, diff --git a/benchmark/infrastructure/Benchmark.h b/benchmark/infrastructure/Benchmark.h index ae5b93d410..569fb226da 100644 --- a/benchmark/infrastructure/Benchmark.h +++ b/benchmark/infrastructure/Benchmark.h @@ -14,6 +14,7 @@ #include "../benchmark/infrastructure/BenchmarkMeasurementContainer.h" #include "../benchmark/infrastructure/BenchmarkMetadata.h" +#include "backports/concepts.h" #include "util/ConfigManager/ConfigManager.h" #include "util/CopyableUniquePtr.h" #include "util/Exception.h" @@ -67,8 +68,8 @@ class BenchmarkResults { @param constructorArgs Arguments to pass to the constructor of the object, that the new `CopyableUniquePtr` will own. */ - template < - ad_utility::SameAsAny EntryType> + template ) EntryType> static EntryType& addEntryToContainerVector( PointerVector& targetVector, auto&&... constructorArgs) { targetVector.push_back(ad_utility::make_copyable_unique( diff --git a/benchmark/infrastructure/BenchmarkMeasurementContainer.cpp b/benchmark/infrastructure/BenchmarkMeasurementContainer.cpp index c7736e2e59..04bd28f41a 100644 --- a/benchmark/infrastructure/BenchmarkMeasurementContainer.cpp +++ b/benchmark/infrastructure/BenchmarkMeasurementContainer.cpp @@ -371,7 +371,7 @@ std::ostream& operator<<(std::ostream& os, const ResultGroup& resultGroup) { } // ____________________________________________________________________________ -template T> +template void ResultGroup::deleteEntryImpl(T& entry) { // The vector, that holds our entries. auto& vec = [this]() -> auto& { diff --git a/benchmark/infrastructure/BenchmarkMeasurementContainer.h b/benchmark/infrastructure/BenchmarkMeasurementContainer.h index 6703bb3224..7700e1fd18 100644 --- a/benchmark/infrastructure/BenchmarkMeasurementContainer.h +++ b/benchmark/infrastructure/BenchmarkMeasurementContainer.h @@ -375,7 +375,7 @@ class ResultGroup : public BenchmarkMetadataGetter { private: // The implementation for the general deletion of entries. - template T> + template void deleteEntryImpl(T& entry); }; diff --git a/src/backports/algorithm.h b/src/backports/algorithm.h index 90b4e2884c..86b8ecec8d 100644 --- a/src/backports/algorithm.h +++ b/src/backports/algorithm.h @@ -9,6 +9,8 @@ #include #include +#include "backports/concepts.h" + // The following defines namespaces `ql::ranges` and `ql::views` that are almost // drop-in replacements for `std::ranges` and `std::views`. In C++20 mode (when // the `QLEVER_CPP_17` macro is not used), these namespaces are simply aliases @@ -19,7 +21,6 @@ // currently not aware of, because they only affect functions that we currently // don't use. For those, the following header can be expanded in the future. #ifndef QLEVER_CPP_17 -#include #include #endif @@ -46,14 +47,4 @@ using namespace std::views; #endif } // namespace views -// The namespace `ql::concepts` includes concepts that are contained in the -// C++20 standard as well as in `range-v3`. -namespace concepts { -#ifdef QLEVER_CPP_17 -using namespace ::concepts; -#else -using namespace std; -#endif -} // namespace concepts - } // namespace ql diff --git a/src/backports/concepts.h b/src/backports/concepts.h index ad0159da32..3049c55c5c 100644 --- a/src/backports/concepts.h +++ b/src/backports/concepts.h @@ -1,17 +1,55 @@ -// Copyright 2024, University of Freiburg, -// Chair of Algorithms and Data Structures. -// Author: Johannes Kalmbach +// Copyright 2024, University of Freiburg +// Chair of Algorithms and Data Structures +// Author: Johannes Kalmbach #pragma once +#include +#ifndef QLEVER_CPP_17 +#include +#endif + // Define the following macros: -// `QL_OPT_CONCEPT(arg)` which expands to `arg` in C++20 mode, and to nothing in -// C++17 mode. It can be used to easily opt out of concepts that are only used -// for documentation and increased safety and not for overload resolution. -// Example usage: -// `(QL_OPT_CONCEPT(std::view) auto x = someFunction();` +// +// `QL_CONCEPT_OR_NOTHING(arg)`: expands to `arg` in C++20 mode, and to +// nothing in C++17 mode. It can be used to easily opt out of concepts that are +// only used for documentation and increased safety and not for overload +// resolution. +// +// `QL_CONCEPT_OR_TYPENAME(arg)`: expands to `arg` in C++20 mode, and to +// `typename` in C++17 mode. Example usage: +// +// Example usages: +// +// `QL_CONCEPT_OR_NOTHING(std::view) auto x = someFunction();` +// +// `QL_CONCEPT_OR_NOTHING(SameAsAny)` +// +// `void f(QL_CONCEPT_OR_NOTHING(std::view) auto x) {...}` +// +// `template ) T> void f(){...}` +// +// NOTE: The macros are variadic to allow for commas in the argument, like in +// the second example above. + #ifdef QLEVER_CPP_17 -#define QL_OPT_CONCEPT(arg) +#define QL_CONCEPT_OR_NOTHING(...) +#define QL_CONCEPT_OR_TYPENAME(...) typename #else -#define QL_OPT_CONCEPT(arg) arg +#define QL_CONCEPT_OR_NOTHING(...) __VA_ARGS__ +#define QL_CONCEPT_OR_TYPENAME(...) __VA_ARGS__ #endif + +// The namespace `ql::concepts` includes concepts that are contained in the +// C++20 standard as well as in `range-v3`. +namespace ql { +namespace concepts { + +#ifdef QLEVER_CPP_17 +using namespace ::concepts; +#else +using namespace std; +#endif + +} // namespace concepts +} // namespace ql diff --git a/src/engine/sparqlExpressions/LiteralExpression.h b/src/engine/sparqlExpressions/LiteralExpression.h index 4d9be1db5b..7da6739c75 100644 --- a/src/engine/sparqlExpressions/LiteralExpression.h +++ b/src/engine/sparqlExpressions/LiteralExpression.h @@ -40,11 +40,10 @@ class LiteralExpression : public SparqlExpression { // Evaluating just returns the constant/literal value. ExpressionResult evaluate(EvaluationContext* context) const override { // Common code for the `Literal` and `Iri` case. - auto getIdOrString = - [this, - &context](const ad_utility::SameAsAny auto& s) - -> ExpressionResult { + auto getIdOrString = [this, &context](const U& s) + -> CPP_ret(ExpressionResult)( + requires ad_utility::SameAsAny) { if (auto ptr = cachedResult_.load(std::memory_order_relaxed)) { return *ptr; } diff --git a/src/util/ConfigManager/ConfigManager.cpp b/src/util/ConfigManager/ConfigManager.cpp index 9ff8c4d3d8..3691c2588a 100644 --- a/src/util/ConfigManager/ConfigManager.cpp +++ b/src/util/ConfigManager/ConfigManager.cpp @@ -23,6 +23,7 @@ #include #include "backports/algorithm.h" +#include "backports/concepts.h" #include "util/Algorithm.h" #include "util/ComparisonWithNan.h" #include "util/ConfigManager/ConfigExceptions.h" @@ -254,7 +255,8 @@ requires std::is_object_v auto ConfigManager::allHashMapEntries( } // ____________________________________________________________________________ -template ReturnReference> +template ) + ReturnReference> std::vector> ConfigManager::configurationOptionsImpl( SimilarTo> auto& @@ -855,7 +857,7 @@ bool ConfigManager::containsOption(const ConfigOption& opt) const { } // ____________________________________________________________________________ -template +template void ConfigManager::ConfigurationDocValidatorAssignment::addEntryUnderKey( const T& key, const ConfigOptionValidatorManager& manager) { getHashMapBasedOnType()[&key].push_back(&manager); @@ -869,7 +871,7 @@ template void ConfigManager::ConfigurationDocValidatorAssignment:: const ConfigOptionValidatorManager&); // ____________________________________________________________________________ -template +template auto ConfigManager::ConfigurationDocValidatorAssignment::getEntriesUnderKey( const T& key) const -> ValueGetterReturnType { // The concerned hash map. diff --git a/src/util/ConfigManager/ConfigManager.h b/src/util/ConfigManager/ConfigManager.h index 63d44eed04..48691e96c5 100644 --- a/src/util/ConfigManager/ConfigManager.h +++ b/src/util/ConfigManager/ConfigManager.h @@ -18,6 +18,7 @@ #include #include +#include "backports/concepts.h" #include "util/ConfigManager/ConfigExceptions.h" #include "util/ConfigManager/ConfigOption.h" #include "util/ConfigManager/ConfigOptionProxy.h" @@ -34,7 +35,7 @@ namespace ConfigManagerImpl { // Shorthand concepts, to reduce code duplication. class ConfigManager; template -concept ConfigOptionOrManager = SameAsAny; +CPP_concept ConfigOptionOrManager = SameAsAny; /* Manages a bunch of `ConfigOption`s. @@ -579,7 +580,8 @@ class ConfigManager { @tparam ReturnReference Should be either `ConfigOption&`, or `const ConfigOption&`. */ - template ReturnReference> + template ) ReturnReference> static std::vector> configurationOptionsImpl( SimilarTo> auto& @@ -716,7 +718,7 @@ class ConfigManager { @brief Add a validator to the list of validators, that are assigned to a `ConfigOption`/`ConfigManager`. */ - template + template void addEntryUnderKey(const T& key, const ConfigOptionValidatorManager& manager); @@ -726,12 +728,12 @@ class ConfigManager { @returns If there is no entry for `Key`, return an empty `std::vector`. */ - template + template ValueGetterReturnType getEntriesUnderKey(const T& key) const; private: // Return either `configOption_` or `configManager_`, based on type. - template + template constexpr const MemoryAdressHashMap& getHashMapBasedOnType() const { if constexpr (std::same_as) { return configOption_; @@ -739,7 +741,7 @@ class ConfigManager { return configManager_; } } - template + template constexpr MemoryAdressHashMap& getHashMapBasedOnType() { if constexpr (std::same_as) { return configOption_; diff --git a/src/util/ConfigManager/ConfigOptionProxy.h b/src/util/ConfigManager/ConfigOptionProxy.h index 87a50c7be7..3050b4fcea 100644 --- a/src/util/ConfigManager/ConfigOptionProxy.h +++ b/src/util/ConfigManager/ConfigOptionProxy.h @@ -27,10 +27,10 @@ access to the referenced config option. @tparam ConfigOptionType The kind of config option, this proxy will reference to. Must be `ConfigOption`, or `const ConfigOption`. */ -template < - SupportedConfigOptionType T, - ad_utility::SameAsAny ConfigOptionType> -class ConfigOptionProxyImplementation { +CPP_template(typename T, typename ConfigOptionType)( + requires SupportedConfigOptionType CPP_and ad_utility::SameAsAny< + ConfigOptionType, ConfigOption, + const ConfigOption>) class ConfigOptionProxyImplementation { ConfigOptionType* option_; public: diff --git a/src/util/FsstCompressor.h b/src/util/FsstCompressor.h index b5c54c4d4d..e663bbf057 100644 --- a/src/util/FsstCompressor.h +++ b/src/util/FsstCompressor.h @@ -13,6 +13,7 @@ #include #include +#include "util/Concepts.h" #include "util/Exception.h" #include "util/Log.h" #include "util/TypeTraits.h" @@ -22,12 +23,16 @@ namespace detail { // `const unsigned char*` which is used below because FSST always works on // unsigned character types. Note that this is one of the few cases where a // `reinterpret_cast` is safe. -constexpr auto castToUnsignedPtr = - [] T>(T ptr) { - using Res = std::conditional_t, - const unsigned char*, unsigned char*>; - return reinterpret_cast(ptr); - }; +struct CastToUnsignedPtr { + CPP_template(typename T)( + requires ad_utility::SameAsAny) auto + operator()(T ptr) const { + using Res = std::conditional_t, + const unsigned char*, unsigned char*>; + return reinterpret_cast(ptr); + }; +}; +constexpr CastToUnsignedPtr castToUnsignedPtr{}; } // namespace detail // A simple C++ wrapper around the C-API of the `FSST` library. It consists of diff --git a/src/util/Iterators.h b/src/util/Iterators.h index 1337b63e27..93ae1b51f4 100644 --- a/src/util/Iterators.h +++ b/src/util/Iterators.h @@ -2,13 +2,13 @@ // Chair of Algorithms and Data Structures. // Author: Johannes Kalmbach -#ifndef QLEVER_ITERATORS_H -#define QLEVER_ITERATORS_H +#pragma once #include #include #include +#include "backports/algorithm.h" #include "util/Enums.h" #include "util/TypeTraits.h" @@ -399,5 +399,3 @@ class InputRangeTypeErased { using iterator = typename InputRangeFromGet::Iterator; }; } // namespace ad_utility - -#endif // QLEVER_ITERATORS_H diff --git a/src/util/JoinAlgorithms/JoinAlgorithms.h b/src/util/JoinAlgorithms/JoinAlgorithms.h index 36af501651..89e1b6611f 100644 --- a/src/util/JoinAlgorithms/JoinAlgorithms.h +++ b/src/util/JoinAlgorithms/JoinAlgorithms.h @@ -9,6 +9,7 @@ #include #include "backports/algorithm.h" +#include "backports/concepts.h" #include "engine/idTable/IdTable.h" #include "global/Id.h" #include "util/Generator.h" @@ -762,8 +763,8 @@ struct BlockZipperJoinImpl { #if defined(Side) || defined(Blocks) #error Side or Blocks are already defined #endif -#define Side SameAsAny auto -#define Blocks SameAsAny auto +#define Side QL_CONCEPT_OR_NOTHING(SameAsAny) auto +#define Blocks QL_CONCEPT_OR_NOTHING(SameAsAny) auto // Type alias for the result of the projection. Elements from the left and // right input must be projected to the same type. diff --git a/src/util/TypeTraits.h b/src/util/TypeTraits.h index 7d39e4aa70..8390667a85 100644 --- a/src/util/TypeTraits.h +++ b/src/util/TypeTraits.h @@ -12,7 +12,7 @@ #include #include -#include "backports/algorithm.h" +#include "backports/concepts.h" #include "util/Forward.h" namespace ad_utility { @@ -136,7 +136,7 @@ concept SimilarToAny = (... || isSimilar); /// True iff `T` is the same as any of the `Ts...`. template -concept SameAsAny = (... || ql::concepts::same_as); +CPP_concept SameAsAny = (... || ql::concepts::same_as); /* The implementation for `SimilarToAnyTypeIn` and `SameAsAnyTypeIn` (see below diff --git a/src/util/Views.h b/src/util/Views.h index 74e6eec14c..ddcf867d3f 100644 --- a/src/util/Views.h +++ b/src/util/Views.h @@ -343,7 +343,8 @@ CPP_template(typename Range, typename ElementType)( generator> reChunkAtSeparator( Range generator, ElementType separator) { std::vector buffer; - for (QL_OPT_CONCEPT(ql::ranges::input_range) auto const& chunk : generator) { + for (QL_CONCEPT_OR_NOTHING(ql::ranges::input_range) auto const& chunk : + generator) { for (ElementType c : chunk) { if (c == separator) { co_yield std::span{buffer.data(), buffer.size()}; diff --git a/test/ConfigOptionProxyTest.cpp b/test/ConfigOptionProxyTest.cpp index 29f9ed3fa3..a62e055af7 100644 --- a/test/ConfigOptionProxyTest.cpp +++ b/test/ConfigOptionProxyTest.cpp @@ -22,9 +22,9 @@ namespace ad_utility { @tparam OptionType Exists to define, if the test should be done with `ConfigOption`, or `const ConfigOption`. */ -template