From 5c6af103d364929fce9fdbdc3724113de46f8a56 Mon Sep 17 00:00:00 2001 From: Rustam Gamidov Date: Wed, 24 Jan 2024 12:05:50 +0200 Subject: [PATCH] Check result of cache related operations Skipped as not related to the offline map data: StreamLayerClient, ApiCacheRepository, CatalogSettings Relates-To: OLPEDGE-2852 Signed-off-by: Rustam Gamidov --- .../src/cache/DefaultCacheImpl.cpp | 35 ++- olp-cpp-sdk-core/src/cache/DiskCache.cpp | 2 +- .../repositories/CatalogCacheRepository.cpp | 22 +- .../src/repositories/CatalogCacheRepository.h | 8 +- .../src/repositories/CatalogRepository.cpp | 17 +- .../PartitionsCacheRepository.cpp | 27 +- .../repositories/PartitionsCacheRepository.h | 6 +- .../src/repositories/PartitionsRepository.cpp | 36 ++- .../repositories/PrefetchTilesRepository.cpp | 13 +- .../tests/CatalogRepositoryTest.cpp | 93 +++++- .../tests/PartitionsCacheRepositoryTest.cpp | 68 +++- .../tests/PartitionsRepositoryTest.cpp | 292 +++++++++++++++++- 12 files changed, 554 insertions(+), 65 deletions(-) diff --git a/olp-cpp-sdk-core/src/cache/DefaultCacheImpl.cpp b/olp-cpp-sdk-core/src/cache/DefaultCacheImpl.cpp index bed764515..2395e2e3c 100644 --- a/olp-cpp-sdk-core/src/cache/DefaultCacheImpl.cpp +++ b/olp-cpp-sdk-core/src/cache/DefaultCacheImpl.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2023 HERE Europe B.V. + * Copyright (C) 2019-2024 HERE Europe B.V. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -72,16 +72,28 @@ time_t GetRemainingExpiryTime(const std::string& key, return expiry; } -void PurgeDiskItem(const std::string& key, olp::cache::DiskCache& disk_cache, +bool PurgeDiskItem(const std::string& key, olp::cache::DiskCache& disk_cache, uint64_t& removed_data_size) { + bool result = true; auto expiry_key = CreateExpiryKey(key); uint64_t data_size = 0u; - disk_cache.Remove(key, data_size); + if (!disk_cache.Remove(key, data_size)) { + OLP_SDK_LOG_ERROR_F(kLogTag, "PurgeDiskItem failed to remove key='%s'", + key.c_str()); + result = false; + } removed_data_size += data_size; - disk_cache.Remove(expiry_key, data_size); + if (!disk_cache.Remove(expiry_key, data_size)) { + OLP_SDK_LOG_ERROR_F(kLogTag, + "PurgeDiskItem failed to remove expiry_key='%s'", + expiry_key.c_str()); + result = false; + } removed_data_size += data_size; + + return result; } size_t StoreExpiry(const std::string& key, leveldb::WriteBatch& batch, @@ -392,9 +404,14 @@ bool DefaultCacheImpl::Remove(const std::string& key) { if (mutable_cache_) { uint64_t removed_data_size = 0; - PurgeDiskItem(key, *mutable_cache_, removed_data_size); - + bool purge_passed = PurgeDiskItem(key, *mutable_cache_, removed_data_size); mutable_cache_data_size_ -= removed_data_size; + + if (!purge_passed) { + OLP_SDK_LOG_ERROR_F(kLogTag, "Remove() failed to purge item, key='%s'", + key.c_str()); + return false; + } } return true; @@ -974,7 +991,11 @@ bool DefaultCacheImpl::GetFromDiskCache(const std::string& key, // Data expired in cache -> remove, but not protected keys uint64_t removed_data_size = 0u; - PurgeDiskItem(key, *mutable_cache_, removed_data_size); + if (!PurgeDiskItem(key, *mutable_cache_, removed_data_size)) { + OLP_SDK_LOG_ERROR_F( + kLogTag, "GetFromDiskCache failed to purge an expired item, key='%s'", + key.c_str()); + } mutable_cache_data_size_ -= removed_data_size; RemoveKeyLru(key); } diff --git a/olp-cpp-sdk-core/src/cache/DiskCache.cpp b/olp-cpp-sdk-core/src/cache/DiskCache.cpp index f62a87c23..1641abfc2 100644 --- a/olp-cpp-sdk-core/src/cache/DiskCache.cpp +++ b/olp-cpp-sdk-core/src/cache/DiskCache.cpp @@ -364,9 +364,9 @@ bool DiskCache::Contains(const std::string& key) { } bool DiskCache::Remove(const std::string& key, uint64_t& removed_data_size) { + removed_data_size = 0u; if (!database_) { OLP_SDK_LOG_ERROR(kLogTag, "Remove: Database is not initialized"); - removed_data_size = 0; return false; } diff --git a/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.cpp b/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.cpp index 51fcdda2f..d8b537e14 100644 --- a/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.cpp +++ b/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2021 HERE Europe B.V. + * Copyright (C) 2019-2024 HERE Europe B.V. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -54,14 +54,14 @@ CatalogCacheRepository::CatalogCacheRepository( std::chrono::seconds default_expiry) : hrn_(hrn), cache_(cache), default_expiry_(ConvertTime(default_expiry)) {} -void CatalogCacheRepository::Put(const model::Catalog& catalog) { +bool CatalogCacheRepository::Put(const model::Catalog& catalog) { const std::string hrn(hrn_.ToCatalogHRNString()); const auto key = cache::KeyGenerator::CreateCatalogKey(hrn); OLP_SDK_LOG_DEBUG_F(kLogTag, "Put -> '%s'", key.c_str()); - cache_->Put(key, catalog, - [&]() { return olp::serializer::serialize(catalog); }, - default_expiry_); + return cache_->Put(key, catalog, + [&]() { return olp::serializer::serialize(catalog); }, + default_expiry_); } boost::optional CatalogCacheRepository::Get() { @@ -80,14 +80,14 @@ boost::optional CatalogCacheRepository::Get() { return boost::any_cast(cached_catalog); } -void CatalogCacheRepository::PutVersion(const model::VersionResponse& version) { +bool CatalogCacheRepository::PutVersion(const model::VersionResponse& version) { const std::string hrn(hrn_.ToCatalogHRNString()); const auto key = cache::KeyGenerator::CreateLatestVersionKey(hrn); OLP_SDK_LOG_DEBUG_F(kLogTag, "PutVersion -> '%s'", key.c_str()); - cache_->Put(key, version, - [&]() { return olp::serializer::serialize(version); }, - default_expiry_); + return cache_->Put(key, version, + [&]() { return olp::serializer::serialize(version); }, + default_expiry_); } boost::optional CatalogCacheRepository::GetVersion() { @@ -105,12 +105,12 @@ boost::optional CatalogCacheRepository::GetVersion() { return boost::any_cast(cached_version); } -void CatalogCacheRepository::Clear() { +bool CatalogCacheRepository::Clear() { const std::string hrn(hrn_.ToCatalogHRNString()); const auto key = cache::KeyGenerator::CreateCatalogKey(hrn); OLP_SDK_LOG_INFO_F(kLogTag, "Clear -> '%s'", key.c_str()); - cache_->RemoveKeysWithPrefix(hrn); + return cache_->RemoveKeysWithPrefix(hrn); } } // namespace repository diff --git a/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.h b/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.h index e27b33d63..e5159d84d 100644 --- a/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.h +++ b/olp-cpp-sdk-dataservice-read/src/repositories/CatalogCacheRepository.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2021 HERE Europe B.V. + * Copyright (C) 2019-2024 HERE Europe B.V. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,15 +43,15 @@ class CatalogCacheRepository final { ~CatalogCacheRepository() = default; - void Put(const model::Catalog& catalog); + bool Put(const model::Catalog& catalog); boost::optional Get(); - void PutVersion(const model::VersionResponse& version); + bool PutVersion(const model::VersionResponse& version); boost::optional GetVersion(); - void Clear(); + bool Clear(); private: client::HRN hrn_; diff --git a/olp-cpp-sdk-dataservice-read/src/repositories/CatalogRepository.cpp b/olp-cpp-sdk-dataservice-read/src/repositories/CatalogRepository.cpp index 675e832a6..2180f75a3 100644 --- a/olp-cpp-sdk-dataservice-read/src/repositories/CatalogRepository.cpp +++ b/olp-cpp-sdk-dataservice-read/src/repositories/CatalogRepository.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2022 HERE Europe B.V. + * Copyright (C) 2019-2024 HERE Europe B.V. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -92,7 +92,12 @@ CatalogResponse CatalogRepository::GetCatalog( config_client, catalog_str, request.GetBillingTag(), context); if (catalog_response.IsSuccessful() && fetch_options != OnlineOnly) { - repository.Put(catalog_response.GetResult()); + if (!repository.Put(catalog_response.GetResult())) { + OLP_SDK_LOG_WARNING_F( + kLogTag, + "GetCatalog failed to cache received results, hrn='%s',key='%s'", + catalog_str.c_str(), request_key.c_str()); + } } if (!catalog_response.IsSuccessful()) { const auto& error = catalog_response.GetError(); @@ -163,7 +168,13 @@ CatalogVersionResponse CatalogRepository::GetLatestVersion( // Write or update the version in cache, updating happens only when the new // version is greater than cached. if (!cached_version || (*cached_version).GetVersion() < new_version) { - repository.PutVersion(version_response.GetResult()); + if (!repository.PutVersion(version_response.GetResult())) { + OLP_SDK_LOG_WARNING_F(kLogTag, + "GetLatestVersion failed to cache latest " + "version, hrn='%s', version=%" PRId64, + catalog_.ToCatalogHRNString().c_str(), + new_version); + } if (fetch_option == CacheOnly) { OLP_SDK_LOG_DEBUG_F( kLogTag, "Latest user set version, hrn='%s', version=%" PRId64, diff --git a/olp-cpp-sdk-dataservice-read/src/repositories/PartitionsCacheRepository.cpp b/olp-cpp-sdk-dataservice-read/src/repositories/PartitionsCacheRepository.cpp index b2cc6ca73..ce0dadc8c 100644 --- a/olp-cpp-sdk-dataservice-read/src/repositories/PartitionsCacheRepository.cpp +++ b/olp-cpp-sdk-dataservice-read/src/repositories/PartitionsCacheRepository.cpp @@ -169,15 +169,15 @@ boost::optional PartitionsCacheRepository::Get( return partitions; } -void PartitionsCacheRepository::Put( +bool PartitionsCacheRepository::Put( int64_t catalog_version, const model::LayerVersions& layer_versions) { const auto key = cache::KeyGenerator::CreateLayerVersionsKey(catalog_, catalog_version); OLP_SDK_LOG_DEBUG_F(kLogTag, "Put -> '%s'", key.c_str()); - cache_->Put(key, layer_versions, - [&]() { return serializer::serialize(layer_versions); }, - default_expiry_); + return cache_->Put(key, layer_versions, + [&]() { return serializer::serialize(layer_versions); }, + default_expiry_); } boost::optional PartitionsCacheRepository::Get( @@ -237,25 +237,30 @@ bool PartitionsCacheRepository::Get(geo::TileKey tile_key, int32_t depth, return false; } -void PartitionsCacheRepository::Clear() { +bool PartitionsCacheRepository::Clear() { auto key = catalog_ + "::" + layer_id_ + "::"; OLP_SDK_LOG_INFO_F(kLogTag, "Clear -> '%s'", key.c_str()); - cache_->RemoveKeysWithPrefix(key); + return cache_->RemoveKeysWithPrefix(key); } -void PartitionsCacheRepository::ClearPartitions( +bool PartitionsCacheRepository::ClearPartitions( const std::vector& partition_ids, const boost::optional& version) { OLP_SDK_LOG_INFO_F(kLogTag, "ClearPartitions -> '%s'", catalog_.c_str()); auto cached_partitions = Get(partition_ids, version); + bool passed = true; // Partitions not processed here are not cached to begin with. for (const auto& partition : cached_partitions.GetPartitions()) { - cache_->RemoveKeysWithPrefix(catalog_ + "::" + layer_id_ + - "::" + partition.GetDataHandle()); - cache_->RemoveKeysWithPrefix(catalog_ + "::" + layer_id_ + - "::" + partition.GetPartition()); + passed = cache_->RemoveKeysWithPrefix(catalog_ + "::" + layer_id_ + + "::" + partition.GetDataHandle()) && + passed; + passed = cache_->RemoveKeysWithPrefix(catalog_ + "::" + layer_id_ + + "::" + partition.GetPartition()) && + passed; } + + return passed; } bool PartitionsCacheRepository::ClearQuadTree( diff --git a/olp-cpp-sdk-dataservice-read/src/repositories/PartitionsCacheRepository.h b/olp-cpp-sdk-dataservice-read/src/repositories/PartitionsCacheRepository.h index 0e8b0f4ab..297dae844 100644 --- a/olp-cpp-sdk-dataservice-read/src/repositories/PartitionsCacheRepository.h +++ b/olp-cpp-sdk-dataservice-read/src/repositories/PartitionsCacheRepository.h @@ -59,7 +59,7 @@ class PartitionsCacheRepository final { const PartitionsRequest& request, const boost::optional& version); - void Put(int64_t catalog_version, const model::LayerVersions& layer_versions); + bool Put(int64_t catalog_version, const model::LayerVersions& layer_versions); boost::optional Get(int64_t catalog_version); @@ -70,9 +70,9 @@ class PartitionsCacheRepository final { bool Get(geo::TileKey tile_key, int32_t depth, const boost::optional& version, QuadTreeIndex& tree); - void Clear(); + bool Clear(); - void ClearPartitions(const std::vector& partition_ids, + bool ClearPartitions(const std::vector& partition_ids, const boost::optional& version); bool ClearQuadTree(geo::TileKey tile_key, int32_t depth, diff --git a/olp-cpp-sdk-dataservice-read/src/repositories/PartitionsRepository.cpp b/olp-cpp-sdk-dataservice-read/src/repositories/PartitionsRepository.cpp index 5385bd3e3..9c3a3daa5 100644 --- a/olp-cpp-sdk-dataservice-read/src/repositories/PartitionsRepository.cpp +++ b/olp-cpp-sdk-dataservice-read/src/repositories/PartitionsRepository.cpp @@ -320,7 +320,11 @@ PartitionsRepository::GetPartitionsExtendedResponse( kLogTag, "GetPartitions 403 received, remove from cache, hrn='%s', key='%s'", catalog_str.c_str(), key.c_str()); - cache_.Clear(); + if (!cache_.Clear()) { + OLP_SDK_LOG_ERROR_F( + kLogTag, "Failed to clear data from cache, hrn='%s', key='%s'", + catalog_.ToCatalogHRNString().c_str(), key.c_str()); + } } } @@ -381,7 +385,16 @@ PartitionsResponse PartitionsRepository::GetPartitionById( OLP_SDK_LOG_DEBUG_F(kLogTag, "GetPartitionById put to cache, hrn='%s', key='%s'", catalog_.ToCatalogHRNString().c_str(), key.c_str()); - cache_.Put(query_response.GetResult(), version, boost::none); + const auto put_result = + cache_.Put(query_response.GetResult(), version, boost::none); + if (!put_result.IsSuccessful()) { + OLP_SDK_LOG_ERROR_F(kLogTag, + "GetPartitionById failed to write data to cache, " + "hrn='%s', key='%s', error=%s", + catalog_.ToCatalogHRNString().c_str(), key.c_str(), + put_result.GetError().GetMessage().c_str()); + } + } else if (!query_response.IsSuccessful()) { const auto& error = query_response.GetError(); if (error.GetHttpStatusCode() == http::HttpStatusCode::FORBIDDEN) { @@ -390,7 +403,13 @@ PartitionsResponse PartitionsRepository::GetPartitionById( "hrn='%s', key='%s'", catalog_.ToCatalogHRNString().c_str(), key.c_str()); // Delete partitions only but not the layer - cache_.ClearPartitions(partitions, version); + if (!cache_.ClearPartitions(partitions, version)) { + OLP_SDK_LOG_ERROR_F( + kLogTag, + "GetPartitionById failed to clear partitions from cache, " + "hrn='%s', key='%s'", + catalog_.ToCatalogHRNString().c_str(), key.c_str()); + } } } @@ -507,7 +526,16 @@ QuadTreeIndexResponse PartitionsRepository::GetQuadTreeIndexForTile( } if (fetch_option != OnlineOnly) { - cache_.Put(root_tile_key, kAggregateQuadTreeDepth, tree, version); + const auto put_result = + cache_.Put(root_tile_key, kAggregateQuadTreeDepth, tree, version); + if (!put_result.IsSuccessful()) { + OLP_SDK_LOG_ERROR_F(kLogTag, + "GetQuadTreeIndexForTile failed to cache data, " + "hrn='%s', key='%s', error=%s", + catalog_.ToCatalogHRNString().c_str(), + root_tile_here.c_str(), + put_result.GetError().GetMessage().c_str()); + } } return QuadTreeIndexResponse(std::move(tree), diff --git a/olp-cpp-sdk-dataservice-read/src/repositories/PrefetchTilesRepository.cpp b/olp-cpp-sdk-dataservice-read/src/repositories/PrefetchTilesRepository.cpp index 685dfe120..6ee9157f9 100644 --- a/olp-cpp-sdk-dataservice-read/src/repositories/PrefetchTilesRepository.cpp +++ b/olp-cpp-sdk-dataservice-read/src/repositories/PrefetchTilesRepository.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2023 HERE Europe B.V. + * Copyright (C) 2019-2024 HERE Europe B.V. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -303,8 +303,15 @@ SubQuadsResponse PrefetchTilesRepository::GetVolatileSubQuads( subtile.ToHereTile())); } - // add to cache - cache_repository_.Put(partitions, boost::none, boost::none, false); + const auto put_result = + cache_repository_.Put(partitions, boost::none, boost::none, false); + if (!put_result.IsSuccessful()) { + OLP_SDK_LOG_ERROR_F(kLogTag, + "GetVolatileSubQuads failed to write data to cache, " + "hrn='%s', key='%s', error=%s", + catalog_str_.c_str(), tile_key.c_str(), + put_result.GetError().GetMessage().c_str()); + } return result; } diff --git a/olp-cpp-sdk-dataservice-read/tests/CatalogRepositoryTest.cpp b/olp-cpp-sdk-dataservice-read/tests/CatalogRepositoryTest.cpp index ce004c2ea..ea2f30af3 100644 --- a/olp-cpp-sdk-dataservice-read/tests/CatalogRepositoryTest.cpp +++ b/olp-cpp-sdk-dataservice-read/tests/CatalogRepositoryTest.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019-2021 HERE Europe B.V. + * Copyright (C) 2019-2024 HERE Europe B.V. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -168,10 +168,8 @@ TEST_F(CatalogRepositoryTest, GetLatestVersionCacheOnlyRequestWithMinVersion) { .WithStartVersion(kStartVersion); EXPECT_CALL(*cache_, Get(_, _)) - .Times(1) - .WillOnce(testing::Return(boost::any{})); - - EXPECT_CALL(*cache_, Put(_, _, _, _)).Times(1); + .Times(2) + .WillRepeatedly(testing::Return(boost::any{})); ON_CALL(*network_, Send(_, _, _, _, _)) .WillByDefault([](olp::http::NetworkRequest, olp::http::Network::Payload, @@ -184,10 +182,30 @@ TEST_F(CatalogRepositoryTest, GetLatestVersionCacheOnlyRequestWithMinVersion) { ApiLookupClient lookup_client(kHrn, settings_); repository::CatalogRepository repository(kHrn, settings_, lookup_client); - auto response = repository.GetLatestVersion(request, context); - EXPECT_TRUE(response.IsSuccessful()); - EXPECT_EQ(response.GetResult().GetVersion(), kStartVersion); + { + SCOPED_TRACE("Put succeeded"); + + EXPECT_CALL(*cache_, Put(_, _, _, _)) + .Times(1) + .WillOnce(testing::Return(true)); + + auto response = repository.GetLatestVersion(request, context); + EXPECT_TRUE(response.IsSuccessful()); + EXPECT_EQ(response.GetResult().GetVersion(), kStartVersion); + } + + { + SCOPED_TRACE("Put failed"); + + EXPECT_CALL(*cache_, Put(_, _, _, _)) + .Times(1) + .WillOnce(testing::Return(false)); + + auto response = repository.GetLatestVersion(request, context); + EXPECT_TRUE(response.IsSuccessful()); + EXPECT_EQ(response.GetResult().GetVersion(), kStartVersion); + } } TEST_F(CatalogRepositoryTest, GetLatestVersionOnlineOnlyNotFound) { @@ -396,10 +414,6 @@ TEST_F(CatalogRepositoryTest, GetCatalogOnlineOnlyFound) { return boost::any{}; }); - EXPECT_CALL(*cache_, Put(testing::Eq(kCatalogCacheKey), _, _, _)).Times(0); - - EXPECT_CALL(*cache_, Put(testing::Eq(kConfigCacheKey), _, _, _)).Times(0); - ON_CALL(*network_, Send(IsGetRequest(kUrlLookupConfig), _, _, _, _)) .WillByDefault(ReturnHttpResponse(olp::http::NetworkResponse().WithStatus( olp::http::HttpStatusCode::OK), @@ -417,6 +431,61 @@ TEST_F(CatalogRepositoryTest, GetCatalogOnlineOnlyFound) { ASSERT_TRUE(response.IsSuccessful()); } +TEST_F(CatalogRepositoryTest, GetCatalogOnlineIfNotFound) { + olp::client::CancellationContext context; + + auto request = read::CatalogRequest(); + request.WithFetchOption(read::OnlineIfNotFound); + + EXPECT_CALL(*cache_, Get(_, _)).WillRepeatedly(testing::Return(boost::any{})); + + EXPECT_CALL(*cache_, + Put(testing::Eq(kCatalog + "::config::v1::api"), _, _, _)) + .WillRepeatedly(testing::Return(true)); + EXPECT_CALL(*cache_, + Put(testing::Eq(kCatalog + "::pipelines::v1::api"), _, _, _)) + .WillRepeatedly(testing::Return(true)); + EXPECT_CALL(*cache_, + Put(testing::Eq(kCatalog + "::pipelines::v2::api"), _, _, _)) + .WillRepeatedly(testing::Return(true)); + + ON_CALL(*network_, Send(IsGetRequest(kUrlLookupConfig), _, _, _, _)) + .WillByDefault(ReturnHttpResponse(olp::http::NetworkResponse().WithStatus( + olp::http::HttpStatusCode::OK), + kResponseLookupConfig)); + + ON_CALL(*network_, Send(IsGetRequest(kUrlConfig), _, _, _, _)) + .WillByDefault(ReturnHttpResponse(olp::http::NetworkResponse().WithStatus( + olp::http::HttpStatusCode::OK), + kResponseConfig)); + + { + SCOPED_TRACE("Put failed"); + + EXPECT_CALL(*cache_, Put(testing::Eq(kCatalogCacheKey), _, _, _)) + .Times(1) + .WillOnce(testing::Return(false)); + + ApiLookupClient lookup_client(kHrn, settings_); + repository::CatalogRepository repository(kHrn, settings_, lookup_client); + auto response = repository.GetCatalog(request, context); + ASSERT_TRUE(response.IsSuccessful()); + } + + { + SCOPED_TRACE("Put succeeded"); + + EXPECT_CALL(*cache_, Put(testing::Eq(kCatalogCacheKey), _, _, _)) + .Times(1) + .WillOnce(testing::Return(true)); + + ApiLookupClient lookup_client(kHrn, settings_); + repository::CatalogRepository repository(kHrn, settings_, lookup_client); + auto response = repository.GetCatalog(request, context); + ASSERT_TRUE(response.IsSuccessful()); + } +} + TEST_F(CatalogRepositoryTest, GetCatalogCacheOnlyFound) { olp::client::CancellationContext context; diff --git a/olp-cpp-sdk-dataservice-read/tests/PartitionsCacheRepositoryTest.cpp b/olp-cpp-sdk-dataservice-read/tests/PartitionsCacheRepositoryTest.cpp index e5c0be7fe..d7c11ddc8 100644 --- a/olp-cpp-sdk-dataservice-read/tests/PartitionsCacheRepositoryTest.cpp +++ b/olp-cpp-sdk-dataservice-read/tests/PartitionsCacheRepositoryTest.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2020 HERE Europe B.V. + * Copyright (C) 2020-2024 HERE Europe B.V. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -224,4 +224,70 @@ TEST(PartitionsCacheRepositoryTest, GetPartitionHandle) { } } +TEST(PartitionsCacheRepositoryTest, ClearPartitions) { + const auto hrn = HRN::FromString(kCatalog); + const auto layer = "layer"; + + model::Partition some_partition; + some_partition.SetPartition(kPartitionId); + some_partition.SetDataHandle(kDataHandle); + model::Partitions partitions; + auto& partitions_vector = partitions.GetMutablePartitions(); + partitions_vector.push_back(some_partition); + + const std::string kPartitionDataHandle = + std::string(kCatalog) + "::" + layer + "::" + kDataHandle; + const std::string kPartition = + std::string(kCatalog) + "::" + layer + "::" + kPartitionId; + const std::string kPartitionKey = kPartition + "::partition"; + + auto cache = std::make_shared>(); + + EXPECT_CALL(*cache, Put(testing::Eq(kPartitionKey), testing::_, testing::_, + testing::_)) + .WillRepeatedly(testing::Return(true)); + + EXPECT_CALL(*cache, Get(testing::Eq(kPartitionKey), testing::_)) + .WillRepeatedly(testing::Return(some_partition)); + + { + SCOPED_TRACE("RemoveKeysWithPrefix data handle failed"); + + EXPECT_CALL(*cache, RemoveKeysWithPrefix(testing::Eq(kPartitionDataHandle))) + .WillOnce(testing::Return(false)); + EXPECT_CALL(*cache, RemoveKeysWithPrefix(testing::Eq(kPartition))) + .WillOnce(testing::Return(true)); + + repository::PartitionsCacheRepository repository(hrn, layer, cache); + repository.Put(partitions, boost::none, true); + EXPECT_FALSE(repository.ClearPartitions({kPartitionId}, boost::none)); + } + + { + SCOPED_TRACE("RemoveKeysWithPrefix partition failed"); + + EXPECT_CALL(*cache, RemoveKeysWithPrefix(testing::Eq(kPartitionDataHandle))) + .WillOnce(testing::Return(true)); + EXPECT_CALL(*cache, RemoveKeysWithPrefix(testing::Eq(kPartition))) + .WillOnce(testing::Return(false)); + + repository::PartitionsCacheRepository repository(hrn, layer, cache); + repository.Put(partitions, boost::none, true); + EXPECT_FALSE(repository.ClearPartitions({kPartitionId}, boost::none)); + } + + { + SCOPED_TRACE("RemoveKeysWithPrefix passed"); + + EXPECT_CALL(*cache, RemoveKeysWithPrefix(testing::Eq(kPartitionDataHandle))) + .WillOnce(testing::Return(true)); + EXPECT_CALL(*cache, RemoveKeysWithPrefix(testing::Eq(kPartition))) + .WillOnce(testing::Return(true)); + + repository::PartitionsCacheRepository repository(hrn, layer, cache); + repository.Put(partitions, boost::none, true); + EXPECT_TRUE(repository.ClearPartitions({kPartitionId}, boost::none)); + } +} + } // namespace diff --git a/olp-cpp-sdk-dataservice-read/tests/PartitionsRepositoryTest.cpp b/olp-cpp-sdk-dataservice-read/tests/PartitionsRepositoryTest.cpp index e6373406a..80b0c59e5 100644 --- a/olp-cpp-sdk-dataservice-read/tests/PartitionsRepositoryTest.cpp +++ b/olp-cpp-sdk-dataservice-read/tests/PartitionsRepositoryTest.cpp @@ -49,6 +49,9 @@ using olp::client::OlpClientSettings; using olp::dataservice::read::DataRequest; using olp::geo::TileKey; using testing::_; +using testing::Eq; +using testing::Mock; +using testing::Return; namespace parser = olp::parser; namespace read = olp::dataservice::read; @@ -1277,6 +1280,29 @@ TEST_F(PartitionsRepositoryTest, GetTile) { testing::Mock::VerifyAndClearExpectations(mock_cache.get()); } + { + SCOPED_TRACE( + "Get tile not aggregated, partition not found, caching failed"); + + setup_get_cached_quad_expectations(); + EXPECT_CALL(*mock_network, + Send(IsGetRequest(kQueryQuadTreeIndex), _, _, _, _)) + .WillOnce(ReturnHttpResponse(olp::http::NetworkResponse().WithStatus( + olp::http::HttpStatusCode::OK), + kSubQuadsWithParent)); + EXPECT_CALL(*mock_cache, Put(quad_cache_key(root), _, _)) + .WillOnce(Return(false)); + + repository::PartitionsRepository repository(hrn, kVersionedLayerId, + settings, lookup_client); + const auto response = repository.GetTile(request, kVersion, context); + + ASSERT_FALSE(response); + + testing::Mock::VerifyAndClearExpectations(mock_network.get()); + testing::Mock::VerifyAndClearExpectations(mock_cache.get()); + } + const auto kHereTile = "5904591"; tile_key = TileKey::FromHereTile(kHereTile); root = tile_key.ChangedLevelBy(-depth); @@ -1646,6 +1672,154 @@ TEST_F(PartitionsRepositoryTest, GetVersionedPartitionsBatch) { } } +TEST_F(PartitionsRepositoryTest, GetVersionedPartitionsBatch_MockedCache) { + using testing::Return; + using PartitionIds = read::PartitionsRequest::PartitionIds; + + auto mock_network = std::make_shared(); + const auto catalog = HRN::FromString(kCatalog); + const size_t kQueryRequestLimit = 100; + const std::string kPrefixToClear = kCatalog + "::" + kVersionedLayerId + "::"; + const PartitionIds kPartitions{110, kPartitionId}; + + const auto appendVersion = [&](std::string input, + int version) -> std::string { + input.append(R"(&version=)" + std::to_string(version)); + return input; + }; + + const auto appendPartitions = + [&](std::string input, const PartitionIds& partitions) -> std::string { + if (partitions.empty()) { + return input; + } + + input.append("?partition=" + partitions.front()); + if (partitions.size() == 1) { + return input; + } + + std::for_each(partitions.cbegin() + 1, partitions.cend(), + [&](const std::string& partition) { + input.append("&partition="); + input.append(partition); + }); + + return input; + }; + + const auto createBatchedUrls = [&](const PartitionIds& partitions) { + std::vector urls; + + for (size_t i = 0; i < partitions.size(); i += kQueryRequestLimit) { + urls.push_back(appendVersion( + appendPartitions( + kOlpSdkUrlPartitionByIdBase, + {partitions.begin() + i, + partitions.begin() + + std::min(partitions.size(), i + kQueryRequestLimit)}), + kVersion)); + } + + return urls; + }; + + const auto setupExpectations = + [&](std::shared_ptr> cache, + OlpClientSettings& settings) { + settings.cache = cache; + settings.network_request_handler = mock_network; + settings.retry_settings.timeout = 1; + + EXPECT_CALL(*cache, Get(_, _)) + .WillRepeatedly(testing::Return(boost::any{})); + EXPECT_CALL(*cache, Put(Eq(kCacheKeyMetadata), _, _, _)) + .WillRepeatedly(Return(true)); + + EXPECT_CALL(*mock_network, + Send(IsGetRequest(kOlpSdkUrlLookupQuery), _, _, _, _)) + .WillOnce( + ReturnHttpResponse(olp::http::NetworkResponse().WithStatus( + olp::http::HttpStatusCode::OK), + kOlpSdkHttpResponseLookupQuery)); + + const auto batchedUrls = createBatchedUrls(kPartitions); + ASSERT_GT(batchedUrls.size(), 1); + + std::for_each(batchedUrls.cbegin(), batchedUrls.cend() - 1, + [&](const std::string& url) { + EXPECT_CALL(*mock_network, + Send(IsGetRequest(url), _, _, _, _)) + .WillOnce(ReturnHttpResponse( + olp::http::NetworkResponse().WithStatus( + olp::http::HttpStatusCode::OK), + kOlpSdkHttpResponsePartitionById)); + }); + + EXPECT_CALL(*mock_network, + Send(IsGetRequest(batchedUrls.back()), _, _, _, _)) + .WillOnce( + ReturnHttpResponse(olp::http::NetworkResponse().WithStatus( + olp::http::HttpStatusCode::FORBIDDEN), + kOlpSdkHttpResponsePartitionById)); + }; + + auto cache = std::make_shared>(); + OlpClientSettings settings; + + { + SCOPED_TRACE( + "Forbidden fetch from network with a list of partitions. Cache clear " + "succeeded"); + + setupExpectations(cache, settings); + + client::CancellationContext context; + ApiLookupClient lookup_client(catalog, settings); + repository::PartitionsRepository repository(catalog, kVersionedLayerId, + settings, lookup_client); + read::PartitionsRequest request; + request.WithPartitionIds(kPartitions); + + EXPECT_CALL(*cache, RemoveKeysWithPrefix(Eq(kPrefixToClear))) + .WillOnce(Return(true)); + + auto response = repository.GetVersionedPartitionsExtendedResponse( + request, kVersion, context); + + ASSERT_FALSE(response.IsSuccessful()); + EXPECT_TRUE(response.GetResult().GetPartitions().empty()); + EXPECT_EQ(response.GetError().GetErrorCode(), + olp::client::ErrorCode::AccessDenied); + } + + { + SCOPED_TRACE( + "Forbidden fetch from network with a list of partitions. Cache clear " + "failed"); + + setupExpectations(cache, settings); + + client::CancellationContext context; + ApiLookupClient lookup_client(catalog, settings); + repository::PartitionsRepository repository(catalog, kVersionedLayerId, + settings, lookup_client); + read::PartitionsRequest request; + request.WithPartitionIds(kPartitions); + + EXPECT_CALL(*cache, RemoveKeysWithPrefix(Eq(kPrefixToClear))) + .WillOnce(Return(false)); + + auto response = repository.GetVersionedPartitionsExtendedResponse( + request, kVersion, context); + + ASSERT_FALSE(response.IsSuccessful()); + EXPECT_TRUE(response.GetResult().GetPartitions().empty()); + EXPECT_EQ(response.GetError().GetErrorCode(), + olp::client::ErrorCode::AccessDenied); + } +} + TEST_F(PartitionsRepositoryTest, ParsePartitionsStream) { const auto kTimeout = std::chrono::seconds(5); const auto catalog = HRN::FromString(kCatalog); @@ -1712,7 +1886,8 @@ TEST_F(PartitionsRepositoryTest, ParsePartitionsStream) { response = repository.ParsePartitionsStream(async_stream, {}, context_to_cancel); parsing_promise.set_value(); - }).detach(); + }) + .detach(); // give the parsing time to start std::this_thread::sleep_for(std::chrono::milliseconds(100u)); @@ -1772,10 +1947,6 @@ TEST_F(PartitionsRepositoryTest, ParsePartitionsStream) { } TEST_F(PartitionsRepositoryTest, StreamPartitions) { - using testing::Eq; - using testing::Mock; - using testing::Return; - const auto catalog = HRN::FromString(kCatalog); const std::string cacheKeyMetadata = kCatalog + "::metadata::v1::api"; @@ -1914,4 +2085,115 @@ TEST_F(PartitionsRepositoryTest, StreamPartitions) { } } +class PartitionsRepositoryTest_GetPartitionById + : public PartitionsRepositoryTest { + public: + void SetUp() override { + settings_ = std::make_shared(); + settings_->cache = cache_; + settings_->network_request_handler = network_; + settings_->retry_settings.timeout = 1; + + ApiLookupClient lookup_client(catalog_hrn_, *settings_); + repository_ = std::make_shared( + catalog_hrn_, kVersionedLayerId, *settings_, lookup_client); + } + + protected: + std::shared_ptr> cache_ = + std::make_shared>(); + std::shared_ptr> network_ = + std::make_shared>(); + + const HRN catalog_hrn_ = HRN::FromString(kCatalog); + const DataRequest request_{DataRequest().WithPartitionId(kPartitionId)}; + + std::shared_ptr repository_; + std::shared_ptr settings_; + + const std::string datahandle_prefix_ = + kCatalog + "::" + kVersionedLayerId + "::qwerty"; + const std::string partition_prefix_ = + kCatalog + "::" + kVersionedLayerId + "::" + kPartitionId; + const std::string cache_key_no_version_ = partition_prefix_ + "::partition"; + const std::string cache_key_ = + partition_prefix_ + "::" + std::to_string(kVersion) + "::partition"; + const std::string query_cache_response_ = + R"jsonString({"version":100,"partition":"1111","layer":"testlayer","dataHandle":"qwerty"})jsonString"; +}; + +TEST_F(PartitionsRepositoryTest_GetPartitionById, + FetchFromOnlineOK_FailedPutCauseNoError) { + EXPECT_CALL(*cache_, Get(cache_key_, _)).WillOnce(Return(boost::any{})); + EXPECT_CALL(*cache_, Get(Eq(kCacheKeyMetadata), _)) + .WillOnce(Return(boost::any{})); + + EXPECT_CALL(*network_, Send(IsGetRequest(kOlpSdkUrlLookupQuery), _, _, _, _)) + .WillOnce(ReturnHttpResponse(olp::http::NetworkResponse().WithStatus( + olp::http::HttpStatusCode::OK), + kOlpSdkHttpResponseLookupQuery)); + EXPECT_CALL(*cache_, Put(Eq(kCacheKeyMetadata), _, _, _)) + .WillOnce(Return(true)); + + EXPECT_CALL(*network_, + Send(IsGetRequest(kOlpSdkUrlPartitionById), _, _, _, _)) + .WillOnce(ReturnHttpResponse(olp::http::NetworkResponse().WithStatus( + olp::http::HttpStatusCode::OK), + kOlpSdkHttpResponsePartitionById)); + + EXPECT_CALL(*cache_, Put(cache_key_, _, _, _)).WillOnce(Return(false)); + + client::CancellationContext context; + auto response = repository_->GetPartitionById( + DataRequest(request_).WithFetchOption(read::OnlineIfNotFound), kVersion, + context); + + ASSERT_TRUE(response.IsSuccessful()); + const auto& result = response.GetResult(); + const auto& partitions = result.GetPartitions(); + EXPECT_EQ(partitions.size(), 1); + const auto& partition = partitions.front(); + EXPECT_EQ(partition.GetDataHandle(), "PartitionsRepositoryTest-partitionId"); + EXPECT_EQ(partition.GetVersion().value_or(0), 42); + EXPECT_EQ(partition.GetPartition(), "1111"); +} + +TEST_F(PartitionsRepositoryTest_GetPartitionById, + FetchOnlineFORBIDDEN_ClearPartitionsFails) { + SCOPED_TRACE("Fetch from online FORBIDDEN. Clear partitions fails"); + + EXPECT_CALL(*cache_, Get(cache_key_, _)) + .Times(2) + .WillOnce(Return(boost::any{})) + .WillOnce(Return(parser::parse(query_cache_response_))); + EXPECT_CALL(*cache_, Get(Eq(kCacheKeyMetadata), _)) + .WillOnce(Return(boost::any{})); + + EXPECT_CALL(*network_, Send(IsGetRequest(kOlpSdkUrlLookupQuery), _, _, _, _)) + .WillOnce(ReturnHttpResponse(olp::http::NetworkResponse().WithStatus( + olp::http::HttpStatusCode::OK), + kOlpSdkHttpResponseLookupQuery)); + EXPECT_CALL(*cache_, Put(Eq(kCacheKeyMetadata), _, _, _)) + .WillOnce(Return(true)); + + EXPECT_CALL(*network_, + Send(IsGetRequest(kOlpSdkUrlPartitionById), _, _, _, _)) + .WillOnce(ReturnHttpResponse(olp::http::NetworkResponse().WithStatus( + olp::http::HttpStatusCode::FORBIDDEN), + "{Inappropriate}")); + + EXPECT_CALL(*cache_, RemoveKeysWithPrefix(Eq(datahandle_prefix_))) + .WillOnce(Return(true)); + EXPECT_CALL(*cache_, RemoveKeysWithPrefix(Eq(partition_prefix_))) + .WillOnce(Return(false)); + + client::CancellationContext context; + auto response = repository_->GetPartitionById( + DataRequest(request_).WithFetchOption(read::OnlineIfNotFound), kVersion, + context); + + EXPECT_FALSE(response.IsSuccessful()); + EXPECT_EQ(response.GetError().GetErrorCode(), ErrorCode::AccessDenied); +} + } // namespace