Skip to content

Commit

Permalink
Fix performance with strict_capacity_limit
Browse files Browse the repository at this point in the history
  • Loading branch information
@pdillinger
pdillinger committed Dec 15, 2023
1 parent 736aab0 commit 8ed282b
Show file tree
Hide file tree
Showing 2 changed files with 71 additions and 54 deletions.
89 changes: 56 additions & 33 deletions cache/clock_cache.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
#include <cassert>
#include <cinttypes>
#include <cstddef>
#include <cstdint>
#include <exception>
#include <functional>
#include <numeric>
Expand Down Expand Up @@ -360,6 +361,18 @@ void ConstApplyToEntriesRange(const Func& func, const HandleImpl* begin,
}
}

constexpr uint32_t kStrictCapacityLimitBit = 1u << 31;

uint32_t SanitizeEncodeEecAndScl(int eviction_effort_cap,
bool strict_capacit_limit) {
eviction_effort_cap = std::max(int{1}, eviction_effort_cap);
eviction_effort_cap =
std::min(static_cast<int>(~kStrictCapacityLimitBit), eviction_effort_cap);
uint32_t eec_and_scl = static_cast<uint32_t>(eviction_effort_cap);
eec_and_scl |= strict_capacit_limit ? kStrictCapacityLimitBit : 0;
return eec_and_scl;
}

} // namespace

void ClockHandleBasicData::FreeData(MemoryAllocator* allocator) const {
Expand Down Expand Up @@ -389,17 +402,20 @@ HandleImpl* BaseClockTable::StandaloneInsert(

template <class Table>
typename Table::HandleImpl* BaseClockTable::CreateStandalone(
ClockHandleBasicData& proto, size_t capacity, bool strict_capacity_limit,
ClockHandleBasicData& proto, size_t capacity, uint32_t eec_and_scl,
bool allow_uncharged) {
Table& derived = static_cast<Table&>(*this);
typename Table::InsertState state;
derived.StartInsert(state);

const size_t total_charge = proto.GetTotalCharge();
if (strict_capacity_limit) {
// NOTE: we can use eec_and_scl as eviction_effort_cap below because
// strict_capacity_limit=true is supposed to disable the limit on eviction
// effort, and a large value effectively does that.
if (eec_and_scl & kStrictCapacityLimitBit) {
Status s = ChargeUsageMaybeEvictStrict<Table>(
total_charge, capacity,
/*need_evict_for_occupancy=*/false, state);
/*need_evict_for_occupancy=*/false, eec_and_scl, state);
if (!s.ok()) {
if (allow_uncharged) {
proto.total_charge = 0;
Expand All @@ -411,7 +427,7 @@ typename Table::HandleImpl* BaseClockTable::CreateStandalone(
// Case strict_capacity_limit == false
bool success = ChargeUsageMaybeEvictNonStrict<Table>(
total_charge, capacity,
/*need_evict_for_occupancy=*/false, state);
/*need_evict_for_occupancy=*/false, eec_and_scl, state);
if (!success) {
// Force the issue
usage_.FetchAddRelaxed(total_charge);
Expand All @@ -424,7 +440,7 @@ typename Table::HandleImpl* BaseClockTable::CreateStandalone(
template <class Table>
Status BaseClockTable::ChargeUsageMaybeEvictStrict(
size_t total_charge, size_t capacity, bool need_evict_for_occupancy,
typename Table::InsertState& state) {
uint32_t eviction_effort_cap, typename Table::InsertState& state) {
if (total_charge > capacity) {
return Status::MemoryLimit(
"Cache entry too large for a single cache shard: " +
Expand All @@ -450,7 +466,7 @@ Status BaseClockTable::ChargeUsageMaybeEvictStrict(
if (request_evict_charge > 0) {
EvictionData data;
static_cast<Table*>(this)->Evict(request_evict_charge, state, &data,
/*enforce_effort_cap=*/false);
eviction_effort_cap);
occupancy_.FetchSub(data.freed_count);
if (LIKELY(data.freed_charge > need_evict_charge)) {
assert(data.freed_count > 0);
Expand Down Expand Up @@ -480,7 +496,7 @@ Status BaseClockTable::ChargeUsageMaybeEvictStrict(
template <class Table>
inline bool BaseClockTable::ChargeUsageMaybeEvictNonStrict(
size_t total_charge, size_t capacity, bool need_evict_for_occupancy,
typename Table::InsertState& state) {
uint32_t eviction_effort_cap, typename Table::InsertState& state) {
// For simplicity, we consider that either the cache can accept the insert
// with no evictions, or we must evict enough to make (at least) enough
// space. It could lead to unnecessary failures or excessive evictions in
Expand Down Expand Up @@ -517,7 +533,7 @@ inline bool BaseClockTable::ChargeUsageMaybeEvictNonStrict(
EvictionData data;
if (need_evict_charge > 0) {
static_cast<Table*>(this)->Evict(need_evict_charge, state, &data,
/*enforce_effort_cap=*/true);
eviction_effort_cap);
// Deal with potential occupancy deficit
if (UNLIKELY(need_evict_for_occupancy) && data.freed_count == 0) {
assert(data.freed_charge == 0);
Expand Down Expand Up @@ -553,20 +569,20 @@ void BaseClockTable::TrackAndReleaseEvictedEntry(ClockHandle* h) {
MarkEmpty(*h);
}

bool BaseClockTable::IsEvictionEffortExceeded(const EvictionData& data) const {
bool IsEvictionEffortExceeded(const BaseClockTable::EvictionData& data,
uint32_t eviction_effort_cap) {
// Basically checks whether the ratio of useful effort to wasted effort is
// too low, with a start-up allowance for wasted effort before any useful
// effort.
return (data.freed_count + 1U) *
static_cast<uint64_t>(eviction_effort_cap_) <=
return (data.freed_count + 1U) * uint64_t{eviction_effort_cap} <=
data.seen_pinned_count;
}

template <class Table>
Status BaseClockTable::Insert(const ClockHandleBasicData& proto,
typename Table::HandleImpl** handle,
Cache::Priority priority, size_t capacity,
bool strict_capacity_limit) {
uint32_t eec_and_scl) {
using HandleImpl = typename Table::HandleImpl;
Table& derived = static_cast<Table&>(*this);

Expand All @@ -584,9 +600,12 @@ Status BaseClockTable::Insert(const ClockHandleBasicData& proto,
// strict_capacity_limit, but mostly pessimistic.
bool use_standalone_insert = false;
const size_t total_charge = proto.GetTotalCharge();
if (strict_capacity_limit) {
// NOTE: we can use eec_and_scl as eviction_effort_cap below because
// strict_capacity_limit=true is supposed to disable the limit on eviction
// effort, and a large value effectively does that.
if (eec_and_scl & kStrictCapacityLimitBit) {
Status s = ChargeUsageMaybeEvictStrict<Table>(
total_charge, capacity, need_evict_for_occupancy, state);
total_charge, capacity, need_evict_for_occupancy, eec_and_scl, state);
if (!s.ok()) {
// Revert occupancy
occupancy_.FetchSubRelaxed(1);
Expand All @@ -595,7 +614,7 @@ Status BaseClockTable::Insert(const ClockHandleBasicData& proto,
} else {
// Case strict_capacity_limit == false
bool success = ChargeUsageMaybeEvictNonStrict<Table>(
total_charge, capacity, need_evict_for_occupancy, state);
total_charge, capacity, need_evict_for_occupancy, eec_and_scl, state);
if (!success) {
// Revert occupancy
occupancy_.FetchSubRelaxed(1);
Expand Down Expand Up @@ -699,12 +718,11 @@ void BaseClockTable::TEST_ReleaseNMinus1(ClockHandle* h, size_t n) {
#endif

FixedHyperClockTable::FixedHyperClockTable(
size_t capacity, bool /*strict_capacity_limit*/,
CacheMetadataChargePolicy metadata_charge_policy,
size_t capacity, CacheMetadataChargePolicy metadata_charge_policy,
MemoryAllocator* allocator,
const Cache::EvictionCallback* eviction_callback, const uint32_t* hash_seed,
const Opts& opts)
: BaseClockTable(opts, metadata_charge_policy, allocator, eviction_callback,
: BaseClockTable(metadata_charge_policy, allocator, eviction_callback,
hash_seed),
length_bits_(CalcHashBits(capacity, opts.estimated_value_size,
metadata_charge_policy)),
Expand Down Expand Up @@ -1096,7 +1114,7 @@ inline void FixedHyperClockTable::ReclaimEntryUsage(size_t total_charge) {

inline void FixedHyperClockTable::Evict(size_t requested_charge, InsertState&,
EvictionData* data,
bool enforce_effort_cap) {
uint32_t eviction_effort_cap) {
// precondition
assert(requested_charge > 0);

Expand Down Expand Up @@ -1131,7 +1149,7 @@ inline void FixedHyperClockTable::Evict(size_t requested_charge, InsertState&,
if (old_clock_pointer >= max_clock_pointer) {
return;
}
if (enforce_effort_cap && IsEvictionEffortExceeded(*data)) {
if (IsEvictionEffortExceeded(*data, eviction_effort_cap)) {
eviction_effort_exceeded_count_.FetchAddRelaxed(1);
return;
}
Expand All @@ -1149,10 +1167,11 @@ ClockCacheShard<Table>::ClockCacheShard(
const Cache::EvictionCallback* eviction_callback, const uint32_t* hash_seed,
const typename Table::Opts& opts)
: CacheShardBase(metadata_charge_policy),
table_(capacity, strict_capacity_limit, metadata_charge_policy, allocator,
eviction_callback, hash_seed, opts),
table_(capacity, metadata_charge_policy, allocator, eviction_callback,
hash_seed, opts),
capacity_(capacity),
strict_capacity_limit_(strict_capacity_limit) {
eec_and_scl_(SanitizeEncodeEecAndScl(opts.eviction_effort_cap,
strict_capacity_limit)) {
// Initial charge metadata should not exceed capacity
assert(table_.GetUsage() <= capacity_.LoadRelaxed() ||
capacity_.LoadRelaxed() < sizeof(HandleImpl));
Expand Down Expand Up @@ -1228,7 +1247,11 @@ void ClockCacheShard<Table>::SetCapacity(size_t capacity) {
template <class Table>
void ClockCacheShard<Table>::SetStrictCapacityLimit(
bool strict_capacity_limit) {
strict_capacity_limit_.StoreRelaxed(strict_capacity_limit);
if (strict_capacity_limit) {
eec_and_scl_.FetchOrRelaxed(kStrictCapacityLimitBit);
} else {
eec_and_scl_.FetchAndRelaxed(~kStrictCapacityLimitBit);
}
// next Insert will take care of any necessary evictions
}

Expand All @@ -1250,7 +1273,7 @@ Status ClockCacheShard<Table>::Insert(const Slice& key,
proto.total_charge = charge;
return table_.template Insert<Table>(proto, handle, priority,
capacity_.LoadRelaxed(),
strict_capacity_limit_.LoadRelaxed());
eec_and_scl_.LoadRelaxed());
}

template <class Table>
Expand All @@ -1265,9 +1288,9 @@ typename Table::HandleImpl* ClockCacheShard<Table>::CreateStandalone(
proto.value = obj;
proto.helper = helper;
proto.total_charge = charge;
return table_.template CreateStandalone<Table>(
proto, capacity_.LoadRelaxed(), strict_capacity_limit_.LoadRelaxed(),
allow_uncharged);
return table_.template CreateStandalone<Table>(proto, capacity_.LoadRelaxed(),
eec_and_scl_.LoadRelaxed(),
allow_uncharged);
}

template <class Table>
Expand Down Expand Up @@ -1930,12 +1953,11 @@ class AutoHyperClockTable::ChainRewriteLock {
};

AutoHyperClockTable::AutoHyperClockTable(
size_t capacity, bool /*strict_capacity_limit*/,
CacheMetadataChargePolicy metadata_charge_policy,
size_t capacity, CacheMetadataChargePolicy metadata_charge_policy,
MemoryAllocator* allocator,
const Cache::EvictionCallback* eviction_callback, const uint32_t* hash_seed,
const Opts& opts)
: BaseClockTable(opts, metadata_charge_policy, allocator, eviction_callback,
: BaseClockTable(metadata_charge_policy, allocator, eviction_callback,
hash_seed),
array_(MemMapping::AllocateLazyZeroed(
sizeof(HandleImpl) * CalcMaxUsableLength(capacity,
Expand Down Expand Up @@ -3430,7 +3452,8 @@ void AutoHyperClockTable::EraseUnRefEntries() {
}

void AutoHyperClockTable::Evict(size_t requested_charge, InsertState& state,
EvictionData* data, bool enforce_effort_cap) {
EvictionData* data,
uint32_t eviction_effort_cap) {
// precondition
assert(requested_charge > 0);

Expand Down Expand Up @@ -3522,7 +3545,7 @@ void AutoHyperClockTable::Evict(size_t requested_charge, InsertState& state,
return;
}

if (enforce_effort_cap && IsEvictionEffortExceeded(*data)) {
if (IsEvictionEffortExceeded(*data, eviction_effort_cap)) {
eviction_effort_exceeded_count_.FetchAddRelaxed(1);
return;
}
Expand Down
36 changes: 15 additions & 21 deletions cache/clock_cache.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -377,37 +377,31 @@ class BaseClockTable {
public:
struct BaseOpts {
explicit BaseOpts(int _eviction_effort_cap)
: eviction_effort_cap(_eviction_effort_cap) {
eviction_effort_cap = std::max(int{1}, _eviction_effort_cap);
// Avoid overflow in IsEvictionEffortExceeded
eviction_effort_cap = std::min(INT32_MAX, eviction_effort_cap);
}
: eviction_effort_cap(_eviction_effort_cap) {}
explicit BaseOpts(const HyperClockCacheOptions& opts)
: BaseOpts(opts.eviction_effort_cap) {}
int eviction_effort_cap;
};

BaseClockTable(const BaseOpts& opts,
CacheMetadataChargePolicy metadata_charge_policy,
BaseClockTable(CacheMetadataChargePolicy metadata_charge_policy,
MemoryAllocator* allocator,
const Cache::EvictionCallback* eviction_callback,
const uint32_t* hash_seed)
: metadata_charge_policy_(metadata_charge_policy),
allocator_(allocator),
eviction_callback_(*eviction_callback),
eviction_effort_cap_(opts.eviction_effort_cap),
hash_seed_(*hash_seed) {}

template <class Table>
typename Table::HandleImpl* CreateStandalone(ClockHandleBasicData& proto,
size_t capacity,
bool strict_capacity_limit,
uint32_t eec_and_scl,
bool allow_uncharged);

template <class Table>
Status Insert(const ClockHandleBasicData& proto,
typename Table::HandleImpl** handle, Cache::Priority priority,
size_t capacity, bool strict_capacity_limit);
size_t capacity, uint32_t eec_and_scl);

void Ref(ClockHandle& handle);

Expand All @@ -433,8 +427,6 @@ class BaseClockTable {

void TrackAndReleaseEvictedEntry(ClockHandle* h);

bool IsEvictionEffortExceeded(const EvictionData& data) const;

#ifndef NDEBUG
// Acquire N references
void TEST_RefN(ClockHandle& handle, size_t n);
Expand All @@ -458,6 +450,7 @@ class BaseClockTable {
template <class Table>
Status ChargeUsageMaybeEvictStrict(size_t total_charge, size_t capacity,
bool need_evict_for_occupancy,
uint32_t eviction_effort_cap,
typename Table::InsertState& state);

// Helper for updating `usage_` for new entry with given `total_charge`
Expand All @@ -471,7 +464,9 @@ class BaseClockTable {
template <class Table>
bool ChargeUsageMaybeEvictNonStrict(size_t total_charge, size_t capacity,
bool need_evict_for_occupancy,
uint32_t eviction_effort_cap,
typename Table::InsertState& state);

protected: // data
// We partition the following members into different cache lines
// to avoid false sharing among Lookup, Release, Erase and Insert
Expand All @@ -482,6 +477,7 @@ class BaseClockTable {
RelaxedAtomic<uint64_t> clock_pointer_{};

// Counter for number of times we yield to wait on another thread.
// It is normal for this to occur rarely in normal operation.
// (Relaxed: a simple stat counter.)
RelaxedAtomic<uint64_t> yield_count_{};

Expand Down Expand Up @@ -510,9 +506,6 @@ class BaseClockTable {
// A reference to Cache::eviction_callback_
const Cache::EvictionCallback& eviction_callback_;

// See HyperClockCacheOptions::eviction_effort_cap
int eviction_effort_cap_;

// A reference to ShardedCacheBase::hash_seed_
const uint32_t& hash_seed_;
};
Expand Down Expand Up @@ -558,7 +551,7 @@ class FixedHyperClockTable : public BaseClockTable {
size_t estimated_value_size;
};

FixedHyperClockTable(size_t capacity, bool strict_capacity_limit,
FixedHyperClockTable(size_t capacity,
CacheMetadataChargePolicy metadata_charge_policy,
MemoryAllocator* allocator,
const Cache::EvictionCallback* eviction_callback,
Expand All @@ -581,7 +574,7 @@ class FixedHyperClockTable : public BaseClockTable {
// requested_charge. Returns how much is evicted, which could be less
// if it appears impossible to evict the requested amount without blocking.
void Evict(size_t requested_charge, InsertState& state, EvictionData* data,
bool enforce_effort_cap);
uint32_t eviction_effort_cap);

HandleImpl* Lookup(const UniqueId64x2& hashed_key);

Expand Down Expand Up @@ -848,7 +841,7 @@ class AutoHyperClockTable : public BaseClockTable {
size_t min_avg_value_size;
};

AutoHyperClockTable(size_t capacity, bool strict_capacity_limit,
AutoHyperClockTable(size_t capacity,
CacheMetadataChargePolicy metadata_charge_policy,
MemoryAllocator* allocator,
const Cache::EvictionCallback* eviction_callback,
Expand Down Expand Up @@ -876,7 +869,7 @@ class AutoHyperClockTable : public BaseClockTable {
// requested_charge. Returns how much is evicted, which could be less
// if it appears impossible to evict the requested amount without blocking.
void Evict(size_t requested_charge, InsertState& state, EvictionData* data,
bool enforce_effort_cap);
uint32_t eviction_effort_cap);

HandleImpl* Lookup(const UniqueId64x2& hashed_key);

Expand Down Expand Up @@ -1114,9 +1107,10 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShardBase {
// (Relaxed: eventual consistency/update is OK)
RelaxedAtomic<size_t> capacity_;

// Whether to reject insertion if cache reaches its full capacity.
// Encodes eviction_effort_cap (bottom 31 bits) and strict_capacity_limit
// (top bit). See HyperClockCacheOptions::eviction_effort_cap etc.
// (Relaxed: eventual consistency/update is OK)
RelaxedAtomic<bool> strict_capacity_limit_;
RelaxedAtomic<uint32_t> eec_and_scl_;
}; // class ClockCacheShard

template <class Table>
Expand Down

0 comments on commit 8ed282b

Please sign in to comment.