Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable large cache per column family #9127

Closed
wants to merge 10 commits into from
Closed

Enable large cache per column family #9127

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
f367972
add optional to allow large cache in Column
gfukushima Feb 17, 2025
25b84f0
set larger cache for finalized blocks and blobs columns
gfukushima Feb 17, 2025
38ee9ed
add larger cache config definition
gfukushima Feb 17, 2025
772a0a2
create Column family descriptor based on each column
gfukushima Feb 17, 2025
7dda8f8
create the cf descriptor within the try catch
gfukushima Feb 17, 2025
7bd2bec
spotless
gfukushima Feb 17, 2025
b5c7b0c
default should be rocks default
gfukushima Feb 19, 2025
4a2d322
Merge branch 'master' into enable-large-cache-per-cf
gfukushima Feb 19, 2025
ab733b0
log large cache and column being set
gfukushima Feb 19, 2025
58a1a2f
spoless
gfukushima Feb 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,8 @@ public class KvStoreConfiguration {

public static final int DEFAULT_MAX_BACKGROUND_JOBS = 6;
public static final int DEFAULT_BACKGROUND_THREAD_COUNT = 6;
public static final long DEFAULT_CACHE_CAPACITY = 128 << 20; // 128MB
public static final long DEFAULT_CACHE_CAPACITY = 32 << 20;
public static final long LARGER_CACHE_CAPACITY = 128 << 20; // 128MB
public static final long DEFAULT_WRITE_BUFFER_CAPACITY = 128 << 20;
private static final boolean DEFAULT_OPTIMISE_FOR_SMALL_DB = false;

Expand Down Expand Up @@ -164,6 +165,10 @@ public long getCacheCapacity() {
return cacheCapacity;
}

public long getLargerCacheCapacity() {
return LARGER_CACHE_CAPACITY;
}

public long getWriteBufferCapacity() {
return writeBufferCapacity;
}
Expand Down
21 changes: 19 additions & 2 deletions storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/schema/KvStoreColumn.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,28 +16,41 @@
import static tech.pegasys.teku.infrastructure.unsigned.ByteUtil.toByteExact;

import java.util.Objects;
import java.util.Optional;
import org.apache.tuweni.bytes.Bytes;
import tech.pegasys.teku.storage.server.kvstore.serialization.KvStoreSerializer;

public class KvStoreColumn<TKey, TValue> {
private final Bytes id;
private final KvStoreSerializer<TKey> keySerializer;
private final KvStoreSerializer<TValue> valueSerializer;
private final Optional<Boolean> isLargerCacheAvalilable;

private KvStoreColumn(
final Bytes id,
final KvStoreSerializer<TKey> keySerializer,
final KvStoreSerializer<TValue> valueSerializer) {
final KvStoreSerializer<TValue> valueSerializer,
final Optional<Boolean> isLargerCacheAvalilable) {
this.id = id;
this.keySerializer = keySerializer;
this.valueSerializer = valueSerializer;
this.isLargerCacheAvalilable = isLargerCacheAvalilable;
}

public static <K, V> KvStoreColumn<K, V> create(
final int id,
final KvStoreSerializer<K> keySerializer,
final KvStoreSerializer<V> valueSerializer) {
return new KvStoreColumn<>(asColumnId(id), keySerializer, valueSerializer);
return new KvStoreColumn<>(asColumnId(id), keySerializer, valueSerializer, Optional.empty());
}

public static <K, V> KvStoreColumn<K, V> create(
final int id,
final KvStoreSerializer<K> keySerializer,
final KvStoreSerializer<V> valueSerializer,
final Boolean isLargerCacheAvalilable) {
return new KvStoreColumn<>(
asColumnId(id), keySerializer, valueSerializer, Optional.of(isLargerCacheAvalilable));
}

public static Bytes asColumnId(final int id) {
Expand All @@ -56,6 +69,10 @@ public KvStoreSerializer<TValue> getValueSerializer() {
return valueSerializer;
}

public Optional<Boolean> getIsLargerCacheAvalilable() {
return isLargerCacheAvalilable;
}

@Override
public boolean equals(final Object o) {
if (this == o) {
Expand Down
6 changes: 4 additions & 2 deletions ...c/main/java/tech/pegasys/teku/storage/server/kvstore/schema/V6SchemaCombinedSnapshot.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,8 @@ private V6SchemaCombinedSnapshot(final Spec spec, final int finalizedOffset) {
KvStoreColumn.create(
finalizedOffset + 2,
UINT64_SERIALIZER,
KvStoreSerializer.createSignedBlockSerializer(spec));
KvStoreSerializer.createSignedBlockSerializer(spec),
true);
finalizedStatesBySlot =
KvStoreColumn.create(
finalizedOffset + 3, UINT64_SERIALIZER, KvStoreSerializer.createStateSerializer(spec));
Expand All @@ -74,7 +75,8 @@ private V6SchemaCombinedSnapshot(final Spec spec, final int finalizedOffset) {
KvStoreColumn.create(
finalizedOffset + 12,
SLOT_AND_BLOCK_ROOT_AND_BLOB_INDEX_KEY_SERIALIZER,
BYTES_SERIALIZER);
BYTES_SERIALIZER,
true);

nonCanonicalBlobSidecarBySlotRootBlobIndex =
KvStoreColumn.create(
Expand Down
6 changes: 4 additions & 2 deletions .../main/java/tech/pegasys/teku/storage/server/kvstore/schema/V6SchemaCombinedTreeState.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,8 @@ public V6SchemaCombinedTreeState(final Spec spec) {
KvStoreColumn.create(
V6_FINALIZED_OFFSET + 7,
UINT64_SERIALIZER,
KvStoreSerializer.createSignedBlockSerializer(spec));
KvStoreSerializer.createSignedBlockSerializer(spec),
true);
nonCanonicalBlocksByRoot =
KvStoreColumn.create(
V6_FINALIZED_OFFSET + 8,
Expand All @@ -82,7 +83,8 @@ public V6SchemaCombinedTreeState(final Spec spec) {
KvStoreColumn.create(
finalizedOffset + 14,
SLOT_AND_BLOCK_ROOT_AND_BLOB_INDEX_KEY_SERIALIZER,
BYTES_SERIALIZER);
BYTES_SERIALIZER,
true);
nonCanonicalBlobSidecarBySlotRootBlobIndex =
KvStoreColumn.create(
finalizedOffset + 15,
Expand Down
82 changes: 65 additions & 17 deletions storage/src/main/java/tech/pegasys/teku/storage/server/rocksdb/RocksDbInstanceFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.tuweni.bytes.Bytes;
import org.hyperledger.besu.plugin.services.MetricsSystem;
import org.hyperledger.besu.plugin.services.metrics.MetricCategory;
Expand Down Expand Up @@ -56,6 +58,8 @@ public class RocksDbInstanceFactory {
RocksDbUtil.loadNativeLibrary();
}

private static final Logger LOG = LogManager.getLogger();

public static KvStoreAccessor create(
final MetricsSystem metricsSystem,
final MetricCategory metricCategory,
Expand All @@ -75,19 +79,16 @@ public static KvStoreAccessor create(
final TransactionDBOptions txOptions = new TransactionDBOptions();
final RocksDbStats rocksDbStats = new RocksDbStats(metricsSystem, metricCategory);
final DBOptions dbOptions = createDBOptions(configuration, rocksDbStats.getStats());
final LRUCache blockCache = new LRUCache(configuration.getCacheCapacity());
final ColumnFamilyOptions columnFamilyOptions =
createColumnFamilyOptions(configuration, blockCache);

final List<AutoCloseable> resources =
new ArrayList<>(
List.of(txOptions, dbOptions, columnFamilyOptions, rocksDbStats, blockCache));
new ArrayList<>(List.of(txOptions, dbOptions, rocksDbStats));

List<ColumnFamilyDescriptor> columnDescriptors =
createColumnFamilyDescriptors(columns, deletedColumns, columnFamilyOptions);
Map<Bytes, KvStoreColumn<?, ?>> columnsById =
columns.stream().collect(Collectors.toMap(KvStoreColumn::getId, Function.identity()));

try {
final List<ColumnFamilyDescriptor> columnDescriptors =
createColumnFamilyDescriptors(columns, deletedColumns, configuration);
// columnHandles will be filled when the db is opened
final List<ColumnFamilyHandle> columnHandles = new ArrayList<>(columnDescriptors.size());
final TransactionDB db =
Expand Down Expand Up @@ -164,24 +165,71 @@ private static DBOptions createDBOptions(
}

private static ColumnFamilyOptions createColumnFamilyOptions(
final KvStoreConfiguration configuration, final Cache cache) {
return new ColumnFamilyOptions()
.setCompressionType(configuration.getCompressionType())
.setBottommostCompressionType(configuration.getBottomMostCompressionType())
.setLevelCompactionDynamicLevelBytes(true)
.setTableFormatConfig(createBlockBasedTableConfig(cache));
final KvStoreConfiguration configuration, final KvStoreColumn<?, ?> column) {
final ColumnFamilyOptions cfOptions;
try {
final LRUCache cache =
column
.getIsLargerCacheAvalilable()
.map(
isLarger -> {
if (isLarger) {
LOG.info("Using larger cache for column {}", column.getId().toHexString());
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

NIT: not sure if we need this as a info. It seems to be pretty low level and I belive we are happy that most users are gonna use default values anyway.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was just for testing purposes.

return new LRUCache(configuration.getLargerCacheCapacity());
} else {
return new LRUCache(configuration.getCacheCapacity());
}
})
.orElse(new LRUCache(configuration.getCacheCapacity()));
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If getIsLargerCacheAvalilable() is Optional.empty(), it means that we are on the default case, right? If we don't want to change the existing behaviour we should use the large capacity (matching the old value) when getIsLargerCacheAvalilable() is empty. Maybe I am missing something but it does look like to me that we would be changing every cache to 32MB.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So long story short the default value used to be 8MB, which is low. We've changed to 128MB as a first step to get the db performing better and we're trying to narrow down what columns need larger caches, that being said the default should be something lower than 128MB, Rocks default is 32MB (as of release 8.x) which is what I believe we should be using by default as well, 8MB was set long time ago and I don't think is suitable anymore for the current needs.

cfOptions =
new ColumnFamilyOptions()
.setCompressionType(configuration.getCompressionType())
.setBottommostCompressionType(configuration.getBottomMostCompressionType())
.setLevelCompactionDynamicLevelBytes(true)
.setTableFormatConfig(createBlockBasedTableConfig(cache));
} catch (Exception e) {
throw new RuntimeException("Error creating column family options", e);
}
return cfOptions;
}

private static ColumnFamilyOptions createColumnFamilyOptions(
final KvStoreConfiguration configuration) {
final ColumnFamilyOptions cfOptions;
try {
final LRUCache cache = new LRUCache(configuration.getCacheCapacity());
cfOptions =
new ColumnFamilyOptions()
.setCompressionType(configuration.getCompressionType())
.setBottommostCompressionType(configuration.getBottomMostCompressionType())
.setLevelCompactionDynamicLevelBytes(true)
.setTableFormatConfig(createBlockBasedTableConfig(cache));
} catch (Exception e) {
throw new RuntimeException("Error creating column family options", e);
}
return cfOptions;
}

private static List<ColumnFamilyDescriptor> createColumnFamilyDescriptors(
final Collection<KvStoreColumn<?, ?>> columns,
final Collection<Bytes> deletedColumns,
final ColumnFamilyOptions columnFamilyOptions) {
final KvStoreConfiguration configuration) {
final List<ColumnFamilyDescriptor> columnDescriptors =
Stream.concat(columns.stream().map(KvStoreColumn::getId), deletedColumns.stream())
.map(id -> new ColumnFamilyDescriptor(id.toArrayUnsafe(), columnFamilyOptions))
columns.stream()
.map(
column ->
new ColumnFamilyDescriptor(
column.getId().toArrayUnsafe(),
createColumnFamilyOptions(configuration, column)))
.collect(Collectors.toCollection(ArrayList::new));
columnDescriptors.addAll(
deletedColumns.stream()
.map(Bytes::toArrayUnsafe)
.map(id -> new ColumnFamilyDescriptor(id, createColumnFamilyOptions(configuration)))
.toList());
columnDescriptors.add(
new ColumnFamilyDescriptor(Schema.DEFAULT_COLUMN_ID.toArrayUnsafe(), columnFamilyOptions));
new ColumnFamilyDescriptor(
Schema.DEFAULT_COLUMN_ID.toArrayUnsafe(), createColumnFamilyOptions(configuration)));
return Collections.unmodifiableList(columnDescriptors);
}

Expand Down