Skip to content

Commit

Permalink
Merge branch 'master' into 7356-2-peer-count-remote
Browse files Browse the repository at this point in the history
  • Loading branch information
mehdi-aouadi authored Jul 4, 2024
2 parents d07558c + 3696386 commit 79b277f
Show file tree
Hide file tree
Showing 27 changed files with 65 additions and 45 deletions.
13 changes: 4 additions & 9 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,16 @@

## Upcoming Breaking Changes

- Next release will introduce a breaking change to Teku's metrics. This is due to some metrics changing names after a library upgrade.
We recommend all users of the `Teku - Detailed` dashboard to upgrade to version [Revision 12](https://grafana.com/api/dashboards/16737/revisions/12/download)
as soon as possible. Documentation with all metrics that have been renamed will be provided.
- Next release will require Java 21. The current release is compatible, please consider upgrading before the next release.
- From the next release, you will need to explicitly set `--data-storage-mode=(prune|archive)` unless you're using minimal data-storage-mode (which is the default behaviour).

## Current Releases

For information on changes in released versions of Teku, see
the [releases page](https://github.com/Consensys/teku/releases).

## Unreleased Changes

### Breaking Changes

- Updated counter metrics to incorporate the suffix `_total`. If you are using a custom dashboard to monitor Teku metrics, you might need to update the metrics manually when breaking changes are introduced. For more information, see [Update metrics](../../how-to/monitor/update-metrics.md).

### Additions and Improvements

- Updated bootnodes for Sepolia network.

### Bug Fixes
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,12 @@ public class ReferenceTestFinder {
Path.of("src", "referenceTest", "resources", "consensus-spec-tests", "tests");
private static final List<String> SUPPORTED_FORKS =
List.of(
TestFork.PHASE0, TestFork.ALTAIR, TestFork.BELLATRIX, TestFork.CAPELLA, TestFork.DENEB);
TestFork.PHASE0,
TestFork.ALTAIR,
TestFork.BELLATRIX,
TestFork.CAPELLA,
TestFork.DENEB,
TestFork.ELECTRA);

@MustBeClosed
public static Stream<TestDefinition> findReferenceTests() throws IOException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ public static ExecutionLayerManagerImpl create(
final LabelledMetric<Counter> executionPayloadSourceCounter =
metricsSystem.createLabelledCounter(
TekuMetricCategory.BEACON,
"execution_payload_source",
"execution_payload_source_total",
"Counter recording the source of the execution payload during block production",
"source",
"fallback_reason");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -569,7 +569,7 @@ private void verifyEngineCalled(
private void verifySourceCounter(final Source source, final FallbackReason reason) {
final long actualCount =
stubMetricsSystem
.getCounter(TekuMetricCategory.BEACON, "execution_payload_source")
.getCounter(TekuMetricCategory.BEACON, "execution_payload_source_total")
.getValue(source.toString(), reason.toString());
assertThat(actualCount).isOne();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -858,7 +858,7 @@ private void verifyEngineCalled(
private void verifySourceCounter(final Source source, final FallbackReason reason) {
final long actualCount =
stubMetricsSystem
.getCounter(TekuMetricCategory.BEACON, "execution_payload_source")
.getCounter(TekuMetricCategory.BEACON, "execution_payload_source_total")
.getValue(source.toString(), reason.toString());
assertThat(actualCount).isOne();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -781,10 +781,13 @@ private Builder applySepoliaNetworkDefaults() {
"https://github.com/eth-clients/merge-testnets/raw/9c873ab67b902aa676370a549129e5e91013afa3/sepolia/genesis.ssz")
.discoveryBootnodes(
// EF bootnodes
"enr:-Iq4QMCTfIMXnow27baRUb35Q8iiFHSIDBJh6hQM5Axohhf4b6Kr_cOCu0htQ5WvVqKvFgY28893DHAg8gnBAXsAVqmGAX53x8JggmlkgnY0gmlwhLKAlv6Jc2VjcDI1NmsxoQK6S-Cii_KmfFdUJL2TANL3ksaKUnNXvTCv1tLwXs0QgIN1ZHCCIyk",
"enr:-KG4QE5OIg5ThTjkzrlVF32WT_-XT14WeJtIz2zoTqLLjQhYAmJlnk4ItSoH41_2x0RX0wTFIe5GgjRzU2u7Q1fN4vADhGV0aDKQqP7o7pAAAHAyAAAAAAAAAIJpZIJ2NIJpcISlFsStiXNlY3AyNTZrMaEC-Rrd_bBZwhKpXzFCrStKp1q_HmGOewxY3KwM8ofAj_ODdGNwgiMog3VkcIIjKA",
"enr:-Ku4QDZ_rCowZFsozeWr60WwLgOfHzv1Fz2cuMvJqN5iJzLxKtVjoIURY42X_YTokMi3IGstW5v32uSYZyGUXj9Q_IECh2F0dG5ldHOIAAAAAAAAAACEZXRoMpCo_ujukAAAaf__________gmlkgnY0gmlwhIpEe5iJc2VjcDI1NmsxoQNHTpFdaNSCEWiN_QqT396nb0PzcUpLe3OVtLph-AciBYN1ZHCCIy0",
"enr:-Ku4QHRyRwEPT7s0XLYzJ_EeeWvZTXBQb4UCGy1F_3m-YtCNTtDlGsCMr4UTgo4uR89pv11uM-xq4w6GKfKhqU31hTgCh2F0dG5ldHOIAAAAAAAAAACEZXRoMpCo_ujukAAAaf__________gmlkgnY0gmlwhIrFM7WJc2VjcDI1NmsxoQI4diTwChN3zAAkarf7smOHCdFb1q3DSwdiQ_Lc_FdzFIN1ZHCCIy0",
"enr:-Ku4QOkvvf0u5Hg4-HhY-SJmEyft77G5h3rUM8VF_e-Hag5cAma3jtmFoX4WElLAqdILCA-UWFRN1ZCDJJVuEHrFeLkDh2F0dG5ldHOIAAAAAAAAAACEZXRoMpCo_ujukAAAaf__________gmlkgnY0gmlwhJK-AWeJc2VjcDI1NmsxoQLFcT5VE_NMiIC8Ll7GypWDnQ4UEmuzD7hF_Hf4veDJwIN1ZHCCIy0",
"enr:-Ku4QH6tYsHKITYeHUu5kdfXgEZWI18EWk_2RtGOn1jBPlx2UlS_uF3Pm5Dx7tnjOvla_zs-wwlPgjnEOcQDWXey51QCh2F0dG5ldHOIAAAAAAAAAACEZXRoMpCo_ujukAAAaf__________gmlkgnY0gmlwhIs7Mc6Jc2VjcDI1NmsxoQIET4Mlv9YzhrYhX_H9D7aWMemUrvki6W4J2Qo0YmFMp4N1ZHCCIy0",
"enr:-Ku4QDmz-4c1InchGitsgNk4qzorWMiFUoaPJT4G0IiF8r2UaevrekND1o7fdoftNucirj7sFFTTn2-JdC2Ej0p1Mn8Ch2F0dG5ldHOIAAAAAAAAAACEZXRoMpCo_ujukAAAaf__________gmlkgnY0gmlwhKpA-liJc2VjcDI1NmsxoQMpHP5U1DK8O_JQU6FadmWbE42qEdcGlllR8HcSkkfWq4N1ZHCCIy0",
// Teku bootnode
"enr:-Ly4QFoZTWR8ulxGVsWydTNGdwEESueIdj-wB6UmmjUcm-AOPxnQi7wprzwcdo7-1jBW_JxELlUKJdJES8TDsbl1EdNlh2F0dG5ldHOI__78_v2bsV-EZXRoMpA2-lATkAAAcf__________gmlkgnY0gmlwhBLYJjGJc2VjcDI1NmsxoQI0gujXac9rMAb48NtMqtSTyHIeNYlpjkbYpWJw46PmYYhzeW5jbmV0cw-DdGNwgiMog3VkcIIjKA",
"enr:-KO4QP7MmB3juk8rUjJHcUoxZDU9Np4FlW0HyDEGIjSO7GD9PbSsabu7713cWSUWKDkxIypIXg1A-6lG7ySRGOMZHeGCAmuEZXRoMpDTH2GRkAAAc___________gmlkgnY0gmlwhBSoyGOJc2VjcDI1NmsxoQNta5b_bexSSwwrGW2Re24MjfMntzFd0f2SAxQtMj3ueYN0Y3CCIyiDdWRwgiMo",
// Another bootnode
"enr:-L64QC9Hhov4DhQ7mRukTOz4_jHm4DHlGL726NWH4ojH1wFgEwSin_6H95Gs6nW2fktTWbPachHJ6rUFu0iJNgA0SB2CARqHYXR0bmV0c4j__________4RldGgykDb6UBOQAABx__________-CaWSCdjSCaXCEA-2vzolzZWNwMjU2azGhA17lsUg60R776rauYMdrAz383UUgESoaHEzMkvm4K6k6iHN5bmNuZXRzD4N0Y3CCIyiDdWRwgiMo");
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -218,8 +218,6 @@ private static List<Withdrawal> getExpectedWithdrawals(
specConfigCapella.getMaxValidatorsPerWithdrawalSweep();
final int bound = Math.min(validatorCount, maxValidatorsPerWithdrawalsSweep);

final UInt64 maxEffectiveBalance = specConfigCapella.getMaxEffectiveBalance();

UInt64 withdrawalIndex =
partialWithdrawals.isEmpty()
? preState.getNextWithdrawalIndex()
Expand All @@ -228,7 +226,7 @@ private static List<Withdrawal> getExpectedWithdrawals(

for (int i = 0; i < bound; i++) {
final Validator validator = validators.get(validatorIndex);
if (predicates.hasEth1WithdrawalCredential(validator)) {
if (predicates.hasExecutionWithdrawalCredential(validator)) {
final UInt64 balance = balances.get(validatorIndex).get();

if (predicates.isFullyWithdrawableValidatorCredentialsChecked(validator, balance, epoch)) {
Expand All @@ -246,7 +244,7 @@ private static List<Withdrawal> getExpectedWithdrawals(
withdrawalIndex,
UInt64.valueOf(validatorIndex),
new Bytes20(validator.getWithdrawalCredentials().slice(12)),
balance.minus(maxEffectiveBalance)));
balance.minusMinZero(predicates.getValidatorMaxEffectiveBalance(validator))));
withdrawalIndex = withdrawalIndex.increment();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,16 @@ public static boolean isEth1WithdrawalCredential(final Bytes32 withdrawalCredent
return withdrawalCredentials.get(0) == ETH1_ADDRESS_WITHDRAWAL_BYTE;
}

/**
* has_execution_withdrawal_credential
*
* @param validator
* @return
*/
public boolean hasExecutionWithdrawalCredential(final Validator validator) {
return hasEth1WithdrawalCredential(validator);
}

/**
* Get the execution address from a validator's withdrawal credentials. This method does not check
* if the validator has the correct type of withdrawal credentials (e.g. prefixes 0x01 and 0x02).
Expand Down Expand Up @@ -150,7 +160,7 @@ public boolean isPartiallyWithdrawableValidator(final Validator validator, final

public boolean isPartiallyWithdrawableValidatorEth1CredentialsChecked(
final Validator validator, final UInt64 balance) {
final UInt64 maxEffectiveBalance = specConfig.getMaxEffectiveBalance();
final UInt64 maxEffectiveBalance = getValidatorMaxEffectiveBalance(validator);
final boolean hasMaxEffectiveBalance =
validator.getEffectiveBalance().equals(maxEffectiveBalance);
final boolean hasExcessBalance = balance.isGreaterThan(maxEffectiveBalance);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,9 @@ public boolean isFullyWithdrawableValidator(
* @param validator
* @return
*/
@Override
public boolean hasExecutionWithdrawalCredential(final Validator validator) {
return hasCompoundingWithdrawalCredential(validator) || hasEth1WithdrawalCredential(validator);
return hasEth1WithdrawalCredential(validator) || hasCompoundingWithdrawalCredential(validator);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@
import org.hyperledger.besu.config.GenesisConfigFile;
import org.hyperledger.besu.config.GenesisConfigOptions;
import org.hyperledger.besu.datatypes.Wei;
import org.hyperledger.besu.ethereum.chain.BadBlockManager;
import org.hyperledger.besu.ethereum.chain.GenesisState;
import org.hyperledger.besu.ethereum.core.Block;
import org.hyperledger.besu.ethereum.core.BlockHeader;
import org.hyperledger.besu.ethereum.core.MiningParameters;
import org.hyperledger.besu.ethereum.mainnet.MainnetProtocolSchedule;
import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
import tech.pegasys.teku.infrastructure.bytes.Bytes20;
Expand All @@ -35,8 +37,10 @@ public static ExecutionPayloadHeader createPayloadForBesuGenesis(
final GenesisConfigFile configFile = GenesisConfigFile.fromConfig(genesisConfigFile);
final GenesisConfigOptions genesisConfigOptions =
configFile.getConfigOptions(Collections.emptyMap());
final BadBlockManager badBlockManager = new BadBlockManager();
final ProtocolSchedule protocolSchedule =
MainnetProtocolSchedule.fromConfig(genesisConfigOptions);
MainnetProtocolSchedule.fromConfig(
genesisConfigOptions, MiningParameters.MINING_DISABLED, badBlockManager);
final GenesisState genesisState = GenesisState.fromConfig(configFile, protocolSchedule);
final Block genesisBlock = genesisState.getBlock();
final BlockHeader header = genesisBlock.getHeader();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ public MappedOperationPool(
validationReasonCounter =
metricsSystem.createLabelledCounter(
TekuMetricCategory.BEACON,
OPERATION_POOL_SIZE_VALIDATION_REASON + metricType,
OPERATION_POOL_SIZE_VALIDATION_REASON + metricType + "_total",
"Total number of attempts to add an operation to the pool, broken down by validation result",
"result");

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ private SimpleOperationPool(
validationReasonCounter =
metricsSystem.createLabelledCounter(
TekuMetricCategory.BEACON,
OPERATION_POOL_SIZE_VALIDATION_REASON + metricType,
OPERATION_POOL_SIZE_VALIDATION_REASON + metricType + "_total",
"Total number of attempts to add an operation to the pool, broken down by validation result",
"result");
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ public PoolFactory(final MetricsSystem metricsSystem) {
this.blockBlobSidecarsTrackersPoolStats =
metricsSystem.createLabelledCounter(
TekuMetricCategory.BEACON,
"block_blobs_trackers_pool_stats",
"block_blobs_trackers_pool_stats_total",
"Block-blobs trackers pool statistics",
"type",
"subtype");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,12 +81,12 @@ public class AggregatingSignatureVerificationService extends SignatureVerificati
batchCounter =
metricsSystem.createCounter(
TekuMetricCategory.EXECUTOR,
"signature_verifications_batch_count",
"signature_verifications_batch_count_total",
"Reports the number of verification batches processed");
taskCounter =
metricsSystem.createCounter(
TekuMetricCategory.EXECUTOR,
"signature_verifications_task_count",
"signature_verifications_task_count_total",
"Reports the number of individual verification tasks processed");
batchSizeHistogram =
MetricsHistogram.create(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1066,7 +1066,8 @@ private static BlobIdentifier blobIdentifierFromBlobSidecar(final BlobSidecar bl
}

private void assertStats(final String type, final String subType, final double count) {
assertThat(getMetricsValues("block_blobs_trackers_pool_stats").get(List.of(type, subType)))
assertThat(
getMetricsValues("block_blobs_trackers_pool_stats_total").get(List.of(type, subType)))
.isEqualTo(count);
}

Expand Down
6 changes: 3 additions & 3 deletions gradle/versions.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -137,14 +137,14 @@ dependencyManagement {

dependency 'org.xerial.snappy:snappy-java:1.1.10.5'

dependency 'io.prometheus:simpleclient:0.9.0'
dependency 'io.prometheus:simpleclient:0.16.0'

dependencySet(group: 'org.hyperledger.besu.internal', version: '24.1.2') {
dependencySet(group: 'org.hyperledger.besu.internal', version: '24.5.2') {
entry('metrics-core')
entry('core')
entry('config')
}
dependencySet(group: 'org.hyperledger.besu', version: '24.1.2') {
dependencySet(group: 'org.hyperledger.besu', version: '24.5.2') {
entry('besu-datatypes')
entry('evm')
entry('plugin-api')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,14 @@ class DirectEventDeliverer<T> extends EventDeliverer<T> {
consumedEventCounter =
metricsSystem.createLabelledCounter(
EVENTBUS,
"event_consumed_count",
"event_consumed_count_total",
"Total number of events consumed",
"channel",
"subscriber");
failedEventCounter =
metricsSystem.createLabelledCounter(
EVENTBUS,
"event_failed_count",
"event_failed_count_total",
"Number of events which failed to be processed",
"channel",
"subscriber");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ protected EventDeliverer(final MetricsSystem metricsSystem) {
publishedEventCounter =
metricsSystem.createLabelledCounter(
TekuMetricCategory.EVENTBUS,
"event_published_count",
"event_published_count_total",
"Total number of events published",
"channel");
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ void shouldCountWithDefault() {
metricsSystem
.streamObservations(CATEGORY)
.filter(ob -> ob.getMetricName().equals(COUNTER_NAME))
.filter(ob -> !ob.getLabels().contains("created"))
.collect(Collectors.toMap(Observation::getLabels, Observation::getValue));

assertThat(values)
Expand Down Expand Up @@ -117,6 +118,7 @@ void shouldNotCountNonMatching() {
metricsSystem
.streamObservations(CATEGORY)
.filter(ob -> ob.getMetricName().equals(COUNTER_NAME))
.filter(ob -> !ob.getLabels().contains("created"))
.collect(Collectors.toMap(Observation::getLabels, Observation::getValue));

assertThat(values).containsOnly(entry(List.of("label1Val1", "label2UnknownVal", "[0,∞)"), 1d));
Expand All @@ -141,6 +143,7 @@ void shouldInitCounters() {
metricsSystem
.streamObservations(CATEGORY)
.filter(ob -> ob.getMetricName().equals(COUNTER_NAME))
.filter(ob -> !ob.getLabels().contains("created"))
.collect(Collectors.toMap(Observation::getLabels, Observation::getValue));

assertThat(values).containsOnly(entry(List.of("a", "b", "[0,∞)"), 0d));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ private PeerChainValidator(
final LabelledMetric<Counter> validationCounter =
metricsSystem.createLabelledCounter(
TekuMetricCategory.NETWORK,
"peer_chain_validation_attempts",
"peer_chain_validation_attempts_total",
"Number of peers chain verification has been performed on",
"status");
validationStartedCounter = validationCounter.labels("started");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ public ConnectionManager(
final LabelledMetric<Counter> connectionAttemptCounter =
metricsSystem.createLabelledCounter(
TekuMetricCategory.NETWORK,
"peer_connection_attempt_count",
"peer_connection_attempt_count_total",
"Total number of outbound connection attempts made",
"status");
attemptedConnectionCounter = connectionAttemptCounter.labels("attempted");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ public BeaconChainMetrics(
final LabelledMetric<Counter> versionCounter =
metricsSystem.createLabelledCounter(
TekuMetricCategory.BEACON,
VersionProvider.CLIENT_IDENTITY + "_version",
VersionProvider.CLIENT_IDENTITY + "_version_total",
"Teku version in use",
"version");
versionCounter.labels(version).inc();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,18 +49,18 @@ public V4FinalizedStateTreeStorageLogic(
this.branchNodeStoredCounter =
metricsSystem.createLabelledCounter(
TekuMetricCategory.STORAGE_FINALIZED_DB,
"state_branch_nodes",
"state_branch_nodes_total",
"Number of finalized state tree branch nodes stored vs skipped",
"type");
this.leafNodeStoredCounter =
metricsSystem.createCounter(
TekuMetricCategory.STORAGE_FINALIZED_DB,
"state_leaf_nodes",
"state_leaf_nodes_total",
"Number of finalized state tree leaf nodes stored");
statesStoredCounter =
metricsSystem.createCounter(
TekuMetricCategory.STORAGE_FINALIZED_DB,
"states_stored",
"states_stored_total",
"Number of finalized states stored");
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ static void validateMetrics(
final long failCount,
final long timeoutCount) {
final StubCounter labelledCounter =
metricsSystem.getCounter(TekuMetricCategory.VALIDATOR, "external_signer_requests");
metricsSystem.getCounter(TekuMetricCategory.VALIDATOR, "external_signer_requests_total");
assertThat(labelledCounter.getValue("success")).isEqualTo(successCount);
assertThat(labelledCounter.getValue("failed")).isEqualTo(failCount);
assertThat(labelledCounter.getValue("timeout")).isEqualTo(timeoutCount);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ private PendingDuties(
dutiesPerformedCounter =
metricsSystem.createLabelledCounter(
TekuMetricCategory.VALIDATOR,
"duties_performed",
"duties_performed_total",
"Count of the failed duties, by duty type",
"type",
"result");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ public ExternalSigner(
final LabelledMetric<Counter> labelledCounter =
metricsSystem.createLabelledCounter(
TekuMetricCategory.VALIDATOR,
"external_signer_requests",
"external_signer_requests_total",
"Completed external signer counts",
"result");
successCounter = labelledCounter.labels("success");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ void shouldReportTotalNumberOfSuccessesAndFailuresToMetrics() {

private void validateMetrics(final String duty, final long successCount, final long failCount) {
final StubCounter labelledCounter =
metricsSystem.getCounter(TekuMetricCategory.VALIDATOR, "duties_performed");
metricsSystem.getCounter(TekuMetricCategory.VALIDATOR, "duties_performed_total");
assertThat(labelledCounter.getValue(duty, "success")).isEqualTo(successCount);
assertThat(labelledCounter.getValue(duty, "failed")).isEqualTo(failCount);
}
Expand Down

0 comments on commit 79b277f

Please sign in to comment.