Merge pull request #574 from lidofinance/feat/oracle-v5

[EPIC] Oracle V5 (Pectra Duck)

.github/workflows/mainnet_fork_tests.yml

@@ -0,0 +1,56 @@
+name: Mainnet Fork Tests
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - edited
+      - closed
+    branches:
+      - main
+      - develop
+    paths:
+      - "src/**"
+
+permissions:
+  contents: read
+  security-events: write
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"
+
+      - name: Setup poetry
+        run: >
+          curl -sSL https://install.python-poetry.org | python - &&
+          echo "$POETRY_HOME/bin" >> "$GITHUB_PATH"
+        env:
+          POETRY_HOME: "/opt/poetry"
+          POETRY_VERSION: 1.3.2
+
+      - name: Install Python dependencies
+        run: |
+          poetry install --no-interaction --with=dev
+
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+
+      - name: Mainnet Fork Tests
+        run: poetry run pytest -m 'fork' -n auto tests
+        env:
+          EXECUTION_CLIENT_URI: ${{ secrets.EXECUTION_CLIENT_URI }}
+          CONSENSUS_CLIENT_URI: ${{ secrets.CONSENSUS_CLIENT_URI }}
+          KEYS_API_URI: ${{ secrets.KEYS_API_URI }}
+          LIDO_LOCATOR_ADDRESS: "0xC1d0b3DE6792Bf6b4b37EccdcC24e45978Cfd2Eb"
+          CSM_MODULE_ADDRESS: "0xdA7dE2ECdDfccC6c3AF10108Db212ACBBf9EA83F"
+

README.md

@@ -58,6 +58,7 @@ Only Oracle:
 - Memory - 8 GB
 
 Oracle + KAPI:
+
 - vCPU - 4
 - Memory - 16 GB
 
@@ -79,13 +80,13 @@ Oracle needs two weeks of archived data.
 
 Also, to calculate some metrics for bunker mode Oracle needs [archive](https://ethereum.org/en/developers/docs/nodes-and-clients/#archive-node) consensus node.
 
-| Client                                            | Tested | Notes                                                                                                                                           |
-|---------------------------------------------------|:------:|-------------------------------------------------------------------------------------------------------------------------------------------------|
-| [Lighthouse](https://lighthouse.sigmaprime.io/)   |   🟢   | Use `--reconstruct-historic-states` param                                                                                                       |
-| [Lodestar](https://lodestar.chainsafe.io)         |   🔴   | Not tested yet                                                                                                                                  |
-| [Nimbus](https://nimbus.team)                     |   🔴   | Not tested yet                                                                                                                                  |
-| [Prysm](https://github.com/prysmaticlabs/prysm)   |   🟢   | Use <br> `--grpc-max-msg-size=104857600` <br> `--enable-historical-state-representation=true` <br> `--slots-per-archive-point=1024` <br> params |
-| [Teku](https://docs.teku.consensys.net)           |   🟢   | Use <br> `--data-storage-mode=archive` <br>`--data-storage-archive-frequency=1024`<br> `--reconstruct-historic-states=true`<br> params          |
+| Client                                          | Tested | Notes                                                                                                                                           |
+|-------------------------------------------------|:------:|-------------------------------------------------------------------------------------------------------------------------------------------------|
+| [Lighthouse](https://lighthouse.sigmaprime.io/) |   🟢   | Use `--reconstruct-historic-states` param                                                                                                       |
+| [Lodestar](https://lodestar.chainsafe.io)       |   🔴   | Not tested yet                                                                                                                                  |
+| [Nimbus](https://nimbus.team)                   |   🔴   | Not tested yet                                                                                                                                  |
+| [Prysm](https://github.com/prysmaticlabs/prysm) |   🟢   | Use <br> `--grpc-max-msg-size=104857600` <br> `--enable-historical-state-representation=true` <br> `--slots-per-archive-point=1024` <br> params |
+| [Teku](https://docs.teku.consensys.net)         |   🟢   | Use <br> `--data-storage-mode=archive` <br>`--data-storage-archive-frequency=1024`<br> `--reconstruct-historic-states=true`<br> params          |
 
 ### Keys API Service
 
@@ -281,9 +282,12 @@ Special metrics for ejector oracle:
 
 Special metrics for CSM oracle:
 
-| Metric name                       | Description                                 | Labels |
-|-----------------------------------|---------------------------------------------|--------|
-| TBD                               | TBD                                         |        |
+| Metric name                     | Description                            | Labels |
+|---------------------------------|----------------------------------------|--------|
+| csm_current_frame_range_l_epoch | Left epoch of the current frame range  |        |
+| csm_current_frame_range_r_epoch | Right epoch of the current frame range |        |
+| csm_unprocessed_epochs_count    | Unprocessed epochs count               |        |
+| csm_min_unprocessed_epoch       | Minimum unprocessed epoch              |        |
 
 # Development
 

pyproject.toml

@@ -57,6 +57,7 @@ markers = [
     "integration: tests with using providers",
     "possible_integration: tests with using providers, but can be run using mocks",
     "e2e: complex tests with using providers and real Ethereum network",
+    "fork: tests with using forked Ethereum network",
 ]
 addopts = "-s -vv --pdbcls pudb.debugger:Debugger"
 

src/constants.py

@@ -1,31 +1,46 @@
+from src.types import Gwei, SlotNumber
+
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#misc
-FAR_FUTURE_EPOCH = 2 ** 64 - 1
+GENESIS_SLOT = SlotNumber(0)
+FAR_FUTURE_EPOCH = 2**64 - 1
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#time-parameters-1
 MIN_VALIDATOR_WITHDRAWABILITY_DELAY = 2**8
-SHARD_COMMITTEE_PERIOD = 256
+SHARD_COMMITTEE_PERIOD = 2**8
 MAX_SEED_LOOKAHEAD = 4
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#state-list-lengths
 EPOCHS_PER_SLASHINGS_VECTOR = 2**13
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#rewards-and-penalties
 PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX = 3
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#gwei-values
-EFFECTIVE_BALANCE_INCREMENT = 2 ** 0 * 10 ** 9
-MAX_EFFECTIVE_BALANCE = 32 * 10 ** 9
+EFFECTIVE_BALANCE_INCREMENT = Gwei(2**0 * 10**9)
+MAX_EFFECTIVE_BALANCE = Gwei(32 * 10**9)
+# https://github.com/ethereum/consensus-specs/blob/dev/specs/electra/beacon-chain.md#gwei-values
+MAX_EFFECTIVE_BALANCE_ELECTRA = Gwei(2**11 * 10**9)
+MIN_ACTIVATION_BALANCE = Gwei(2**5 * 10**9)
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/capella/beacon-chain.md#execution
-MAX_WITHDRAWALS_PER_PAYLOAD = 2 ** 4
+MAX_WITHDRAWALS_PER_PAYLOAD = 2**4
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#withdrawal-prefixes
 ETH1_ADDRESS_WITHDRAWAL_PREFIX = '0x01'
+# https://github.com/ethereum/consensus-specs/blob/dev/specs/electra/beacon-chain.md#withdrawal-prefixes
+COMPOUNDING_WITHDRAWAL_PREFIX = '0x02'
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#validator-cycle
-MIN_PER_EPOCH_CHURN_LIMIT = 2 ** 2
-CHURN_LIMIT_QUOTIENT = 2 ** 16
+MIN_PER_EPOCH_CHURN_LIMIT = 2**2
+CHURN_LIMIT_QUOTIENT = 2**16
+# https://github.com/ethereum/consensus-specs/blob/dev/specs/electra/beacon-chain.md#validator-cycle
+MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA = Gwei(2**7 * 10**9)
+MAX_PER_EPOCH_ACTIVATION_EXIT_CHURN_LIMIT = Gwei(2**8 * 10**9)
 # https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#time-parameters
-SLOTS_PER_HISTORICAL_ROOT = 8192
+SLOTS_PER_HISTORICAL_ROOT = 2**13 # 8192
 
-# Local constants
-GWEI_TO_WEI = 10 ** 9
-SHARE_RATE_PRECISION_E27 = 10 ** 27
+# https://github.com/ethereum/consensus-specs/blob/dev/specs/electra/beacon-chain.md#withdrawals-processing
+MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP = 2**3
+
+# Lido contracts constants
+LIDO_DEPOSIT_AMOUNT = MIN_ACTIVATION_BALANCE
+SHARE_RATE_PRECISION_E27 = 10**27
 TOTAL_BASIS_POINTS = 10000
 
+# Local constants
+GWEI_TO_WEI = 10**9
 MAX_BLOCK_GAS_LIMIT = 30_000_000
-
-UINT64_MAX = 2 ** 64 - 1
+UINT64_MAX = 2**64 - 1

src/main.py

@@ -24,14 +24,13 @@
     KeysAPIClientModule,
     LidoValidatorsProvider,
     FallbackProviderModule,
-    LazyCSM
+    LazyCSM,
 )
 from src.web3py.middleware import metrics_collector
 from src.web3py.types import Web3
 
 from src.web3py.contract_tweak import tweak_w3_contracts
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -42,22 +41,10 @@ def main(module_name: OracleModule):
         'variables': {
             **build_info,
             'module': module_name,
-            'ACCOUNT': variables.ACCOUNT.address if variables.ACCOUNT else 'Dry',
-            'LIDO_LOCATOR_ADDRESS': variables.LIDO_LOCATOR_ADDRESS,
-            'CSM_MODULE_ADDRESS': variables.CSM_MODULE_ADDRESS,
-            'FINALIZATION_BATCH_MAX_REQUEST_COUNT': variables.FINALIZATION_BATCH_MAX_REQUEST_COUNT,
-            'EL_REQUESTS_BATCH_SIZE': variables.EL_REQUESTS_BATCH_SIZE,
-            'MAX_CYCLE_LIFETIME_IN_SECONDS': variables.MAX_CYCLE_LIFETIME_IN_SECONDS,
+            **variables.PUBLIC_ENV_VARS,
         },
     })
-    ENV_VARIABLES_INFO.info({
-        "ACCOUNT": str(variables.ACCOUNT.address) if variables.ACCOUNT else 'Dry',
-        "LIDO_LOCATOR_ADDRESS": str(variables.LIDO_LOCATOR_ADDRESS),
-        "CSM_MODULE_ADDRESS": str(variables.CSM_MODULE_ADDRESS),
-        "FINALIZATION_BATCH_MAX_REQUEST_COUNT": str(variables.FINALIZATION_BATCH_MAX_REQUEST_COUNT),
-        "EL_REQUESTS_BATCH_SIZE": str(variables.EL_REQUESTS_BATCH_SIZE),
-        "MAX_CYCLE_LIFETIME_IN_SECONDS": str(variables.MAX_CYCLE_LIFETIME_IN_SECONDS),
-    })
+    ENV_VARIABLES_INFO.info(variables.PUBLIC_ENV_VARS)
     BUILD_INFO.info(build_info)
 
     logger.info({'msg': f'Start healthcheck server for Docker container on port {variables.HEALTHCHECK_SERVER_PORT}'})
@@ -171,18 +158,18 @@ def ipfs_providers() -> Iterator[IPFSProvider]:
 
 if __name__ == '__main__':
     module_name_arg = sys.argv[-1]
-    if module_name_arg not in iter(OracleModule):
+    if module_name_arg not in OracleModule:
         msg = f'Last arg should be one of {[str(item) for item in OracleModule]}, received {module_name_arg}.'
         logger.error({'msg': msg})
         raise ValueError(msg)
 
     module = OracleModule(module_name_arg)
-    if module == OracleModule.CHECK:
+    if module is OracleModule.CHECK:
         errors = variables.check_uri_required_variables()
         variables.raise_from_errors(errors)
 
         sys.exit(check())
 
-    errors = variables.check_all_required_variables()
+    errors = variables.check_all_required_variables(module)
     variables.raise_from_errors(errors)
     main(module)