chore(ci): clean up env some more & downsize compose

Author: armyhaylenko

.env.example

@@ -1,21 +1,36 @@
 RPC_HOST='https://mainnet-aura.metaplex.com/{personal_rpc_key}'
 
+#Configurable app parts that could be enabled or disabled. (values in the default positions)
+RUN_API=true
+RUN_BACKFILLER=true
+RUN_BUBBLEGUM_BACKFILLER=true
+RUN_GAPFILLER=false
+SHOULD_REINGEST=false
+RUN_PROFILING=false
+RESTORE_ROCKS_DB=false
+ENABLE_ROCKS_MIGRATION=true
+CHECK_PROOFS=false
+SKIP_CHECK_TREE_GAPS=false
+
 #Postgres
 PG_DATABASE_URL='postgres://solana:solana@localhost:5432/aura_db'
+POSTGRE_DB_PATH="/postgre/db/path"
 
 #Redis
 REDIS_CONNECTION_CONFIG='{"redis_connection_str":"redis://127.0.0.1:6379/0"}'
 
 #RocksDB
 ROCKS_DB_PATH="/usr/src/rocksdb-data"
-ROCKS_DB_SECONDARY_PATH="path/to/rocks/secondary/db"
+ROCKS_DB_SECONDARY_PATH="/path/to/rocks/secondary/db"
 # path to the slots data, required for the backfiller to work
 ROCKS_SLOTS_DB_PATH=/path/to/slots-data
 ROCKS_SECONDARY_SLOTS_DB_PATH=/path/to/secondary/ingester-slots
-ROCKS_ARCHIVES_DIR="path/to/rocks/backup/archives"
-ROCKS_BACKUP_ARCHIVES_DIR="path/to/rocks/backup/archives"
+ROCKS_ARCHIVES_DIR="/path/to/rocks/backup/archives"
+ROCKS_BACKUP_ARCHIVES_DIR="/path/to/rocks/backup/archives"
 ROCKS_BACKUP_URL="127.0.0.1:3051/snapshot"
 ROCKS_MIGRATION_STORAGE_PATH=/path/to/migration_storage
+ROCKS_BACKUP_DIR="/path/to/rocks/backup/"
+ROCKS_DUMP_PATH="/path/to/dump"
 
 #Backfiller
 BACKFILLER_SOURCE_MODE=bigtable
@@ -33,14 +48,10 @@ API_SERVER_PORT=8990
 
 # GRPC Server port
 PEER_GRPC_PORT=9099
+GAPFILLER_PEER_ADDR="0.0.0.0"
 
-# Docker
-# Required by Postgre container
-POSTGRE_DB_PATH="postgre/db/path"
-ROCKS_BACKUP_DIR="path/to/rocks/backup/"
-FILE_STORAGE_PATH="path/to/file/storage"
+FILE_STORAGE_PATH="/path/to/file/storage"
 FILE_STORAGE_PATH_CONTAINER="/usr/src/app/file_storage"
-ROCKS_DUMP_PATH="/path/to/dump"
 
 #Profiling (optional)
 PROFILING_FILE_PATH_CONTAINER="/usr/src/profiling"
@@ -60,70 +71,3 @@ INTEGRITY_VERIFICATION_TEST_FILE_PATH="./test_keys/test_keys.txt"
 INTEGRITY_VERIFICATION_TEST_FILE_PATH_CONTAINER="/test_keys/test_keys.txt"
 INTEGRITY_VERIFICATION_SLOTS_COLLECT_PATH="./slots_collect"
 INTEGRITY_VERIFICATION_SLOTS_COLLECT_PATH_CONTAINER="/slots_collect"
-
-#Configurable app parts that cold be enabled or disabled. (values in the default positions)
-RUN_API=true
-RUN_BACKFILLER=true
-RUN_BUBBLEGUM_BACKFILLER=true
-RUN_GAPFILLER=false
-SHOULD_REINGEST=false
-RUN_PROFILING=false
-RESTORE_ROCKS_DB=false
-ENABLE_ROCKS_MIGRATION=true
-CHECK_PROOFS=false
-SKIP_CHECK_TREE_GAPS=false
-
-#Changes (todo Remove after review/migration)
-
-# API_RPC_HOST  INGESTER_RPC_HOST -> RPC_HOST
-#INGESTER_ROCKS_DB_PATH -> ROCKS_DB_PATH
-#INGESTER_ROCKS_DB_PATH_CONTAINER -> ROCKS_DB_PATH
-#INGESTER_SYNCHRONIZER_DUMP_PATH -> ROCKS_DUMP_PATH
-#API_ROCKS_DB_PATH_CONTAINER -> ROCKS_DB_PATH
-#INGESTER_FILE_STORAGE_PATH -> FILE_STORAGE_PATH
-#INGESTER_FILE_STORAGE_PATH_CONTAINER -> FILE_STORAGE_PATH_CONTAINER
-# INGESTER_PROFILING_FILE_PATH -> PROFILING_FILE_PATH
-# INGESTER_PROFILING_FILE_PATH_CONTAINER -> PROFILING_FILE_PATH_CONTAINER
-#  INGESTER_MIGRATION_STORAGE_PATH -> ROCKS_MIGRATION_STORAGE_PATH
-#
-# INGESTER_ROCKS_BACKUP_ARCHIVES_DIR -> ROCKS_BACKUP_ARCHIVES_DIR
-# INGESTER_ROCKS_BACKUP_DIR ->  ROCKS_BACKUP_DIR
-#
-#SYNCHRONIZER_DUMP_PATH -> ROCKS_DUMP_PATH
-#INGESTER_SLOTS_DB_PATH -> ROCKS_SLOTS_DB_PATH
-#INGESTER_SECONDARY_SLOTS_DB_PATH -> ROCKS_SECONDARY_SLOTS_DB_PATH
-
-#API_DATABASE_CONFIG ->  PG_MAX_DB_CONNECTIONS  and  PG_DATABASE_URL
-
-#SYNCHRONIZER_DUMP_PATH -> ROCKS_DUMP_PATH
-#SYNCHRONIZER_ROCKS_DB_SECONDARY_PATH_CONTAINER -> ROCKS_DB_SECONDARY_PATH
-#SYNCHRONIZER_DUMP_SYNCHRONIZER_BATCH_SIZE -> DUMP_SYNCHRONIZER_BATCH_SIZE
-#SYNCHRONIZER_DUMP_SYNC_THRESHOLD -> DUMP_SYNC_THRESHOLD
-
-#API_ARCHIVES_DIR -> rocks_archives_dir
-#API_ROCKS_DB_PATH_CONTAINER -> ROCKS_DB_PATH
-#API_ROCKS_DB_SECONDARY_PATH_CONTAINER -> ROCKS_DB_SECONDARY_PATH
-#API_FILE_STORAGE_PATH_CONTAINER -> FILE_STORAGE_PATH_CONTAINER
-#API_JSON_MIDDLEWARE_CONFIG -> JSON_MIDDLEWARE_CONFIG
-#API_CONSISTENCE_SYNCHRONIZATION_API_THRESHOLD - > CONSISTENCE_SYNCHRONIZATION_API_THRESHOLD
-#API_CONSISTENCE_BACKFILLING_SLOTS_THRESHOLD -> CONSISTENCE_BACKFILLING_SLOTS_THRESHOLD
-#
-#API_SKIP_CHECK_TREE_GAPS -> SKIP_CHECK_TREE_GAPS
-# INGESTER_REDIS_MESSENGER_CONFIG -> REDIS_CONNECTION_CONFIG and note the differnce (less complex type)
-# INGESTER_BACKFILLER_SOURCE_MODE -> BACKFILLER_SOURCE_MODE
-# INGESTER_BIG_TABLE_CONFIG -> BIG_TABLE_CONFIG
-# both API_PEER_GRPC_PORT and INGESTER_PEER_GRPC_PORT were replaced with a single PEER_GRPC_PORT - only one will actually work
-# API_PEER_GRPC_MAX_GAP_SLOTS and INGESTER_PEER_GRPC_MAX_GAP_SLOTS -> PEER_GRPC_MAX_GAP_SLOTS (optional)
-# INGESTER_ROCKS_SYNC_INTERVAL_SECONDS and API_ROCKS_SYNC_INTERVAL_SECONDS -> ROCKS_SYNC_INTERVAL_SECONDS (optional)
-# INGESTER_GAPFILLER_PEER_ADDR -> GAPFILLER_PEER_ADDR (optional, only if run_gapfiller is set)
-# INGESTER_ACCOUNTS_BUFFER_SIZE -> ACCOUNT_PROCESSOR_BUFFER_SIZE (has default value, may be skipped)
-# INGESTER_ACCOUNTS_PARSING_WORKERS -> REDIS_ACCOUNTS_PARSING_WORKERS
-# INGESTER_ROCKS_FLUSH_BEFORE_BACKUP -> ROCKS_FLUSH_BEFORE_BACKUP
-# INGESTER_ROCKS_INTERVAL_IN_SECONDS -> ROCKS_INTERVAL_IN_SECONDS
-# INGESTER_TRANSACTIONS_PARSING_WORKERS -> REDIS_TRANSACTIONS_PARSING_WORKERS
-# INGESTER_SHOULD_REINGEST -> SHOULD_REINGEST
-# INGESTER_RUN_SEQUENCE_CONSISTENT_CHECKER -> RUN_SEQUENCE_CONSISTENT_CHECKER
-# INGESTER_RUN_PROFILING -> RUN_PROFILING
-# INGESTER_RUN_BUBBLEGUM_BACKFILLER -> RUN_BUBBLEGUM_BACKFILLER
-# Removed:
-# INGESTER_BACKFILL_RPC_ADDRESS, INGESTER_BACKFILLER_MODE, INGESTER_MESSAGE_SOURCE (redis is used as an only option as of now), INGESTER_DISABLE_SYNCHRONIZER (synchronizer is no longer part of the ingester), INGESTER_CHUNK_SIZE, INGESTER_PERMITTED_TASKS, INGESTER_TCP_CONFIG, INGESTER_WORKERS_COUNT, INGESTER_WAIT_PERIOD_SEC, INGESTER_SNAPSHOT_PARSING_WORKERS, INGESTER_SNAPSHOT_PARSING_BATCH_SIZE, INGESTER_SLOT_UNTIL, INGESTER_SLOT_START_FROM, INGESTER_RUN_FORK_CLEANER, INGESTER_RUN_DUMP_SYNCHRONIZE_ON_START

docker-compose.yaml

@@ -1,9 +1,7 @@
-version: "3.9"
 services:
   ingester:
     container_name: ingester
     restart: always
-    entrypoint: sh -c "./ingester --rocks-slots-db-path $$ROCKS_SLOTS_DB_PATH --rocks-secondary-slots-db-path $$ROCKS_SECONDARY_SLOTS_DB_PATH"
     env_file:
       - .env
     network_mode: host
@@ -12,20 +10,23 @@ services:
       - ${ROCKS_DUMP_PATH:-/tmp/rocks_dump}:${ROCKS_DUMP_PATH:-/tmp/rocks_dump}:rw
       - ${ROCKS_BACKUP_DIR}:${ROCKS_BACKUP_DIR}:rw
       - ${ROCKS_BACKUP_ARCHIVES_DIR}:${ROCKS_BACKUP_ARCHIVES_DIR}:rw
-      - ${PROFILING_FILE_PATH}:${PROFILING_FILE_PATH_CONTAINER}:rw
       - ${FILE_STORAGE_PATH}:${FILE_STORAGE_PATH_CONTAINER}:rw
       - ${ROCKS_MIGRATION_STORAGE_PATH}:${ROCKS_MIGRATION_STORAGE_PATH}:rw
       - ${ROCKS_SLOTS_DB_PATH}:${ROCKS_SLOTS_DB_PATH}:ro
       - ${ROCKS_SECONDARY_SLOTS_DB_PATH}:${ROCKS_SECONDARY_SLOTS_DB_PATH}:rw
       - ./creds.json:/usr/src/app/creds.json
       - ./migrations:/usr/src/app/migrations
       - ./arweave_wallet.json:/usr/src/app/arweave_wallet.json
-      - ./heaps:/usr/src/app/heaps:rw
+      # Only exists for profiling - uncomment if needed
+      # - ./heaps:/usr/src/app/heaps:rw
+      # - ${PROFILING_FILE_PATH}:${PROFILING_FILE_PATH_CONTAINER}:rw
     depends_on:
       - db
     build:
       context: .
       dockerfile: docker/app.Dockerfile
+      # Replace for profiling:
+      # dockerfile: docker/profiling/app.Dockerfile
       args:
         BINARY: ingester
     stop_grace_period: 10m
@@ -44,7 +45,8 @@ services:
       - ${ROCKS_DB_SECONDARY_PATH}/api:${ROCKS_DB_SECONDARY_PATH}:rw
       - ${ROCKS_ARCHIVES_DIR}:${ROCKS_ARCHIVES_DIR}:ro
       - ${FILE_STORAGE_PATH}:${FILE_STORAGE_PATH_CONTAINER}:rw
-      - ./heaps:/usr/src/app/heaps:rw
+      # Only exists for profiling - uncomment if needed
+      # - ./heaps:/usr/src/app/heaps:rw
     depends_on:
       - db
     build:
@@ -67,8 +69,9 @@ services:
       - ${ROCKS_DB_PATH}:${ROCKS_DB_PATH}:rw
       - ${ROCKS_DUMP_PATH:-/tmp/rocks_dump}:${ROCKS_DUMP_PATH:-/tmp/rocks_dump}:rw
       - ${ROCKS_DB_SECONDARY_PATH}/synchronizer:${ROCKS_DB_SECONDARY_PATH}:rw
-      - ./heaps:/usr/src/app/heaps:rw
       - ./migrations:/usr/src/app/migrations
+      # Only exists for profiling - uncomment if needed
+      # - ./heaps:/usr/src/app/heaps:rw
     depends_on:
       - db
     build:
@@ -127,16 +130,10 @@ services:
   slot-persister:
     container_name: slot-persister
     restart: always
-    entrypoint: >
-      sh -c "if [ -f /usr/src/app/creds.json ]; then
-        export BIG_TABLE_CREDENTIALS=/usr/src/app/creds.json;
-      fi;
-      exec ./slot_persister"
     env_file:
       - .env
     environment:
       SLOTS_DB_PRIMARY_PATH: ${ROCKS_SLOTS_DB_PATH}
-      # BIG_TABLE_CREDENTIALS: /usr/src/app/creds.json # refactored this to account for the file doesn't exist case
       METRICS_PORT: 9090
     network_mode: host
     volumes:
@@ -164,7 +161,7 @@ services:
       POSTGRES_PASSWORD: solana # The PostgreSQL password (useful to connect to the database)
       POSTGRES_DB: solana
     volumes:
-      - ${POSTGRE_DB_PATH:-./db-data}:/var/lib/postgresql/data/:rw
+      - ${POSTGRE_DB_PATH}:/var/lib/postgresql/data/:rw
       - ${ROCKS_DUMP_PATH:-/tmp/rocks_dump}:${ROCKS_DUMP_PATH:-/tmp/rocks_dump}:ro
     logging:
       options:

nft_ingester/src/bin/slot_persister/main.rs

@@ -6,7 +6,11 @@ use entities::models::RawBlock;
 use futures::future::join_all;
 use interface::{signature_persistence::BlockProducer, slot_getter::FinalizedSlotGetter};
 use metrics_utils::{utils::start_metrics, MetricState, MetricsTrait};
-use nft_ingester::{backfiller::BackfillSource, inmemory_slots_dumper::InMemorySlotsDumper};
+use nft_ingester::{
+    backfiller::BackfillSource,
+    config::{parse_json_to_big_table_config, BigTableConfig},
+    inmemory_slots_dumper::InMemorySlotsDumper,
+};
 use rocks_db::{column::TypedColumn, SlotStorage};
 use tokio::sync::{broadcast, Semaphore};
 use tokio_retry::{strategy::ExponentialBackoff, RetryIf};
@@ -42,13 +46,9 @@ struct Args {
     #[arg(short, long)]
     start_slot: Option<u64>,
 
-    /// Big table credentials file path
-    #[arg(short, long, env = "BIG_TABLE_CREDENTIALS")]
-    big_table_credentials: Option<String>,
-
-    /// Optional big table timeout (default: 1000)
-    #[arg(short = 'B', long, default_value_t = 1000)]
-    big_table_timeout: u32,
+    /// Big table config (best passed from env)
+    #[arg(short, long, env, value_parser = parse_json_to_big_table_config)]
+    big_table_config: Option<BigTableConfig>,
 
     /// Metrics port
     /// Default: 9090
@@ -187,11 +187,14 @@ pub async fn main() -> Result<(), Box<dyn std::error::Error>> {
     let rpc_client = Arc::new(BackfillRPC::connect(args.rpc_host.clone()));
 
     let backfill_source = {
-        if let Some(ref bg_creds) = args.big_table_credentials {
+        if let Some(ref big_table_config) = args.big_table_config {
             Arc::new(BackfillSource::Bigtable(Arc::new(
-                BigTableClient::connect_new_with(bg_creds.clone(), args.big_table_timeout)
-                    .await
-                    .expect("expected to connect to big table"),
+                BigTableClient::connect_new_with(
+                    big_table_config.get_big_table_creds_key().expect("get big table greds"),
+                    big_table_config.get_big_table_timeout_key().expect("get big table timeout"),
+                )
+                .await
+                .expect("expected to connect to big table"),
             )))
         } else {
             Arc::new(BackfillSource::Rpc(rpc_client.clone()))

nft_ingester/src/config.rs

@@ -533,7 +533,7 @@ fn parse_json_to_json_middleware_config(s: &str) -> Result<JsonMiddlewareConfig,
     parse_json(s)
 }
 
-fn parse_json_to_big_table_config(s: &str) -> Result<BigTableConfig, String> {
+pub fn parse_json_to_big_table_config(s: &str) -> Result<BigTableConfig, String> {
     parse_json(s)
 }