fix(node): complete state-sync in init container

[andy108369]

fixes #229

Test results

• init pod state-synced up until the halt height as expected:

# kubectl -n akash-services logs akash-node-1-0 --tail=10000 -f -c init |grep -iv p2p
+ apt update

WARNING: apt does not have a stable CLI interface. Use with caution in scripts.

Get:1 http://deb.debian.org/debian bullseye InRelease [116 kB]
Get:2 http://deb.debian.org/debian-security bullseye-security InRelease [48.4 kB]
Get:3 http://deb.debian.org/debian bullseye-updates InRelease [44.1 kB]
Get:4 http://deb.debian.org/debian bullseye/main amd64 Packages [8062 kB]
Get:5 http://deb.debian.org/debian-security bullseye-security/main amd64 Packages [256 kB]
Get:6 http://deb.debian.org/debian bullseye-updates/main amd64 Packages [17.4 kB]
Fetched 8544 kB in 1s (9094 kB/s)
Reading package lists...
Building dependency tree...
Reading state information...
13 packages can be upgraded. Run 'apt list --upgradable' to see them.
+ apt -y --no-install-recommends install ca-certificates curl jq
+ type curl
curl is /usr/bin/curl
+ type jq
jq is /usr/bin/jq
+ '[' '!' -d /root/.akash/data ']'
+ /bin/akash init --chain-id akashnet-2 mynode-1
{"app_message":{"agov":{"deposit_params":{"min_initial_deposit_rate":"0.400000000000000000"}},"astaking":{"params":{"min_commission_rate":"0.050000000000000000"}},"audit":{"attributes":[]},"auth":{"accounts":[],"params":{"max_memo_characters":"256","sig_verify_cost_ed25519":"590","sig_verify_cost_secp256k1":"1000","tx_sig_limit":"7","tx_size_cost_per_byte":"10"}},"authz":{"authorization":[]},"bank":{"balances":[],"denom_metadata":[],"params":{"default_send_enabled":true,"send_enabled":[]},"supply":[]},"capability":{"index":"1","owners":[]},"cert":{"certificates":[]},"crisis":{"constant_fee":{"amount":"1000","denom":"stake"}},"deployment":{"deployments":[],"params":{"min_deposits":[{"amount":"5000000","denom":"uakt"}]}},"distribution":{"delegator_starting_infos":[],"delegator_withdraw_infos":[],"fee_pool":{"community_pool":[]},"outstanding_rewards":[],"params":{"base_proposer_reward":"0.010000000000000000","bonus_proposer_reward":"0.040000000000000000","community_tax":"0.020000000000000000","withdraw_addr_enabled":true},"previous_proposer":"","validator_accumulated_commissions":[],"validator_current_rewards":[],"validator_historical_rewards":[],"validator_slash_events":[]},"escrow":{"accounts":[],"payments":[]},"evidence":{"evidence":[]},"feegrant":{"allowances":[]},"genutil":{"gen_txs":[]},"gov":{"deposit_params":{"max_deposit_period":"172800s","min_deposit":[{"amount":"10000000","denom":"stake"}]},"deposits":[],"proposals":[],"starting_proposal_id":"1","tally_params":{"quorum":"0.334000000000000000","threshold":"0.500000000000000000","veto_threshold":"0.334000000000000000"},"votes":[],"voting_params":{"voting_period":"172800s"}},"ibc":{"channel_genesis":{"ack_sequences":[],"acknowledgements":[],"channels":[],"commitments":[],"next_channel_sequence":"0","receipts":[],"recv_sequences":[],"send_sequences":[]},"client_genesis":{"clients":[],"clients_consensus":[],"clients_metadata":[],"create_localhost":false,"next_client_sequence":"0","params":{"allowed_clients":["06-solomachine","07-tendermint"]}},"connection_genesis":{"client_connection_paths":[],"connections":[],"next_connection_sequence":"0","params":{"max_expected_time_per_block":"30000000000"}}},"inflation":{"params":{"inflation_decay_factor":"2.000000000000000000","initial_inflation":"100.000000000000000000","variance":"0.050000000000000000"}},"market":{"bids":[],"leases":[],"orders":[],"params":{"bid_min_deposit":{"amount":"5000000","denom":"uakt"},"order_max_bids":20}},"mint":{"minter":{"annual_provisions":"0.000000000000000000","inflation":"0.130000000000000000"},"params":{"blocks_per_year":"6311520","goal_bonded":"0.670000000000000000","inflation_max":"0.200000000000000000","inflation_min":"0.070000000000000000","inflation_rate_change":"0.130000000000000000","mint_denom":"stake"}},"params":null,"provider":{"providers":[]},"slashing":{"missed_blocks":[],"params":{"downtime_jail_duration":"600s","min_signed_per_window":"0.500000000000000000","signed_blocks_window":"100","slash_fraction_double_sign":"0.050000000000000000","slash_fraction_downtime":"0.010000000000000000"},"signing_infos":[]},"staking":{"delegations":[],"exported":false,"last_total_power":"0","last_validator_powers":[],"params":{"bond_denom":"stake","historical_entries":10000,"max_entries":7,"max_validators":100,"unbonding_time":"1814400s"},"redelegations":[],"unbonding_delegations":[],"validators":[]},"take":{"params":{"default_take_rate":20,"denom_take_rates":[{"denom":"uakt","rate":2}]}},"transfer":{"denom_traces":[],"params":{"receive_enabled":true,"send_enabled":true},"port_id":"transfer"},"upgrade":{},"vesting":{}},"chain_id":"akashnet-2","gentxs_dir":"","moniker":"mynode-1","node_id":"7817aab1d4e58948d308cd0a3553e81535e9341e"}
+ cd /root/.akash/data
+ curl -s https://raw.githubusercontent.com/akash-network/net/main/mainnet/genesis.json
+ '[' true == true ']'
+ echo 'state-sync is enabled, figure the right trust height & derive its hash'
+ export SNAP_RPC1=https://akash-rpc.polkachu.com:443
+ SNAP_RPC1=https://akash-rpc.polkachu.com:443
+ export SNAP_RPC2=https://akash-rpc.polkachu.com:443
+ SNAP_RPC2=https://akash-rpc.polkachu.com:443
state-sync is enabled, figure the right trust height & derive its hash
++ curl -Ls https://akash-rpc.polkachu.com:443/block
++ jq -r .result.block.header.height
+ LATEST_HEIGHT=13279913
+ HEIGHT_OFFSET=2000
+ BLOCK_HEIGHT=13277913
++ curl -Ls 'https://akash-rpc.polkachu.com:443/block?height=13277913'
++ jq -r .result.block_id.hash
+ TRUST_HASH=76AB3EE2C70C50AF6A92B1ED47850BC826BB87A33A99FD534F84760993D86C16
+ echo 'TRUST HEIGHT: 13277913'
+ echo 'TRUST HASH: 76AB3EE2C70C50AF6A92B1ED47850BC826BB87A33A99FD534F84760993D86C16'
+ export AKASH_STATESYNC_TRUST_HEIGHT=13277913
+ AKASH_STATESYNC_TRUST_HEIGHT=13277913
+ export AKASH_STATESYNC_TRUST_HASH=76AB3EE2C70C50AF6A92B1ED47850BC826BB87A33A99FD534F84760993D86C16
+ AKASH_STATESYNC_TRUST_HASH=76AB3EE2C70C50AF6A92B1ED47850BC826BB87A33A99FD534F84760993D86C16
+ export AKASH_HALT_HEIGHT=13279913
TRUST HEIGHT: 13277913
TRUST HASH: 76AB3EE2C70C50AF6A92B1ED47850BC826BB87A33A99FD534F84760993D86C16
+ AKASH_HALT_HEIGHT=13279913
+ /bin/akash start
INF initializing upgrade `akash_v0.15.0_cosmos_v0.44.x`
INF initializing upgrade `v0.20.0`
INF initializing upgrade `v0.26.0`
INF initializing upgrade `v0.24.0`
INF starting node with ABCI Tendermint in-process
INF service start impl=multiAppConn module=proxy msg={}
INF service start connection=query impl=localClient module=abci-client msg={}
INF service start connection=snapshot impl=localClient module=abci-client msg={}
INF service start connection=mempool impl=localClient module=abci-client msg={}
INF service start connection=consensus impl=localClient module=abci-client msg={}
INF service start impl=EventBus module=events msg={}
INF service start impl=PubSub module=pubsub msg={}
INF service start impl=IndexerService module=txindex msg={}
INF This node is not a validator addr=F5839081C551B256BC1DB1015F4A684F62A31E47 module=consensus pubKey=FQqWwuwqe89YRn7K2yxCjFZfTh+Ze9putLn4LsiZjYM=
INF service start impl=Node msg={}
INF Starting pprof server laddr=0.0.0.0:6060
INF service start impl=Evidence module=evidence msg={}
INF service start impl=StateSync module=statesync msg={}
INF service start impl=PEX module=pex msg={}
INF serve module=rpc-server msg={}
INF service start impl=BlockchainReactor module=blockchain msg={}
INF service start impl=ConsensusReactor module=consensus msg={}
INF Reactor  module=consensus waitSync=true
INF Starting state sync module=statesync
INF Downloading trusted light block using options module=light
INF sync any module=statesync msg={}
INF Discovered new snapshot format=1 hash="\x1f��������\uf7c4\v��\x01Ց�_e���\x16�a\b�W\x19s" height=13278000 module=statesync
INF Discovered new snapshot format=1 hash="�;��\x012t��0����&��IEV���_�7�\b�\x04\x03�" height=13276000 module=statesync
INF Discovered new snapshot format=1 hash="�[\x17�\x1cO�\x0es�\x03@OL�HE]�\x01?�\aIi�G>�;-�" height=13279500 module=statesync
INF Discovered new snapshot format=1 hash="\x1f��������\uf7c4\v��\x01Ց�_e���\x16�a\b�W\x19s" height=13278000 module=statesync
INF Discovered new snapshot format=1 hash="�;��\x012t��0����&��IEV���_�7�\b�\x04\x03�" height=13276000 module=statesync
INF Discovered new snapshot format=1 hash="\x1f��������\uf7c4\v��\x01Ց�_e���\x16�a\b�W\x19s" height=13278000 module=statesync
INF Discovered new snapshot format=1 hash="�;��\x012t��0����&��IEV���_�7�\b�\x04\x03�" height=13276000 module=statesync
INF Discovered new snapshot format=1 hash="\x1f��������\uf7c4\v��\x01Ց�_e���\x16�a\b�W\x19s" height=13278000 module=statesync
INF Discovered new snapshot format=1 hash="�;��\x012t��0����&��IEV���_�7�\b�\x04\x03�" height=13276000 module=statesync
INF Discovered new snapshot format=1 hash="\x1f��������\uf7c4\v��\x01Ց�_e���\x16�a\b�W\x19s" height=13278000 module=statesync
INF Discovered new snapshot format=1 hash="�;��\x012t��0����&��IEV���_�7�\b�\x04\x03�" height=13276000 module=statesync
INF Discovered new snapshot format=1 hash="\x1f��������\uf7c4\v��\x01Ց�_e���\x16�a\b�W\x19s" height=13278000 module=statesync
INF Discovered new snapshot format=1 hash="�;��\x012t��0����&��IEV���_�7�\b�\x04\x03�" height=13276000 module=statesync
INF Discovered new snapshot format=1 hash="\x1f��������\uf7c4\v��\x01Ց�_e���\x16�a\b�W\x19s" height=13278000 module=statesync
INF Discovered new snapshot format=1 hash="�;��\x012t��0����&��IEV���_�7�\b�\x04\x03�" height=13276000 module=statesync
INF Discovered new snapshot format=1 hash="\x1f��������\uf7c4\v��\x01Ց�_e���\x16�a\b�W\x19s" height=13278000 module=statesync
INF Discovered new snapshot format=1 hash="�;��\x012t��0����&��IEV���_�7�\b�\x04\x03�" height=13276000 module=statesync
INF Discovered new snapshot format=1 hash="\x1f��������\uf7c4\v��\x01Ց�_e���\x16�a\b�W\x19s" height=13278000 module=statesync
INF Discovered new snapshot format=1 hash="�;��\x012t��0����&��IEV���_�7�\b�\x04\x03�" height=13276000 module=statesync
INF Discovered new snapshot format=1 hash="v���\a���v��2��MY(��'��\x11��<\x01�{��\a" height=13279000 module=statesync
INF VerifyHeader hash=611DBFB8B08CFADA3270A085B193E7B797BF2519635DA975698EEB0EE3ED3930 height=13279501 module=light
INF VerifyHeader hash=0FE45431283E0EE83BD315A37486D4A510120181054FF6759AD666365B311567 height=13279502 module=light
INF Offering snapshot to ABCI app format=1 hash="�[\x17�\x1cO�\x0es�\x03@OL�HE]�\x01?�\aIi�G>�;-�" height=13279500 module=statesync
INF Snapshot accepted, restoring format=1 hash="�[\x17�\x1cO�\x0es�\x03@OL�HE]�\x01?�\aIi�G>�;-�" height=13279500 module=statesync
INF Fetching snapshot chunk chunk=1 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=2 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=0 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=3 format=1 height=13279500 module=statesync total=33
INF VerifyHeader hash=F99997F9684F37C0116F9356B7DB60E720B4188C34D8917F3A32ABC103358AAA height=13279500 module=light
INF Header has already been verified hash=611DBFB8B08CFADA3270A085B193E7B797BF2519635DA975698EEB0EE3ED3930 height=13279501 module=light
INF Header has already been verified hash=0FE45431283E0EE83BD315A37486D4A510120181054FF6759AD666365B311567 height=13279502 module=light
INF Header has already been verified hash=611DBFB8B08CFADA3270A085B193E7B797BF2519635DA975698EEB0EE3ED3930 height=13279501 module=light
INF Header has already been verified hash=F99997F9684F37C0116F9356B7DB60E720B4188C34D8917F3A32ABC103358AAA height=13279500 module=light
INF Fetching snapshot chunk chunk=4 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=0 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=5 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=6 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=7 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=1 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=2 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=3 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=8 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=4 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=9 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=5 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=10 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=11 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=6 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=12 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=13 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=14 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=15 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=16 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=17 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=7 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=18 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=19 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=20 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=21 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=22 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=23 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=24 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=25 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=26 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=27 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=28 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=29 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=8 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=30 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=31 format=1 height=13279500 module=statesync total=33
INF Fetching snapshot chunk chunk=32 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=9 format=1 height=13279500 module=statesync total=33
INF Discovered new snapshot format=1 hash="��}9�\x11�Tr���\x02\t�+^�Y�\x16�O'\x15dmh��\"�" height=11994000 module=statesync
INF Discovered new snapshot format=1 hash="[sN\x12�j�\f2f\t�\t'p���\x0f�4�Q��HO\x0eы=\x00" height=11992000 module=statesync
INF Discovered new snapshot format=1 hash="�����'2�,�\x1a��-�2*\x17xCF�\x10�ا/O��$0" height=13274000 module=statesync
INF Discovered new snapshot format=1 hash="0�E�7�\x1b;\x0f[0��ό戊Zys���Ҹ\x1c.\x02\fA�" height=13272000 module=statesync
INF Discovered new snapshot format=1 hash="g�I�2h-��x\x1d�F\b\"�1K(�_lȖ\x034}�W�-�" height=13270000 module=statesync
INF Applied snapshot chunk to ABCI app chunk=10 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=11 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=12 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=13 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=14 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=15 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=16 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=17 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=18 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=19 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=20 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=21 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=22 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=23 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=24 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=25 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=26 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=27 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=28 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=29 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=30 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=31 format=1 height=13279500 module=statesync total=33
INF Applied snapshot chunk to ABCI app chunk=32 format=1 height=13279500 module=statesync total=33
INF Verified ABCI app appHash="�<*\x7f)�\x01]\x0f���x}\nEѥǠq�NM^^B\x17J\x1e9\x05" height=13279500 module=statesync
INF Snapshot restored format=1 hash="�[\x17�\x1cO�\x0es�\x03@OL�HE]�\x01?�\aIi�G>�;-�" height=13279500 module=statesync
INF service start impl=BlockPool module=blockchain msg={}
INF minted coins from module account amount=3517732uakt from=mint module=x/bank
INF executed block height=13279501 module=state num_invalid_txs=0 num_valid_txs=0
INF commit synced commit=436F6D6D697449447B5B3133362032312032333820313730203138372036203230352031393320373520313035203136302037342032303820352031383120333220353520323532203233332033322032332037203134342031303920313520323030203134312031383920393420323038203936203136385D3A4341413130447D
INF committed state app_hash=8815EEAABB06CDC14B69A04AD005B52037FCE9201707906D0FC88DBD5ED060A8 height=13279501 module=state num_txs=0
INF indexed block exents height=13279501 module=txindex
INF minted coins from module account amount=3517732uakt from=mint module=x/bank
INF executed block height=13279502 module=state num_invalid_txs=0 num_valid_txs=0
INF commit synced commit=436F6D6D697449447B5B3133322031303920393320313820323133203134332032313320302031353120323237203139312032323220343220363320323232203231322031353020323435203136392031303920313735203139392031363620393420323335203133203132372037382038322031313020313339203136325D3A4341413130457D
INF committed state app_hash=846D5D12D58FD50097E3BFDE2A3FDED496F5A96DAFC7A65EEB0D7F4E526E8BA2 height=13279502 module=state num_txs=0
INF indexed block exents height=13279502 module=txindex
INF minted coins from module account amount=3517732uakt from=mint module=x/bank
INF executed block height=13279503 module=state num_invalid_txs=0 num_valid_txs=2
INF commit synced commit=436F6D6D697449447B5B313330203133382032313920313735203931203131382031363720313436203234382032313620373820323320353420313130203233372031322032353220353720323331203134203530203131352036362033332031393220323135203633203235302031303320323020313237203132335D3A4341413130467D
INF committed state app_hash=828ADBAF5B76A792F8D84E17366EED0CFC39E70E32734221C0D73FFA67147F7B height=13279503 module=state num_txs=2
INF indexed block exents height=13279503 module=txindex
INF minted coins from module account amount=3517733uakt from=mint module=x/bank
INF executed block height=13279504 module=state num_invalid_txs=0 num_valid_txs=0
INF commit synced commit=436F6D6D697449447B5B3739203134372031323420323136203130342031323120363120383920313931203234382031333620313538203532203138322031353720343220313937203130322039372032333820323431203130302032382031323620363120373320383320323035203937203234203635203138335D3A4341413131307D
...
...
...
INF committed state app_hash=6442C8A645026F5971F8B612F0E828B0259F4FD4A259310B68FDFD7DE0D4AC45 height=13279912 module=state num_txs=1
INF indexed block exents height=13279912 module=txindex
INF minted coins from module account amount=3517863uakt from=mint module=x/bank
INF executed block height=13279913 module=state num_invalid_txs=0 num_valid_txs=0
INF commit synced commit=436F6D6D697449447B5B32333020343920313439203338203837203830203133302031303720313031203233203139322035332036312034362031383520363120323820323337203139302032303220313635203933203234392032323420313033203431203134362038352032302038203334203135365D3A4341413241397D
INF halting node per configuration height=13279913
INF committed state app_hash=E63195265750826B6517C0353D2EB93D1CEDBECAA55DF9E0672992551408229C height=13279913 module=state num_txs=0
...
INF Closing rpc listener listener={"Listener":{}}
INF RPC HTTP server stopped err="accept tcp [::]:26657: use of closed network connection" module=rpc-server
ERR Error serving server err="accept tcp [::]:26657: use of closed network connection"
INF exiting...

• pod went from init to Running state as expected

root@node1:~/helm-charts# kubectl -n akash-services get pods -l app=akash-node
NAME             READY   STATUS    RESTARTS   AGE
akash-node-1-0   1/1     Running   0          6m21s

• main pod continued from where init pod state-synced and is running well

root@node1:~/helm-charts# kubectl -n akash-services logs akash-node-1-0 --tail=10000 -f |grep -iv p2p
Defaulted container "akash-node" out of: akash-node, init (init)
INF initializing upgrade `v0.26.0`
INF initializing upgrade `v0.24.0`
INF initializing upgrade `akash_v0.15.0_cosmos_v0.44.x`
INF initializing upgrade `v0.20.0`
INF starting node with ABCI Tendermint in-process
INF service start impl=multiAppConn module=proxy msg={}
INF service start connection=query impl=localClient module=abci-client msg={}
INF service start connection=snapshot impl=localClient module=abci-client msg={}
INF service start connection=mempool impl=localClient module=abci-client msg={}
INF service start connection=consensus impl=localClient module=abci-client msg={}
INF service start impl=EventBus module=events msg={}
INF service start impl=PubSub module=pubsub msg={}
INF service start impl=IndexerService module=txindex msg={}
INF Found local state with non-zero height, skipping state sync
INF ABCI Handshake App Info hash="�1�&WP�ke\x17�5=.�=\x1c��ʥ]��g)�U\x14\b\"�" height=13279913 module=consensus protocol-version=0 software-version=v0.26.1
INF ABCI Replay Blocks appHeight=13279913 module=consensus stateHeight=13279913 storeHeight=13279913
INF Completed ABCI Handshake - CometBFT and App are synced appHash="�1�&WP�ke\x17�5=.�=\x1c��ʥ]��g)�U\x14\b\"�" appHeight=13279913 module=consensus
INF This node is not a validator addr=F5839081C551B256BC1DB1015F4A684F62A31E47 module=consensus pubKey=FQqWwuwqe89YRn7K2yxCjFZfTh+Ze9putLn4LsiZjYM=
INF service start impl=Node msg={}
INF Starting pprof server laddr=0.0.0.0:6060
INF service start impl=BlockchainReactor module=blockchain msg={}
INF service start impl=BlockPool module=blockchain msg={}
INF service start impl=ConsensusReactor module=consensus msg={}
INF Reactor  module=consensus waitSync=true
INF service start impl=Evidence module=evidence msg={}
INF serve module=rpc-server msg={}
INF service start impl=StateSync module=statesync msg={}
INF service start impl=PEX module=pex msg={}
INF minted coins from module account amount=3517864uakt from=mint module=x/bank
INF executed block height=13279914 module=state num_invalid_txs=0 num_valid_txs=0
INF commit synced commit=436F6D6D697449447B5B31393220323333203132392032353320313836203134352031383820313233203833203830203137322031363120323031203137382031363620343220313233203136352034342033302031303520363420323420323438203137312032303520323438203230352033382038342036352033395D3A4341413241417D
INF committed state app_hash=C0E981FDBA91BC7B5350ACA1C9B2A62A7BA52C1E694018F8ABCDF8CD26544127 height=13279914 module=state num_txs=0
INF indexed block exents height=13279914 module=txindex
INF minted coins from module account amount=3517864uakt from=mint module=x/bank
INF packet sent dst_channel=channel-1 dst_port=transfer module=x/ibc/channel sequence=155120 src_channel=channel-9 src_port=transfer
INF IBC fungible token transfer amount=1200000 module=x/ibc-transfer receiver=osmo1hn7f4x23xtajz3hhevy83pcm7n0m0wpj6cpyap sender=akash1hn7f4x23xtajz3hhevy83pcm7n0m0wpjlclnjf token=uakt
INF executed block height=13279915 module=state num_invalid_txs=0 num_valid_txs=1
INF commit synced commit=436F6D6D697449447B5B3233372032333320323530203131302031363020313430203130332031313420313930203131382036332034342038322032343520323235203934203331203820313932203936203235352032303120312031333420313332203139332032323520363820363420313730203231332032395D3A4341413241427D
INF committed state app_hash=EDE9FA6EA08C6772BE763F2C52F5E15E1F08C060FFC9018684C1E14440AAD51D height=13279915 module=state num_txs=1
INF indexed block exents height=13279915 module=txindex
INF minted coins from module account amount=3517864uakt from=mint module=x/bank
INF executed block height=13279916 module=state num_invalid_txs=0 num_valid_txs=0
INF commit synced commit=436F6D6D697449447B5B313335203231372031353720313538203138312031353120353120313832203530203134342031313920313935203132362033203136203136332031383120323820353020363620323532203131332039302031333320313736203232372032303320343920313638203436203838203139375D3A4341413241437D

• the RPC node looks good and caught up:

# export AKASH_NODE="http://$(kubectl -n akash-services get ep akash-node-1 -o jsonpath='{.subsets[0].addresses[0].ip}'):26657"

# curl -s $AKASH_NODE/status | jq -r .
...
    "sync_info": {
      "latest_block_hash": "9E8B78FFBC331CD0781B76864A156E0B35E49B756FA5D07A2FA49E9893C5B0BE",
      "latest_app_hash": "0B707904476D2F5CDB883FE299BCE56EF5462AB8ABF59FF559FDCBE98239788E",
      "latest_block_height": "13280027",
      "latest_block_time": "2023-10-17T13:42:43.374318326Z",
      "earliest_block_hash": "611DBFB8B08CFADA3270A085B193E7B797BF2519635DA975698EEB0EE3ED3930",
      "earliest_app_hash": "A33C2A7F29EF015D0FC9FDF9787D0A45D1A5C7A071FB4E4D5E5E42174A1E3905",
      "earliest_block_height": "13279501",
      "earliest_block_time": "2023-10-17T12:50:23.442916835Z",
      "catching_up": false
    },
...

• deleting the pod caused it to state-sync again as expected

because no persistent storage was used, pod deletion permanently removes any ephemeral data.

• bouncing the pod by killing its process causes instant node restart as expected

# kubectl -n akash-services get pods -l app=akash-node
NAME             READY   STATUS    RESTARTS   AGE
akash-node-1-0   1/1     Running   0          16m

# kubectl -n akash-services exec -ti $(kubectl -n akash-services get pods -l app=akash-node -o name) -- bash
Defaulted container "akash-node" out of: akash-node, init (init)
root@akash-node-1-0:/# kill 1
root@akash-node-1-0:/# command terminated with exit code 137

# kubectl -n akash-services get pods -l app=akash-node
NAME             READY   STATUS    RESTARTS      AGE
akash-node-1-0   1/1     Running   1 (11s ago)   16m

# kubectl -n akash-services logs akash-node-1-0 --tail=1000 -f  |grep -iv p2p
...
INF indexed block exents height=13280232 module=txindex


# export AKASH_NODE="http://$(kubectl -n akash-services get ep akash-node-1 -o jsonpath='{.subsets[0].addresses[0].ip}'):26657"
# curl -s $AKASH_NODE/status | jq -r .
...
      "latest_block_height": "13280235",
      "latest_block_time": "2023-10-17T14:03:22.799449103Z",
...