From 0461856634b00729f6f63cf813f8696d96cf3aba Mon Sep 17 00:00:00 2001 From: Vitruvius Date: Thu, 24 Feb 2022 09:38:14 -0800 Subject: [PATCH 1/2] fix clickhouse syntax error --- dao/clickhouse/accounts.go | 2 +- dao/clickhouse/blocks.go | 4 ++-- dao/clickhouse/rewards.go | 2 +- dao/clickhouse/transactions.go | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dao/clickhouse/accounts.go b/dao/clickhouse/accounts.go index 852e830..5354f4d 100644 --- a/dao/clickhouse/accounts.go +++ b/dao/clickhouse/accounts.go @@ -45,7 +45,7 @@ func (cl Clickhouse) CreateAccountBalances(balances []dmodels.AccountBalance) (e return err } stmt, err := tx.Prepare( - fmt.Sprintf("INSERT INTO %s (blk_lvl, blk_time, acb_account, acb_nonce, acb_general_balance, acb_escrow_balance_active, acb_escrow_balance_share, acb_escrow_debonding_active, acb_escrow_debonding_share, acb_delegations_balance, acb_debonding_delegations_balance, acb_self_delegation_balance , acb_commission_schedule)"+ + fmt.Sprintf("INSERT INTO %s (blk_lvl, blk_time, acb_account, acb_nonce, acb_general_balance, acb_escrow_balance_active, acb_escrow_balance_share, acb_escrow_debonding_active, acb_escrow_debonding_share, acb_delegations_balance, acb_debonding_delegations_balance, acb_self_delegation_balance , acb_commission_schedule) "+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", dmodels.AccountBalanceTable)) if err != nil { return err diff --git a/dao/clickhouse/blocks.go b/dao/clickhouse/blocks.go index 407d4e8..14b6acc 100644 --- a/dao/clickhouse/blocks.go +++ b/dao/clickhouse/blocks.go @@ -21,7 +21,7 @@ func (cl Clickhouse) CreateBlocks(blocks []dmodels.Block) (err error) { return err } stmt, err := tx.Prepare( - fmt.Sprintf("INSERT INTO %s (blk_lvl, blk_created_at, blk_hash, blk_proposer_address, blk_validator_hash, blk_epoch)"+ + fmt.Sprintf("INSERT INTO %s (blk_lvl, blk_created_at, blk_hash, blk_proposer_address, blk_validator_hash, blk_epoch) "+ "VALUES (?, ?, ?, ?, ?, ?)", dmodels.BlocksTable)) if err != nil { return err @@ -64,7 +64,7 @@ func (cl Clickhouse) CreateBlockSignatures(blocks []dmodels.BlockSignature) erro return err } stmt, err := tx.Prepare( - fmt.Sprintf("INSERT INTO %s (blk_lvl, sig_timestamp, sig_block_id_flag, sig_validator_address, sig_blk_signature)"+ + fmt.Sprintf("INSERT INTO %s (blk_lvl, sig_timestamp, sig_block_id_flag, sig_validator_address, sig_blk_signature) "+ "VALUES (?, ?, ?, ?, ?)", dmodels.BlockSignaturesTable)) if err != nil { return err diff --git a/dao/clickhouse/rewards.go b/dao/clickhouse/rewards.go index 8f2d4fb..91e4916 100644 --- a/dao/clickhouse/rewards.go +++ b/dao/clickhouse/rewards.go @@ -14,7 +14,7 @@ func (cl Clickhouse) CreateRewards(rewards []dmodels.Reward) error { return err } stmt, err := tx.Prepare( - fmt.Sprintf("INSERT INTO %s (blk_lvl, blk_epoch, reg_entity_address, acb_account, rwd_amount, rwd_type, created_at)"+ + fmt.Sprintf("INSERT INTO %s (blk_lvl, blk_epoch, reg_entity_address, acb_account, rwd_amount, rwd_type, created_at) "+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?)", dmodels.RewardsTable)) if err != nil { return err diff --git a/dao/clickhouse/transactions.go b/dao/clickhouse/transactions.go index 2a5b01e..c6b14f7 100644 --- a/dao/clickhouse/transactions.go +++ b/dao/clickhouse/transactions.go @@ -23,7 +23,7 @@ func (cl Clickhouse) CreateTransfers(transfers []dmodels.Transaction) error { } stmt, err := tx.Prepare( - fmt.Sprintf("INSERT INTO %s (blk_lvl, blk_hash, tx_time, tx_hash, tx_amount, tx_escrow_amount, tx_escrow_reclaim_amount, tx_type, tx_status, tx_error, tx_sender, tx_receiver, tx_nonce, tx_fee, tx_gas_limit, tx_gas_price)"+ + fmt.Sprintf("INSERT INTO %s (blk_lvl, blk_hash, tx_time, tx_hash, tx_amount, tx_escrow_amount, tx_escrow_reclaim_amount, tx_type, tx_status, tx_error, tx_sender, tx_receiver, tx_nonce, tx_fee, tx_gas_limit, tx_gas_price) "+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", dmodels.TransactionsTable)) if err != nil { return err @@ -77,7 +77,7 @@ func (cl Clickhouse) CreateRegisterNodeTransactions(txs []dmodels.NodeRegistryTr } stmt, err := tx.Prepare( - fmt.Sprintf("INSERT INTO %s (blk_lvl, tx_time, tx_hash, reg_id, reg_address, reg_entity_id, reg_entity_address, reg_expiration, reg_p2p_id, reg_consensus_id, reg_consensus_address, reg_physical_address, reg_roles)"+ + fmt.Sprintf("INSERT INTO %s (blk_lvl, tx_time, tx_hash, reg_id, reg_address, reg_entity_id, reg_entity_address, reg_expiration, reg_p2p_id, reg_consensus_id, reg_consensus_address, reg_physical_address, reg_roles) "+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", dmodels.RegisterNodeTable)) if err != nil { return err @@ -129,7 +129,7 @@ func (cl Clickhouse) CreateRegisterEntityTransactions(txs []dmodels.EntityRegist } stmt, err := tx.Prepare( - fmt.Sprintf("INSERT INTO %s (blk_lvl, tx_time, tx_hash, reg_entity_id, reg_entity_address, reg_nodes)"+ + fmt.Sprintf("INSERT INTO %s (blk_lvl, tx_time, tx_hash, reg_entity_id, reg_entity_address, reg_nodes) "+ "VALUES (?, ?, ?, ?, ?, ?)", dmodels.RegisterEntityTable)) if err != nil { return err From 9278c457666f100cd9ebf85ced6738ccbc9779b1 Mon Sep 17 00:00:00 2001 From: Vitruvius Date: Mon, 7 Mar 2022 12:31:06 -0800 Subject: [PATCH 2/2] fix migration syntax --- dao/clickhouse/migrations/004_block_view.up.sql | 6 +++--- dao/clickhouse/migrations/008_entity_nodes_view.up.sql | 8 ++++---- dao/clickhouse/migrations/010_validators.up.sql | 5 +++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/dao/clickhouse/migrations/004_block_view.up.sql b/dao/clickhouse/migrations/004_block_view.up.sql index 562bf42..2639ad4 100644 --- a/dao/clickhouse/migrations/004_block_view.up.sql +++ b/dao/clickhouse/migrations/004_block_view.up.sql @@ -1,7 +1,7 @@ --CREATE VIEW IF NOT EXISTS block_row_view AS --select * from blocks ANY LEFT JOIN -- (select blk_lvl, sum(tx_gas_price), sum(tx_fee), count() ---from transactions group by blk_lvl) as s USING blk_lvl ORDER BY blk_lvl DESC; +--from transactions group by blk_lvl) as s USING blk_lvl ORDER BY blk_lvl DESC -- HERE IS FULL SCAN OF txs table, cause it ordered by hash, not block height -- SO we have create Materialized view to store ordered data @@ -37,6 +37,6 @@ ANY LEFT JOIN GROUP BY blk_lvl ) AS s USING (blk_lvl) ORDER BY blk_lvl DESC -) +); -CREATE VIEW IF NOT EXISTS blocks_sig_count AS select blk_lvl, count() sig_count from block_signatures group by blk_lvl; \ No newline at end of file +CREATE VIEW IF NOT EXISTS blocks_sig_count AS select blk_lvl, count() sig_count FROM block_signatures GROUP BY blk_lvl; diff --git a/dao/clickhouse/migrations/008_entity_nodes_view.up.sql b/dao/clickhouse/migrations/008_entity_nodes_view.up.sql index 1fa1c46..3136ee1 100644 --- a/dao/clickhouse/migrations/008_entity_nodes_view.up.sql +++ b/dao/clickhouse/migrations/008_entity_nodes_view.up.sql @@ -9,7 +9,7 @@ -- select tx_receiver, tx_sender, sum(tx_escrow_reclaim_amount) output -- from transactions -- where tx_type = 'reclaimescrow' and tx_status = 1 --- group by tx_receiver, tx_sender) remove USING tx_receiver, tx_sender; +-- group by tx_receiver, tx_sender) remove USING tx_receiver, tx_sender -- ┌─tx_receiver────────────────────────────────────┬─tx_sender──────────────────────────────────────┬────────escrow_since─┬────────input─┬─output─┬──────balance─┐ -- │ oasis1qq0xmq7r0z9sdv02t5j9zs7en3n6574gtg8v9fyt │ oasis1qr07pxep97z79d5qnsvcue85pr924rt34gdguytc │ 2022-02-09 18:26:36 │ 100000000000 │ 0 │ 100000000000 │ -- │ oasis1qp60saapdcrhe5zp3c3zk52r4dcfkr2uyuc5qjxp │ oasis1qqjhx3qsfyevtyxpl58dxnmqrzkl9mceys5kjkux │ 2022-02-12 16:46:46 │ 110900000000 │ 0 │ 110900000000 │ @@ -40,8 +40,7 @@ AS from transactions where tx_type = 'reclaimescrow' and tx_status = 1 group by tx_receiver, tx_sender) remove USING tx_receiver, tx_sender -) - +); CREATE VIEW IF NOT EXISTS entity_active_depositors_counter_view AS SELECT tx_receiver reg_entity_address, count() depositors_num from entity_depositors_view @@ -118,6 +117,7 @@ ANY LEFT JOIN GROUP BY sig_validator_address ) AS blk USING (reg_consensus_address) ) + -- Memory troubles: try to fix with MVIEW. ALSO: this DDL was changed! -- CREATE VIEW IF NOT EXISTS entity_nodes_view AS -- select * @@ -141,5 +141,5 @@ ANY LEFT JOIN -- --Blocks signatures count -- select sig_validator_address reg_consensus_address, max(sig_timestamp) last_signature_time, count() c_block_signatures -- from block_signatures --- group by sig_validator_address) blk USING reg_consensus_address; +-- group by sig_validator_address) blk USING reg_consensus_address diff --git a/dao/clickhouse/migrations/010_validators.up.sql b/dao/clickhouse/migrations/010_validators.up.sql index 8315d1f..626163a 100644 --- a/dao/clickhouse/migrations/010_validators.up.sql +++ b/dao/clickhouse/migrations/010_validators.up.sql @@ -140,7 +140,7 @@ from ( group by reg_entity_address ) val_lvl USING reg_entity_address) validator ANY LEFT JOIN (SELECT acb_account reg_entity_address, acb_escrow_balance_active, - acb_general_balance, acb_escrow_balance_share, acb_escrow_debonding_active,\ + acb_general_balance, acb_escrow_balance_share, acb_escrow_debonding_active, acb_delegations_balance , acb_debonding_delegations_balance , acb_self_delegation_balance, acb_commission_schedule , depositors_num @@ -152,4 +152,5 @@ from ( OR signatures > 0 OR reg_expiration >= (select max(blk_epoch) from blocks)) prep ANY - LEFT JOIN public_validators USING reg_entity_address; --public validators 134 rows \ No newline at end of file + LEFT JOIN public_validators USING reg_entity_address --public validators 134 rows +;