From f33d297ca7d5c74289ccf60a5ddd8bae07824d79 Mon Sep 17 00:00:00 2001 From: Will Hester Date: Thu, 11 Apr 2024 15:42:58 +0100 Subject: [PATCH 1/5] new gh action step --- .github/workflows/manual-deploy-testnet-l2.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/manual-deploy-testnet-l2.yml b/.github/workflows/manual-deploy-testnet-l2.yml index dc992abaec..f8983c7360 100644 --- a/.github/workflows/manual-deploy-testnet-l2.yml +++ b/.github/workflows/manual-deploy-testnet-l2.yml @@ -112,6 +112,17 @@ jobs: inlineScript: | $(az resource list --tag ${{ vars.AZURE_DEPLOY_GROUP_L2 }}=true --query '[]."id"' -o tsv | xargs -n1 az resource delete --verbose -g Testnet --ids) || true + # Delete old database tables from previous deployment + - name: 'Delete host databases' + uses: azure/CLI@v1 + with: + inlineScript: | + databases=$(az postgres server list --resource-group Testnet --query "[?starts_with(name, 'host_')].[name]" -o tsv) + + for db in $databases; do + az postgres db delete --name "$db" --resource-group Testnet --server-name postgres-ten-${{ github.event.inputs.testnet_type }}.postgres.database.azure.com --yes + done + - name: 'Upload L1 deployer container logs' uses: actions/upload-artifact@v3 with: From c7ea1b85e81178071703dae747df5f4ca2c8906c Mon Sep 17 00:00:00 2001 From: Will Hester Date: Thu, 11 Apr 2024 16:01:03 +0100 Subject: [PATCH 2/5] fix az query --- .github/workflows/manual-deploy-testnet-l2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual-deploy-testnet-l2.yml b/.github/workflows/manual-deploy-testnet-l2.yml index f8983c7360..57b6f60983 100644 --- a/.github/workflows/manual-deploy-testnet-l2.yml +++ b/.github/workflows/manual-deploy-testnet-l2.yml @@ -117,7 +117,7 @@ jobs: uses: azure/CLI@v1 with: inlineScript: | - databases=$(az postgres server list --resource-group Testnet --query "[?starts_with(name, 'host_')].[name]" -o tsv) + databases=$(az postgres flexible-server db list --resource-group Testnet --server-name postgres-ten-${{ github.event.inputs.testnet_type }} --query "[?starts_with(name, 'host_')].[name]" -o tsv) for db in $databases; do az postgres db delete --name "$db" --resource-group Testnet --server-name postgres-ten-${{ github.event.inputs.testnet_type }}.postgres.database.azure.com --yes From d865f78fccae3180f1c00d64fa7a016aff6ddc37 Mon Sep 17 00:00:00 2001 From: Will Hester Date: Thu, 11 Apr 2024 16:32:14 +0100 Subject: [PATCH 3/5] add flexible server arg --- .github/workflows/manual-deploy-testnet-l2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual-deploy-testnet-l2.yml b/.github/workflows/manual-deploy-testnet-l2.yml index 57b6f60983..13e7d2b897 100644 --- a/.github/workflows/manual-deploy-testnet-l2.yml +++ b/.github/workflows/manual-deploy-testnet-l2.yml @@ -120,7 +120,7 @@ jobs: databases=$(az postgres flexible-server db list --resource-group Testnet --server-name postgres-ten-${{ github.event.inputs.testnet_type }} --query "[?starts_with(name, 'host_')].[name]" -o tsv) for db in $databases; do - az postgres db delete --name "$db" --resource-group Testnet --server-name postgres-ten-${{ github.event.inputs.testnet_type }}.postgres.database.azure.com --yes + az postgres flexible-server db delete --name "$db" --resource-group Testnet --server-name postgres-ten-${{ github.event.inputs.testnet_type }} --yes done - name: 'Upload L1 deployer container logs' From 75dfd35e31a7161f5fef1ad3cc24718877cfacca Mon Sep 17 00:00:00 2001 From: Will Hester Date: Thu, 11 Apr 2024 16:42:52 +0100 Subject: [PATCH 4/5] switch to database-name --- .github/workflows/manual-deploy-testnet-l2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/manual-deploy-testnet-l2.yml b/.github/workflows/manual-deploy-testnet-l2.yml index 13e7d2b897..ecbb900159 100644 --- a/.github/workflows/manual-deploy-testnet-l2.yml +++ b/.github/workflows/manual-deploy-testnet-l2.yml @@ -120,7 +120,7 @@ jobs: databases=$(az postgres flexible-server db list --resource-group Testnet --server-name postgres-ten-${{ github.event.inputs.testnet_type }} --query "[?starts_with(name, 'host_')].[name]" -o tsv) for db in $databases; do - az postgres flexible-server db delete --name "$db" --resource-group Testnet --server-name postgres-ten-${{ github.event.inputs.testnet_type }} --yes + az postgres flexible-server db delete --database-name "$db" --resource-group Testnet --server-name postgres-ten-${{ github.event.inputs.testnet_type }} --yes done - name: 'Upload L1 deployer container logs' From 1ac1904bd346add3e2d44d956b31553789649c60 Mon Sep 17 00:00:00 2001 From: Will Hester Date: Fri, 12 Apr 2024 10:45:47 +0100 Subject: [PATCH 5/5] set max connections on postgres db & add batch by tx hash check to in-mem test --- go/host/storage/init/postgres/postgres.go | 2 + .../simulation/p2p/in_mem_obscuro_client.go | 25 +++++++++ integration/simulation/validate_chain.go | 51 +++++++++++++++++++ 3 files changed, 78 insertions(+) diff --git a/go/host/storage/init/postgres/postgres.go b/go/host/storage/init/postgres/postgres.go index 716ad6727f..7b2601e850 100644 --- a/go/host/storage/init/postgres/postgres.go +++ b/go/host/storage/init/postgres/postgres.go @@ -14,6 +14,7 @@ import ( const ( defaultDatabase = "postgres" + maxDBPoolSize = 100 ) func CreatePostgresDBConnection(baseURL string, dbName string) (*sql.DB, error) { @@ -46,6 +47,7 @@ func CreatePostgresDBConnection(baseURL string, dbName string) (*sql.DB, error) dbURL = fmt.Sprintf("%s%s", baseURL, dbName) db, err = sql.Open("postgres", dbURL) + db.SetMaxOpenConns(maxDBPoolSize) if err != nil { return nil, fmt.Errorf("failed to connect to PostgreSQL database %s: %v", dbName, err) } diff --git a/integration/simulation/p2p/in_mem_obscuro_client.go b/integration/simulation/p2p/in_mem_obscuro_client.go index 1992769eda..6db8332e1c 100644 --- a/integration/simulation/p2p/in_mem_obscuro_client.go +++ b/integration/simulation/p2p/in_mem_obscuro_client.go @@ -109,6 +109,9 @@ func (c *inMemObscuroClient) Call(result interface{}, method string, args ...int case rpc.GetRollupListing: return c.getRollupListing(result, args) + case rpc.GetPublicTransactionData: + return c.getPublicTransactionData(result, args) + default: return fmt.Errorf("RPC method %s is unknown", method) } @@ -369,6 +372,28 @@ func (c *inMemObscuroClient) getRollupListing(result interface{}, args []interfa return nil } +func (c *inMemObscuroClient) getPublicTransactionData(result interface{}, args []interface{}) error { + if len(args) != 1 { + return fmt.Errorf("expected 1 arg to %s, got %d", rpc.GetPublicTransactionData, len(args)) + } + pagination, ok := args[0].(*common.QueryPagination) + if !ok { + return fmt.Errorf("first arg to %s is of type %T, expected type int", rpc.GetPublicTransactionData, args[0]) + } + + txs, err := c.tenScanAPI.GetPublicTransactionData(pagination) + if err != nil { + return fmt.Errorf("`%s` call failed. Cause: %w", rpc.GetPublicTransactionData, err) + } + + res, ok := result.(*common.TransactionListingResponse) + if !ok { + return fmt.Errorf("result is of type %T, expected *common.BatchListingResponseDeprecated", result) + } + *res = *txs + return nil +} + // getEncryptedBytes expects args to have a single element and it to be of type bytes (client doesn't know anything about what's getting passed through on sensitive methods) func getEncryptedBytes(args []interface{}, methodName string) ([]byte, error) { if len(args) != 1 { diff --git a/integration/simulation/validate_chain.go b/integration/simulation/validate_chain.go index 3d29af563e..07bc422f96 100644 --- a/integration/simulation/validate_chain.go +++ b/integration/simulation/validate_chain.go @@ -722,6 +722,11 @@ func checkTenscan(t *testing.T, s *Simulation) { checkTotalTransactions(t, client, idx) checkForLatestBatches(t, client, idx) checkForLatestRollups(t, client, idx) + + txHashes := getLatestTransactions(t, client, idx) + for _, txHash := range txHashes { + checkBatchFromTxs(t, client, txHash, idx) + } } } @@ -762,3 +767,49 @@ func checkForLatestRollups(t *testing.T, client rpc.Client, nodeIdx int) { t.Errorf("node %d: expected at least %d transactions, but only received %d", nodeIdx, 5, len(latestRollups.RollupsData)) } } + +func getLatestTransactions(t *testing.T, client rpc.Client, nodeIdx int) []gethcommon.Hash { + var transactionResponse common.TransactionListingResponse + var txHashes []gethcommon.Hash + pagination := common.QueryPagination{Offset: uint64(0), Size: uint(5)} + err := client.Call(&transactionResponse, rpc.GetPublicTransactionData, &pagination) + if err != nil { + t.Errorf("node %d: could not retrieve latest transactions. Cause: %s", nodeIdx, err) + } + + for _, transaction := range transactionResponse.TransactionsData { + txHashes = append(txHashes, transaction.TransactionHash) + } + + return txHashes +} + +// Retrieves the batch using the transaction hash, and validates it. +func checkBatchFromTxs(t *testing.T, client rpc.Client, txHash gethcommon.Hash, nodeIdx int) { + var batchByTx *common.ExtBatch + err := client.Call(&batchByTx, rpc.GetBatchByTx, txHash) + if err != nil { + t.Errorf("node %d: could not retrieve batch for transaction. Cause: %s", nodeIdx, err) + return + } + + var containsTx bool + for _, txHashFromBatch := range batchByTx.TxHashes { + if txHashFromBatch == txHash { + containsTx = true + } + } + if !containsTx { + t.Errorf("node %d: retrieved batch by transaction, but transaction was missing from batch", nodeIdx) + } + + var batchByHash *common.ExtBatch + err = client.Call(&batchByHash, rpc.GetBatch, batchByTx.Header.Hash()) + if err != nil { + t.Errorf("node %d: could not retrieve batch by hash. Cause: %s", nodeIdx, err) + return + } + if batchByHash.Header.Hash() != batchByTx.Header.Hash() { + t.Errorf("node %d: retrieved batch by hash, but hash was incorrect", nodeIdx) + } +}