diff --git a/.github/workflows/manual-deploy-obscuro-gateway.yml b/.github/workflows/manual-deploy-obscuro-gateway.yml index 96986e1f66..55b2c820bb 100644 --- a/.github/workflows/manual-deploy-obscuro-gateway.yml +++ b/.github/workflows/manual-deploy-obscuro-gateway.yml @@ -35,6 +35,14 @@ on: options: - "primary" - "DEXYNTH" + recreate_vm: + description: "Delete and recreate VM" + required: false + default: "false" + type: choice + options: + - "false" + - "true" jobs: validate-inputs: @@ -59,8 +67,7 @@ jobs: INSTANCE_PREFIX="" if [[ "${{ github.event.inputs.instance_type }}" != "primary" ]]; then - INSTANCE_SUFFIX="_${{ github.event.inputs.instance_type }}" - INSTANCE_SUFFIX2="-${{ github.event.inputs.instance_type }}" + INSTANCE_SUFFIX="-${{ github.event.inputs.instance_type }}" INSTANCE_PREFIX="${{ github.event.inputs.instance_type }}_" fi @@ -68,10 +75,10 @@ jobs: echo "INSTANCE_PREFIX=$INSTANCE_PREFIX" >> $GITHUB_ENV # Set infrastructure variables - PUBLIC_IP="${{ github.event.inputs.testnet_type }}-OG-static${INSTANCE_SUFFIX2,,}" - DNS_NAME="obscurogateway-${{ github.event.inputs.testnet_type }}${INSTANCE_SUFFIX2,,}" - VM_NAME="${{ github.event.inputs.testnet_type }}-OG-${{ github.run_number }}${INSTANCE_SUFFIX}" - DEPLOY_GROUP="ObscuroGateway-${{ github.event.inputs.testnet_type }}-${{ github.run_number }}${INSTANCE_SUFFIX}" + PUBLIC_IP="${{ github.event.inputs.testnet_type }}-OG-static${INSTANCE_SUFFIX,,}" + DNS_NAME="obscurogateway-${{ github.event.inputs.testnet_type }}${INSTANCE_SUFFIX,,}" + VM_NAME="${{ github.event.inputs.testnet_type }}-OG${INSTANCE_SUFFIX}" + DEPLOY_GROUP="ObscuroGateway-${{ github.event.inputs.testnet_type }}${INSTANCE_SUFFIX}" VNET_NAME="ObscuroGateway-${{ github.event.inputs.testnet_type }}-01VNET${INSTANCE_SUFFIX}" SUBNET_NAME="ObscuroGateway-${{ github.event.inputs.testnet_type }}-01Subnet${INSTANCE_SUFFIX}" @@ -105,7 +112,6 @@ jobs: done - name: "Print environment variables" - # This is a useful record of what the environment variables were at the time the job ran, for debugging and reference run: | echo "INSTANCE_SUFFIX: $INSTANCE_SUFFIX" echo "INSTANCE_PREFIX: $INSTANCE_PREFIX" @@ -125,7 +131,6 @@ jobs: echo "GATEWAY_TLS_DOMAIN: $GATEWAY_TLS_DOMAIN" - name: "Print GitHub variables" - # This is a useful record of what the environment variables were at the time the job ran, for debugging and reference run: | echo "GitHub Variables = ${{ toJSON(vars) }}" @@ -157,57 +162,79 @@ jobs: DOCKER_BUILDKIT=1 docker build --build-arg TESTNET_TYPE=${{ github.event.inputs.testnet_type }} -t ${{ env.DOCKER_BUILD_TAG_GATEWAY }} -f ./tools/walletextension/enclave.Dockerfile . docker push ${{ env.DOCKER_BUILD_TAG_GATEWAY }} - # This will fail some deletions due to resource dependencies ( ie. you must first delete the vm before deleting the disk) + # If recreate_vm = true, delete VMs and their dependencies - name: "Delete deployed VMs" + if: ${{ github.event.inputs.recreate_vm == 'true' }} uses: azure/CLI@v1 with: inlineScript: | $(az resource list --tag ${{ env.AZURE_DEPLOY_GROUP_GATEWAY }}=true --query '[]."id"' -o tsv | xargs -n1 az resource delete --verbose -g Testnet --ids) || true - # This will clean up any lingering dependencies - might fail if there are no resources to cleanup - name: "Delete VMs dependencies" + if: ${{ github.event.inputs.recreate_vm == 'true' }} uses: azure/CLI@v1 with: inlineScript: | $(az resource list --tag ${{ env.AZURE_DEPLOY_GROUP_GATEWAY }}=true --query '[]."id"' -o tsv | xargs -n1 az resource delete --verbose -g Testnet --ids) || true - - name: "Ensure VM Static Public IP Exists" - uses: azure/CLI@v1 - with: - inlineScript: | - az network public-ip show -g Testnet -n "${{ env.PUBLIC_IP }}" || az network public-ip create -g Testnet -n "${{ env.PUBLIC_IP }}" --allocation-method Static --sku Standard + # If recreate_vm = false, check if VM exists + - name: "Check if VM exists" + if: ${{ github.event.inputs.recreate_vm == 'false' }} + id: check_vm + shell: bash + run: | + if ! az vm show -g Testnet -n "${{ env.VM_NAME }}" &> /dev/null; then + echo "vm_exists=false" >> $GITHUB_ENV + else + echo "vm_exists=true" >> $GITHUB_ENV + fi - - name: "Assign/Update DNS Name for Public IP" + - name: "Ensure VM Static Public IP and DNS if needed" + if: ${{ github.event.inputs.recreate_vm == 'true' || env.vm_exists == 'false' }} uses: azure/CLI@v1 with: inlineScript: | + az network public-ip show -g Testnet -n "${{ env.PUBLIC_IP }}" || az network public-ip create -g Testnet -n "${{ env.PUBLIC_IP }}" --allocation-method Static --sku Standard existing_dns_name=$(az network public-ip show -g Testnet -n "${{ env.PUBLIC_IP }}" --query dnsSettings.domainNameLabel -o tsv) if [ -z "$existing_dns_name" ]; then az network public-ip update -g Testnet -n "${{ env.PUBLIC_IP }}" --dns-name "${{ env.DNS_NAME }}" fi - - name: "Create VM for Gateway node on Azure" + - name: "Create VM if it doesn't exist (recreate_vm=false)" + if: ${{ github.event.inputs.recreate_vm == 'false' && env.vm_exists == 'false' }} uses: azure/CLI@v1 with: inlineScript: | az vm create -g Testnet -n "${{ env.VM_NAME }}" \ - --admin-username obscurouser --admin-password "${{ secrets.OBSCURO_NODE_VM_PWD }}" \ - --public-ip-address "${{ env.PUBLIC_IP }}" \ - --tags deploygroup="${{ env.DEPLOY_GROUP }}" ${{ env.AZURE_DEPLOY_GROUP_GATEWAY }}=true \ - --vnet-name "${{ env.VNET_NAME }}" --subnet "${{ env.SUBNET_NAME }}" \ - --size Standard_DC2s_v3 --storage-sku StandardSSD_LRS --image ObscuroConfUbuntu \ - --authentication-type password - - - name: "Open TEN node-${{ matrix.host_id }} ports on Azure" + --admin-username obscurouser --admin-password "${{ secrets.OBSCURO_NODE_VM_PWD }}" \ + --public-ip-address "${{ env.PUBLIC_IP }}" \ + --tags deploygroup="${{ env.DEPLOY_GROUP }}" ${{ env.AZURE_DEPLOY_GROUP_GATEWAY }}=true \ + --vnet-name "${{ env.VNET_NAME }}" --subnet "${{ env.SUBNET_NAME }}" \ + --size Standard_DC2s_v3 --storage-sku StandardSSD_LRS --image ObscuroConfUbuntu \ + --authentication-type password + + az vm open-port -g Testnet -n "${{ env.VM_NAME }}" --port 80,81,443 + + # Allow time for VM initialization + sleep 30 + + - name: "Create VM if recreate_vm = true" + if: ${{ github.event.inputs.recreate_vm == 'true' }} uses: azure/CLI@v1 with: inlineScript: | - az vm open-port -g Testnet -n "${{ env.VM_NAME }}" --port 80,81,443 - - # To overcome issues with critical VM resources being unavailable, we need to wait for the VM to be ready - - name: "Allow time for VM initialization" - shell: bash - run: sleep 30 + az vm create -g Testnet -n "${{ env.VM_NAME }}" \ + --admin-username obscurouser --admin-password "${{ secrets.OBSCURO_NODE_VM_PWD }}" \ + --public-ip-address "${{ env.PUBLIC_IP }}" \ + --tags deploygroup="${{ env.DEPLOY_GROUP }}" ${{ env.AZURE_DEPLOY_GROUP_GATEWAY }}=true \ + --vnet-name "${{ env.VNET_NAME }}" --subnet "${{ env.SUBNET_NAME }}" \ + --size Standard_DC2s_v3 --storage-sku StandardSSD_LRS --image ObscuroConfUbuntu \ + --authentication-type password + + az vm open-port -g Testnet -n "${{ env.VM_NAME }}" --port 80,81,443 + + # Allow time for VM initialization + sleep 30 - name: "Start TEN Gateway on Azure" uses: azure/CLI@v1 @@ -238,8 +265,11 @@ jobs: done curl -fsSL https://get.docker.com -o get-docker.sh && sh ./get-docker.sh + rm -rf /home/obscuro/go-obscuro git clone --depth 1 -b "${{ env.BRANCH_NAME }}" https://github.com/ten-protocol/go-ten.git /home/obscuro/go-obscuro - docker network create --driver bridge node_network || true + if ! docker network inspect node_network >/dev/null 2>&1; then + docker network create --driver bridge node_network + fi cd /home/obscuro/go-obscuro/ # Promtail Integration Start @@ -280,6 +310,9 @@ jobs: - replacement: "${{ env.VM_NAME }}" target_label: "node_name" EOF + + docker stop promtail || true + docker rm promtail || true docker run -d --name promtail \ --network node_network \ @@ -304,29 +337,33 @@ jobs: password: "${{ secrets.LOKI_PASSWORD }}" scrape_configs: # Node metrics - - job_name: node-${{ env.VM_NAME }} - scrape_interval: 5s # Frequent scrapes for node metrics + - job_name: node-${{ env.VM_NAME }} + scrape_interval: 5s static_configs: - targets: - - node_exporter:9100 # Node Exporter instance + - node_exporter:9100 relabel_configs: - source_labels: [job] - target_label: 'node' - replacement: node-${{ env.VM_NAME }} + target_label: "node" + replacement: node-${{ env.VM_NAME }} # Container metrics - - job_name: container-${{ env.VM_NAME }} + - job_name: container-${{ env.VM_NAME }} scrape_interval: 5s static_configs: - targets: - - cadvisor:8080 # cAdvisor instance for container metrics + - cadvisor:8080 relabel_configs: - source_labels: [job] - target_label: 'node' - replacement: container-${{ env.VM_NAME }} + target_label: "node" + replacement: container-${{ env.VM_NAME }} EOF - docker volume create prometheus-data + + docker stop prometheus || true + docker rm prometheus || true + + docker volume create prometheus-data || true docker run -d --name prometheus \ --network node_network \ -p 9090:9090 \ @@ -335,6 +372,10 @@ jobs: prom/prometheus:latest \ --config.file=/etc/prometheus/prometheus.yml + + docker stop node_exporter || true + docker rm node_exporter || true + docker run -d --name node_exporter \ --network node_network \ -p 9100:9100 \ @@ -343,6 +384,10 @@ jobs: quay.io/prometheus/node-exporter:latest \ --path.rootfs=/host + + docker stop cadvisor || true + docker rm cadvisor || true + docker run -d --name cadvisor \ --network node_network \ -p 8080:8080 \ @@ -355,13 +400,16 @@ jobs: gcr.io/cadvisor/cadvisor:latest # Promtail Integration End - # Create a named volume for persistence - docker volume create "${{ env.VM_NAME }}-data" + docker volume create "TENGateway-${{ github.event.inputs.testnet_type }}-data" || true + + # Stop and remove existing container if it exists + docker stop "${{ env.VM_NAME }}" || true + docker rm "${{ env.VM_NAME }}" || true # Start Ten Gateway Container docker run -d -p 80:80 -p 81:81 -p 443:443 --name "${{ env.VM_NAME }}" \ --device /dev/sgx_enclave --device /dev/sgx_provision \ - -v "${{ env.VM_NAME }}-data:/data" \ + -v "TENGateway-${{ github.event.inputs.testnet_type }}-data:/data" \ -e OBSCURO_GATEWAY_VERSION="${{ github.run_number }}-${{ github.sha }}" \ -e OE_SIMULATION=0 \ "${{ env.DOCKER_BUILD_TAG_GATEWAY }}" \ @@ -376,8 +424,6 @@ jobs: -enableTLS=true \ -tlsDomain="${{ env.GATEWAY_TLS_DOMAIN }}" - - # After starting the container, verify the volume mount docker exec "${{ env.VM_NAME }}" sh -c " echo \"Checking volume mount...\"; df -h | grep /data; @@ -391,5 +437,3 @@ jobs: ps aux; " ' - - diff --git a/.github/workflows/manual-deploy-testnet-l2.yml b/.github/workflows/manual-deploy-testnet-l2.yml index 177a034460..0e42f82d66 100644 --- a/.github/workflows/manual-deploy-testnet-l2.yml +++ b/.github/workflows/manual-deploy-testnet-l2.yml @@ -174,6 +174,13 @@ jobs: host_id: 1 - node_l1_ws_lookup: L1_WS_URL_2 host_id: 2 + # sequencer has an HA setup with 2 enclaves + - num_enclaves: 1 + host_id: 0 + - num_enclaves: 1 + host_id: 1 + - num_enclaves: 1 + host_id: 2 steps: - name: 'Extract branch name' @@ -336,6 +343,7 @@ jobs: && sudo go run /home/obscuro/go-obscuro/go/node/cmd \ -is_genesis=${{ matrix.is_genesis }} \ -node_type=${{ matrix.node_type }} \ + -num_enclaves=${{ matrix.num_enclaves }} \ -is_sgx_enabled=true \ -host_id=${{ vars[matrix.node_addr_lookup] }} \ -l1_ws_url=${{ secrets[matrix.node_l1_ws_lookup] }} \ @@ -426,7 +434,7 @@ jobs: shell: bash run: | go run ./testnet/launcher/l2contractdeployer/cmd \ - -l2_host=obscuronode-0-${{ github.event.inputs.testnet_type }}.uksouth.cloudapp.azure.com \ + -l2_host=obscuronode-1-${{ github.event.inputs.testnet_type }}.uksouth.cloudapp.azure.com \ -l1_http_url=${{ secrets.L1_HTTP_URL }} \ -l2_ws_port=81 \ -private_key=${{ secrets.ACCOUNT_PK_WORKER }} \ diff --git a/.github/workflows/manual-upgrade-testnet-l2.yml b/.github/workflows/manual-upgrade-testnet-l2.yml index 4b200e6f39..0b86c14989 100644 --- a/.github/workflows/manual-upgrade-testnet-l2.yml +++ b/.github/workflows/manual-upgrade-testnet-l2.yml @@ -134,6 +134,13 @@ jobs: host_id: 1 - node_l1_ws_lookup: L1_WS_URL_2 host_id: 2 + # sequencer has an HA setup with 2 enclaves + - num_enclaves: 1 + host_id: 0 + - num_enclaves: 1 + host_id: 1 + - num_enclaves: 1 + host_id: 2 steps: - name: 'Extract branch name' @@ -162,6 +169,7 @@ jobs: && sudo go run /home/obscuro/go-obscuro/go/node/cmd \ -is_genesis=${{ matrix.is_genesis }} \ -node_type=${{ matrix.node_type }} \ + -num_enclaves=${{ matrix.num_enclaves }} \ -is_sgx_enabled=true \ -host_id=${{ vars[matrix.node_addr_lookup] }} \ -l1_ws_url=${{ secrets[matrix.node_l1_ws_lookup] }} \ diff --git a/dockerfiles/enclave.Dockerfile b/dockerfiles/enclave.Dockerfile index 2a92372244..2fab3c2305 100644 --- a/dockerfiles/enclave.Dockerfile +++ b/dockerfiles/enclave.Dockerfile @@ -9,7 +9,7 @@ # /home/obscuro/go-obscuro/go/enclave/main contains the executable for the enclave # -FROM ghcr.io/edgelesssys/ego-dev:v1.5.3 AS build-base +FROM ghcr.io/edgelesssys/ego-dev:v1.6.0 AS build-base # setup container data structure RUN mkdir -p /home/obscuro/go-obscuro @@ -36,7 +36,7 @@ FROM build-enclave as build-enclave RUN ego sign enclave.json # Trigger a new build stage and use the smaller ego version: -FROM ghcr.io/edgelesssys/ego-deploy:v1.5.3 +FROM ghcr.io/edgelesssys/ego-deploy:v1.6.0 # Copy just the binary for the enclave into this build stage COPY --from=build-enclave \ diff --git a/go/common/custom_query_types.go b/go/common/custom_query_types.go index 662c71d6c9..58805b92b2 100644 --- a/go/common/custom_query_types.go +++ b/go/common/custom_query_types.go @@ -19,7 +19,6 @@ import "github.com/ethereum/go-ethereum/common" // CustomQuery methods const ( - UserIDRequestCQMethod = "0x0000000000000000000000000000000000000001" ListPrivateTransactionsCQMethod = "0x0000000000000000000000000000000000000002" CreateSessionKeyCQMethod = "0x0000000000000000000000000000000000000003" ActivateSessionKeyCQMethod = "0x0000000000000000000000000000000000000004" diff --git a/go/common/errutil/evm_serialisable.go b/go/common/errutil/evm_serialisable.go index c1eebf9aae..a3382a0fd8 100644 --- a/go/common/errutil/evm_serialisable.go +++ b/go/common/errutil/evm_serialisable.go @@ -1,5 +1,7 @@ package errutil +import "fmt" + // DataError is an API error that encompasses an EVM error with a code and a reason type DataError struct { Code int `json:"code"` @@ -18,3 +20,7 @@ func (e DataError) ErrorCode() int { func (e DataError) ErrorData() interface{} { return e.Reason } + +func (e DataError) String() string { + return fmt.Sprintf("Data Error. Message: %s, Data: %v", e.Err, e.Reason) +} diff --git a/go/common/types.go b/go/common/types.go index f826d09aab..1f4001fc5d 100644 --- a/go/common/types.go +++ b/go/common/types.go @@ -5,6 +5,7 @@ import ( "fmt" "math/big" + gethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/common" @@ -196,3 +197,13 @@ func (cf *ChainFork) String() string { func MaskedSender(address L2Address) L2Address { return common.BigToAddress(big.NewInt(0).Sub(address.Big(), big.NewInt(1))) } + +type SystemContractAddresses map[string]*gethcommon.Address + +func (s *SystemContractAddresses) ToString() string { + var str string + for name, addr := range *s { + str += fmt.Sprintf("%s: %s; ", name, addr.Hex()) + } + return str +} diff --git a/go/config/defaults/0-base-config.yaml b/go/config/defaults/0-base-config.yaml index 60373b59c9..e8dd4cbb2f 100644 --- a/go/config/defaults/0-base-config.yaml +++ b/go/config/defaults/0-base-config.yaml @@ -8,16 +8,16 @@ network: batch: interval: 1s maxInterval: 1s # if this is greater than batch.interval then we make batches more slowly when there are no transactions - maxSize: 56320 # 55kb + maxSize: 125952 # (128-5)kb - the size of the rollup minus overhead rollup: interval: 5s maxInterval: 10m # rollups will be produced after this time even if the data blob is not full - maxSize: 131072 # 128kb + maxSize: 131072 # 128kb - the size of a blob gas: # todo: ask stefan about these fields baseFee: 1000000000 # using geth's initial base fee for EIP-1559 blocks. minGasPrice: 1000000000 # using geth's initial base fee for EIP-1559 blocks. paymentAddress: 0xd6C9230053f45F873Cb66D8A02439380a37A4fbF - batchExecutionLimit: 300000000000 # 300 gwei + batchExecutionLimit: 30000000 # same as Ethereum blocks localExecutionCap: 300000000000 # 300 gwei l1: chainId: 1337 @@ -74,6 +74,7 @@ host: enclave: enableAttestation: false + storeExecutedTransactions: true db: useInMemory: true edgelessDBHost: "" # host address for postgres db when used diff --git a/go/config/defaults/testnet-launcher/2-sequencer.yaml b/go/config/defaults/testnet-launcher/2-sequencer.yaml index d4a91134c2..8e7b06b235 100644 --- a/go/config/defaults/testnet-launcher/2-sequencer.yaml +++ b/go/config/defaults/testnet-launcher/2-sequencer.yaml @@ -9,9 +9,9 @@ host: p2p: bindAddress: 0.0.0.0:15000 enclave: - rpcAddresses: [ "sequencer-enclave:11000" ] + rpcAddresses: [ "sequencer-enclave-0:11000" ] enclave: db: - edgelessDBHost: sequencer-edgelessdb + edgelessDBHost: sequencer-edgelessdb-0 rpc: bindAddress: 0.0.0.0:11000 \ No newline at end of file diff --git a/go/config/defaults/testnet-launcher/2-validator.yaml b/go/config/defaults/testnet-launcher/2-validator.yaml index c21bcc9a89..d69474d7c2 100644 --- a/go/config/defaults/testnet-launcher/2-validator.yaml +++ b/go/config/defaults/testnet-launcher/2-validator.yaml @@ -9,12 +9,12 @@ host: p2p: bindAddress: 0.0.0.0:15010 enclave: - rpcAddresses: [ "validator-enclave:11010" ] + rpcAddresses: [ "validator-enclave-0:11010" ] rpc: httpPort: 13010 wsPort: 13011 enclave: db: - edgelessDBHost: validator-edgelessdb + edgelessDBHost: validator-edgelessdb-0 rpc: bindAddress: 0.0.0.0:11010 \ No newline at end of file diff --git a/go/config/enclave.go b/go/config/enclave.go index a177c79b01..d4cc97b857 100644 --- a/go/config/enclave.go +++ b/go/config/enclave.go @@ -7,7 +7,8 @@ import "time" // yaml: `enclave` type EnclaveConfig struct { // EnableAttestation specifies whether the enclave will produce verified attestation report. - EnableAttestation bool `mapstructure:"enableAttestation"` + EnableAttestation bool `mapstructure:"enableAttestation"` + StoreExecutedTransactions bool `mapstructure:"storeExecutedTransactions"` DB *EnclaveDB `mapstructure:"db"` Debug *EnclaveDebug `mapstructure:"debug"` diff --git a/go/enclave/components/batch_executor.go b/go/enclave/components/batch_executor.go index d21f30dcef..489dd5b82d 100644 --- a/go/enclave/components/batch_executor.go +++ b/go/enclave/components/batch_executor.go @@ -8,6 +8,8 @@ import ( "math/big" "sync" + "github.com/ten-protocol/go-ten/go/common/compression" + gethcore "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/vm" @@ -48,18 +50,19 @@ var ErrNoTransactionsToProcess = fmt.Errorf("no transactions to process") // batchExecutor - the component responsible for executing batches type batchExecutor struct { - storage storage.Storage - batchRegistry BatchRegistry - config enclaveconfig.EnclaveConfig - gethEncodingService gethencoding.EncodingService - crossChainProcessors *crosschain.Processors - genesis *genesis.Genesis - logger gethlog.Logger - gasOracle gas.Oracle - chainConfig *params.ChainConfig - systemContracts system.SystemContractCallbacks - entropyService *crypto.EvmEntropyService - mempool *TxPool + storage storage.Storage + batchRegistry BatchRegistry + config enclaveconfig.EnclaveConfig + gethEncodingService gethencoding.EncodingService + crossChainProcessors *crosschain.Processors + dataCompressionService compression.DataCompressionService + genesis *genesis.Genesis + logger gethlog.Logger + gasOracle gas.Oracle + chainConfig *params.ChainConfig + systemContracts system.SystemContractCallbacks + entropyService *crypto.EvmEntropyService + mempool *TxPool // stateDBMutex - used to protect calls to stateDB.Commit as it is not safe for async access. stateDBMutex sync.Mutex @@ -79,24 +82,26 @@ func NewBatchExecutor( systemContracts system.SystemContractCallbacks, entropyService *crypto.EvmEntropyService, mempool *TxPool, + dataCompressionService compression.DataCompressionService, logger gethlog.Logger, ) BatchExecutor { return &batchExecutor{ - storage: storage, - batchRegistry: batchRegistry, - config: config, - gethEncodingService: gethEncodingService, - crossChainProcessors: cc, - genesis: genesis, - chainConfig: chainConfig, - logger: logger, - gasOracle: gasOracle, - stateDBMutex: sync.Mutex{}, - batchGasLimit: config.GasBatchExecutionLimit, - systemContracts: systemContracts, - entropyService: entropyService, - mempool: mempool, - chainContext: evm.NewTenChainContext(storage, gethEncodingService, config, logger), + storage: storage, + batchRegistry: batchRegistry, + config: config, + gethEncodingService: gethEncodingService, + crossChainProcessors: cc, + genesis: genesis, + chainConfig: chainConfig, + logger: logger, + gasOracle: gasOracle, + stateDBMutex: sync.Mutex{}, + batchGasLimit: config.GasBatchExecutionLimit, + systemContracts: systemContracts, + entropyService: entropyService, + mempool: mempool, + dataCompressionService: dataCompressionService, + chainContext: evm.NewTenChainContext(storage, gethEncodingService, config, logger), } } @@ -286,7 +291,7 @@ func (executor *batchExecutor) execBatchTransactions(ec *BatchExecutionContext) } func (executor *batchExecutor) execMempoolTransactions(ec *BatchExecutionContext) error { - sizeLimiter := limiters.NewBatchSizeLimiter(executor.config.MaxBatchSize) + sizeLimiter := limiters.NewBatchSizeLimiter(executor.config.MaxBatchSize, executor.dataCompressionService) pendingTransactions := executor.mempool.PendingTransactions() nrPending, nrQueued := executor.mempool.Stats() @@ -319,10 +324,12 @@ func (executor *batchExecutor) execMempoolTransactions(ec *BatchExecutionContext // check the size limiter err := sizeLimiter.AcceptTransaction(tx) if err != nil { - executor.logger.Info("Unable to accept transaction", log.TxKey, tx.Hash(), log.ErrKey, err) if errors.Is(err, limiters.ErrInsufficientSpace) { // Batch ran out of space - break + executor.logger.Trace("Unable to accept transaction", log.TxKey, tx.Hash()) + mempoolTxs.Pop() + continue } + return fmt.Errorf("failed to apply the batch limiter. Cause: %w", err) } pTx, err := executor.toPricedTx(ec, tx) diff --git a/go/enclave/components/txpool.go b/go/enclave/components/txpool.go index 7aa81f6b68..214b5ff0aa 100644 --- a/go/enclave/components/txpool.go +++ b/go/enclave/components/txpool.go @@ -1,17 +1,17 @@ package components -// unsafe package imported in order to link to a private function in go-ethereum. -// This allows us to validate transactions against the tx pool rules. import ( "fmt" "math/big" + "reflect" "strings" "sync" "sync/atomic" "time" - _ "unsafe" + "unsafe" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/params" "github.com/ten-protocol/go-ten/go/common/log" gethcommon "github.com/ethereum/go-ethereum/common" @@ -24,12 +24,30 @@ import ( "github.com/ten-protocol/go-ten/go/common" ) +const ( + // txSlotSize is used to calculate how many data slots a single transaction + // takes up based on its size. The slots are used as DoS protection, ensuring + // that validating a new transaction remains a constant operation (in reality + // O(maxslots), where max slots are 4 currently). + txSlotSize = 32 * 1024 + + // we assume that at the limit, a single "uncompressable" tx is in a batch which gets rolled-up, and must fit in a 128kb blob + rollupOverhead = 5 * 1024 + + // txMaxSize is the maximum size a single transaction can have. This field has + // non-trivial consequences: larger transactions are significantly harder and + // more expensive to propagate; larger transactions also take more resources + // to validate whether they fit into the pool or not. + txMaxSize = 4*txSlotSize - rollupOverhead // 128KB - overhead +) + // this is how long the node waits to receive the second batch var startMempoolTimeout = 90 * time.Second // TxPool is an obscuro wrapper around geths transaction pool type TxPool struct { txPoolConfig legacypool.Config + chainconfig *params.ChainConfig legacyPool *legacypool.LegacyPool pool *gethtxpool.TxPool Chain *EthChainAdapter @@ -59,6 +77,7 @@ func NewTxPool(blockchain *EthChainAdapter, gasTip *big.Int, validateOnly bool, txp := &TxPool{ Chain: blockchain, + chainconfig: blockchain.Config(), txPoolConfig: txPoolConfig, legacyPool: legacyPool, gasTip: gasTip, @@ -195,6 +214,12 @@ func (t *TxPool) Close() error { // Add adds a new transactions to the pool func (t *TxPool) add(transaction *common.L2Tx) error { + // validate against the consensus rules + err := t.validateTxBasics(transaction, false) + if err != nil { + return err + } + var strErrors []string for _, err := range t.pool.Add([]*types.Transaction{transaction}, false, false) { if err != nil { @@ -208,16 +233,13 @@ func (t *TxPool) add(transaction *common.L2Tx) error { return nil } -//go:linkname validateTxBasics github.com/ethereum/go-ethereum/core/txpool/legacypool.(*LegacyPool).validateTxBasics -func validateTxBasics(_ *legacypool.LegacyPool, _ *types.Transaction, _ bool) error - //go:linkname validateTx github.com/ethereum/go-ethereum/core/txpool/legacypool.(*LegacyPool).validateTx func validateTx(_ *legacypool.LegacyPool, _ *types.Transaction, _ bool) error // Validate - run the underlying tx pool validation logic func (t *TxPool) validate(tx *common.L2Tx) error { // validate against the consensus rules - err := validateTxBasics(t.legacyPool, tx, false) + err := t.validateTxBasics(tx, false) if err != nil { return err } @@ -231,3 +253,41 @@ func (t *TxPool) validate(tx *common.L2Tx) error { func (t *TxPool) Stats() (int, int) { return t.legacyPool.Stats() } + +// validateTxBasics checks whether a transaction is valid according to the consensus +// rules, but does not check state-dependent validation such as sufficient balance. +// This check is meant as an early check which only needs to be performed once, +// and does not require the pool mutex to be held. +func (t *TxPool) validateTxBasics(tx *types.Transaction, local bool) error { + opts := &gethtxpool.ValidationOptions{ + Config: t.chainconfig, + Accept: 0 | + 1< 0 { - return nil, newRevertError(result) + if vmerr := cleanState.Error(); vmerr != nil { + return nil, vmerr } if err != nil { // also return the result as the result can be evaluated on some errors like ErrIntrinsicGas logger.Debug(fmt.Sprintf("Error applying msg %v:", msg), log.CtrErrKey, err) - return result, err + return result, fmt.Errorf("err: %w (supplied gas %d)", err, msg.GasLimit) } return result, nil @@ -344,37 +335,6 @@ func initParams(storage storage.Storage, gethEncodingService gethencoding.Encodi return NewTenChainContext(storage, gethEncodingService, config, l), vmCfg } -func newErrorWithReasonAndCode(err error) error { - result := &errutil.DataError{ - Err: err.Error(), - } - - var e gethrpc.Error - ok := errors.As(err, &e) - if ok { - result.Code = e.ErrorCode() - } - var de gethrpc.DataError - ok = errors.As(err, &de) - if ok { - result.Reason = de.ErrorData() - } - return result -} - -func newRevertError(result *gethcore.ExecutionResult) error { - reason, errUnpack := abi.UnpackRevert(result.Revert()) - err := errors.New("execution reverted") - if errUnpack == nil { - err = fmt.Errorf("execution reverted: %v", reason) - } - return &errutil.DataError{ - Err: err.Error(), - Reason: hexutil.Encode(result.Revert()), - Code: 3, // todo - magic number, really needs thought around the value and made a constant - } -} - // used as a wrapper around the vm.EVM to allow for easier calling of smart contract view functions type localContractCaller struct { evm *vm.EVM diff --git a/go/enclave/genesis/genesis_test.go b/go/enclave/genesis/genesis_test.go index 2838647ada..196c7a54b2 100644 --- a/go/enclave/genesis/genesis_test.go +++ b/go/enclave/genesis/genesis_test.go @@ -42,7 +42,7 @@ func TestDefaultGenesis(t *testing.T) { if err != nil { t.Fatalf("unable to create temp db: %s", err) } - storageDB := storage.NewStorage(backingDB, storage.NewCacheService(gethlog.New(), true), nil, gethlog.New()) + storageDB := storage.NewStorage(backingDB, storage.NewCacheService(gethlog.New(), true), nil, nil, gethlog.New()) stateDB, err := gen.applyAllocations(storageDB) if err != nil { t.Fatalf("unable to apply genesis allocations") @@ -85,7 +85,7 @@ func TestCustomGenesis(t *testing.T) { if err != nil { t.Fatalf("unable to create temp db: %s", err) } - storageDB := storage.NewStorage(backingDB, storage.NewCacheService(gethlog.New(), true), nil, gethlog.New()) + storageDB := storage.NewStorage(backingDB, storage.NewCacheService(gethlog.New(), true), nil, nil, gethlog.New()) stateDB, err := gen.applyAllocations(storageDB) if err != nil { t.Fatalf("unable to apply genesis allocations") diff --git a/go/enclave/l2chain/interfaces.go b/go/enclave/l2chain/interfaces.go index ce884bc5cb..c83ce2f29d 100644 --- a/go/enclave/l2chain/interfaces.go +++ b/go/enclave/l2chain/interfaces.go @@ -20,7 +20,7 @@ type ObscuroChain interface { GetBalanceAtBlock(ctx context.Context, accountAddr gethcommon.Address, blockNumber *gethrpc.BlockNumber) (*hexutil.Big, error) // ObsCall - The interface for executing eth_call RPC commands against obscuro. - ObsCall(ctx context.Context, apiArgs *gethapi.TransactionArgs, blockNumber *gethrpc.BlockNumber) (*gethcore.ExecutionResult, error) + Call(ctx context.Context, apiArgs *gethapi.TransactionArgs, blockNumber *gethrpc.BlockNumber) (*gethcore.ExecutionResult, error) // ObsCallAtBlock - Execute eth_call RPC against obscuro for a specific block (batch) number. ObsCallAtBlock(ctx context.Context, apiArgs *gethapi.TransactionArgs, blockNumber *gethrpc.BlockNumber) (*gethcore.ExecutionResult, error) diff --git a/go/enclave/l2chain/l2_chain.go b/go/enclave/l2chain/l2_chain.go index efb704e4e9..3a47dcd85b 100644 --- a/go/enclave/l2chain/l2_chain.go +++ b/go/enclave/l2chain/l2_chain.go @@ -28,7 +28,7 @@ import ( gethrpc "github.com/ten-protocol/go-ten/lib/gethfork/rpc" ) -type obscuroChain struct { +type tenChain struct { chainConfig *params.ChainConfig config enclaveconfig.EnclaveConfig storage storage.Storage @@ -51,7 +51,7 @@ func NewChain( registry components.BatchRegistry, gasEstimationCap uint64, ) ObscuroChain { - return &obscuroChain{ + return &tenChain{ storage: storage, config: config, gethEncodingService: gethEncodingService, @@ -63,7 +63,7 @@ func NewChain( } } -func (oc *obscuroChain) GetBalanceAtBlock(ctx context.Context, accountAddr gethcommon.Address, blockNumber *gethrpc.BlockNumber) (*hexutil.Big, error) { +func (oc *tenChain) GetBalanceAtBlock(ctx context.Context, accountAddr gethcommon.Address, blockNumber *gethrpc.BlockNumber) (*hexutil.Big, error) { chainState, err := oc.Registry.GetBatchStateAtHeight(ctx, blockNumber) if err != nil { return nil, fmt.Errorf("unable to get blockchain state - %w", err) @@ -72,26 +72,20 @@ func (oc *obscuroChain) GetBalanceAtBlock(ctx context.Context, accountAddr gethc return (*hexutil.Big)(chainState.GetBalance(accountAddr).ToBig()), nil } -func (oc *obscuroChain) ObsCall(ctx context.Context, apiArgs *gethapi.TransactionArgs, blockNumber *gethrpc.BlockNumber) (*gethcore.ExecutionResult, error) { +func (oc *tenChain) Call(ctx context.Context, apiArgs *gethapi.TransactionArgs, blockNumber *gethrpc.BlockNumber) (*gethcore.ExecutionResult, error) { result, err := oc.ObsCallAtBlock(ctx, apiArgs, blockNumber) if err != nil { oc.logger.Debug(fmt.Sprintf("Obs_Call: failed to execute contract %s.", apiArgs.To), log.CtrErrKey, err.Error()) return nil, err } - // the execution might have succeeded (err == nil) but the evm contract logic might have failed (result.Failed() == true) - if result.Failed() { - oc.logger.Debug(fmt.Sprintf("Obs_Call: Failed to execute contract %s.", apiArgs.To), log.CtrErrKey, result.Err) - return nil, result.Err - } - if oc.logger.Enabled(context.Background(), gethlog.LevelTrace) { oc.logger.Trace("Obs_Call successful", "result", hexutils.BytesToHex(result.ReturnData)) } return result, nil } -func (oc *obscuroChain) ObsCallAtBlock(ctx context.Context, apiArgs *gethapi.TransactionArgs, blockNumber *gethrpc.BlockNumber) (*gethcore.ExecutionResult, error) { +func (oc *tenChain) ObsCallAtBlock(ctx context.Context, apiArgs *gethapi.TransactionArgs, blockNumber *gethrpc.BlockNumber) (*gethcore.ExecutionResult, error) { // fetch the chain state at given batch blockState, err := oc.Registry.GetBatchStateAtHeight(ctx, blockNumber) if err != nil { @@ -117,18 +111,12 @@ func (oc *obscuroChain) ObsCallAtBlock(ctx context.Context, apiArgs *gethapi.Tra batch.Header.Root.Hex())) } - result, err := evm.ExecuteObsCall(ctx, callMsg, blockState, batch.Header, oc.storage, oc.gethEncodingService, oc.chainConfig, oc.gasEstimationCap, oc.config, oc.logger) - if err != nil { - // also return the result as the result can be evaluated on some errors like ErrIntrinsicGas - return result, err - } - - return result, nil + return evm.ExecuteCall(ctx, callMsg, blockState, batch.Header, oc.storage, oc.gethEncodingService, oc.chainConfig, oc.gasEstimationCap, oc.config, oc.logger) } // GetChainStateAtTransaction Returns the state of the chain at certain block height after executing transactions up to the selected transaction // TODO make this cacheable - why isn't this in the evm_facade? -func (oc *obscuroChain) GetChainStateAtTransaction(ctx context.Context, batch *core.Batch, txIndex int, _ uint64) (*gethcore.Message, vm.BlockContext, *state.StateDB, error) { +func (oc *tenChain) GetChainStateAtTransaction(ctx context.Context, batch *core.Batch, txIndex int, _ uint64) (*gethcore.Message, vm.BlockContext, *state.StateDB, error) { // Short circuit if it's genesis batch. if batch.NumberU64() == 0 { return nil, vm.BlockContext{}, nil, errors.New("no transaction in genesis") diff --git a/go/enclave/limiters/batchlimiter.go b/go/enclave/limiters/batchlimiter.go index 02b5414063..7bc84f823b 100644 --- a/go/enclave/limiters/batchlimiter.go +++ b/go/enclave/limiters/batchlimiter.go @@ -3,6 +3,7 @@ package limiters import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" + "github.com/ten-protocol/go-ten/go/common/compression" ) // BatchSizeLimiter - Acts as a limiter for batches based @@ -10,20 +11,22 @@ import ( // Acts as a calldata reservation system that accounts for both // transactions and cross chain messages. type batchSizeLimiter struct { - remainingSize uint64 // the available size in the limiter + compressionService compression.DataCompressionService + remainingSize uint64 // the available size in the limiter } // NewBatchSizeLimiter - Size is the total space available per batch for calldata in a rollup. -func NewBatchSizeLimiter(size uint64) BatchSizeLimiter { +func NewBatchSizeLimiter(size uint64, compressionService compression.DataCompressionService) BatchSizeLimiter { return &batchSizeLimiter{ - remainingSize: size, + compressionService: compressionService, + remainingSize: size, } } // AcceptTransaction - transaction is rlp encoded as it normally would be when publishing a rollup and // its size is deducted from the remaining limit. func (l *batchSizeLimiter) AcceptTransaction(tx *types.Transaction) error { - rlpSize, err := getRlpSize(tx) + rlpSize, err := l.getCompressedSize(tx) if err != nil { return err } @@ -37,14 +40,20 @@ func (l *batchSizeLimiter) AcceptTransaction(tx *types.Transaction) error { } // todo (@stefan) figure out how to optimize the serialization out of the limiter -func getRlpSize(val interface{}) (int, error) { - // todo (@stefan) - this should have a coefficient for compression +func (l *batchSizeLimiter) getCompressedSize(val interface{}) (int, error) { enc, err := rlp.EncodeToBytes(val) if err != nil { return 0, err } - return len(enc), nil + // compress the transaction. This is useless for small transactions, but might be useful for larger transactions such as deploying contracts + // todo - keep a running compression of the current batch + compr, err := l.compressionService.CompressBatch(enc) + if err != nil { + return 0, err + } + + return len(compr), nil } type unlimitedBatchSize struct{} diff --git a/go/enclave/main/enclave.json b/go/enclave/main/enclave.json index 789803b2a5..dfe3460e7c 100644 --- a/go/enclave/main/enclave.json +++ b/go/enclave/main/enclave.json @@ -25,6 +25,7 @@ { "fromHost": true, "name": "ENCLAVE_DEBUG_ENABLEDEBUGNAMESPACE" }, { "fromHost": true, "name": "ENCLAVE_DEBUG_ENABLEPROFILER" }, { "fromHost": true, "name": "ENCLAVE_ENABLEATTESTATION" }, + { "fromHost": true, "name": "ENCLAVE_STOREEXECUTEDTRANSACTIONS" }, { "fromHost": true, "name": "ENCLAVE_L1_ENABLEBLOCKVALIDATION" }, { "fromHost": true, "name": "ENCLAVE_L1_GENESISJSON" }, { "fromHost": true, "name": "ENCLAVE_LOG_LEVEL" }, diff --git a/go/enclave/rpc/EstimateGas.go b/go/enclave/rpc/EstimateGas.go index 1af819a45b..bf5c37dfd1 100644 --- a/go/enclave/rpc/EstimateGas.go +++ b/go/enclave/rpc/EstimateGas.go @@ -6,9 +6,10 @@ import ( "fmt" "math/big" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ten-protocol/go-ten/go/common" + + "github.com/ethereum/go-ethereum/params" + "github.com/ten-protocol/go-ten/go/common/measure" "github.com/ten-protocol/go-ten/go/enclave/core" @@ -17,7 +18,6 @@ import ( gethcore "github.com/ethereum/go-ethereum/core" "github.com/ten-protocol/go-ten/go/common/gethapi" "github.com/ten-protocol/go-ten/go/common/gethencoding" - "github.com/ten-protocol/go-ten/go/common/syserr" gethrpc "github.com/ten-protocol/go-ten/lib/gethfork/rpc" ) @@ -99,12 +99,12 @@ func EstimateGasExecute(builder *CallBuilder[CallParamsWithBlock, hexutil.Uint64 // Notice that unfortunately, some slots might ve considered warm, which skews the estimation. // The single pass will run once at the highest gas cap and return gas used. Not completely reliable, // but is quick. - executionGasEstimate, gasPrice, err := rpc.estimateGasSinglePass(builder.ctx, txArgs, blockNumber, rpc.config.GasLocalExecutionCapFlag) + executionGasEstimate, revert, gasPrice, err := estimateGasSinglePass(builder.ctx, rpc, txArgs, blockNumber, rpc.config.GasLocalExecutionCapFlag) if err != nil { - err = fmt.Errorf("unable to estimate transaction - %w", err) - - if errors.Is(err, syserr.InternalError{}) { - return err + if len(revert) > 0 { + builder.Err = newRevertError(revert) + rpc.logger.Debug("revert error", "err", builder.Err) + return nil } // return EVM error @@ -127,7 +127,7 @@ func EstimateGasExecute(builder *CallBuilder[CallParamsWithBlock, hexutil.Uint64 return nil } -func (rpc *EncryptionManager) calculateMaxGasCap(ctx context.Context, gasCap uint64, argsGas *hexutil.Uint64) uint64 { +func calculateMaxGasCap(ctx context.Context, rpc *EncryptionManager, gasCap uint64, argsGas *hexutil.Uint64) uint64 { // Fetch the current batch header to get the batch gas limit batchHeader, err := rpc.storage.FetchHeadBatchHeader(ctx) if err != nil { @@ -144,7 +144,7 @@ func (rpc *EncryptionManager) calculateMaxGasCap(ctx context.Context, gasCap uin // If args.Gas is specified, take the minimum of gasCap and args.Gas if argsGas != nil { argsGasUint64 := uint64(*argsGas) - if argsGasUint64 < gasCap { + if argsGasUint64 < gasCap && argsGasUint64 >= params.TxGas { rpc.logger.Debug("Gas cap adjusted based on args.Gas", "argsGas", argsGasUint64, "previousGasCap", gasCap, @@ -189,39 +189,34 @@ func calculateProxyOverhead(txArgs *gethapi.TransactionArgs) uint64 { // The modifications are an overhead buffer and a 20% increase to account for warm storage slots. This is because the stateDB // for the head batch might not be fully clean in terms of the running call. Cold storage slots cost far more than warm ones to // read and write. -func (rpc *EncryptionManager) estimateGasSinglePass(ctx context.Context, args *gethapi.TransactionArgs, blkNumber *gethrpc.BlockNumber, gasCap uint64) (hexutil.Uint64, *big.Int, common.SystemError) { - maxGasCap := rpc.calculateMaxGasCap(ctx, gasCap, args.Gas) +func estimateGasSinglePass(ctx context.Context, rpc *EncryptionManager, args *gethapi.TransactionArgs, blkNumber *gethrpc.BlockNumber, globalGasCap uint64) (hexutil.Uint64, []byte, *big.Int, error) { + maxGasCap := calculateMaxGasCap(ctx, rpc, globalGasCap, args.Gas) // allowance will either be the maxGasCap or the balance allowance. // If the users funds are floaty, this might cause issues combined with the l1 pricing. - allowance, feeCap, err := rpc.normalizeFeeCapAndAdjustGasLimit(ctx, args, blkNumber, maxGasCap) + allowance, feeCap, err := normalizeFeeCapAndAdjustGasLimit(ctx, rpc, args, blkNumber, maxGasCap) if err != nil { - return 0, nil, err + return 0, nil, nil, err } - // Set the gas limit to the provided gasCap - args.Gas = (*hexutil.Uint64)(&allowance) - // Perform a single gas estimation pass using isGasEnough - failed, result, err := rpc.isGasEnough(ctx, args, allowance, blkNumber) + failed, result, err := isGasEnough(ctx, rpc, args, allowance, blkNumber) if err != nil { // Return zero values and the encountered error if estimation fails - return 0, nil, err + return 0, nil, nil, err } if failed { - if result != nil && result.Err != vm.ErrOutOfGas { //nolint: errorlint - if len(result.Revert()) > 0 { - return 0, gethcommon.Big0, newRevertError(result) - } - return 0, gethcommon.Big0, result.Err + if result != nil && !errors.Is(result.Err, vm.ErrOutOfGas) { + rpc.logger.Debug("Failed gas estimation", "error", result.Err) + return 0, result.Revert(), nil, result.Err } // If the gas cap is insufficient, return an appropriate error - return 0, nil, fmt.Errorf("gas required exceeds the provided gas cap (%d)", gasCap) + return 0, nil, nil, fmt.Errorf("gas required exceeds allowance (%d)", globalGasCap) } if result == nil { // If there's no result, something went wrong - return 0, nil, fmt.Errorf("no execution result returned") + return 0, nil, nil, fmt.Errorf("no execution result returned") } // Extract the gas used from the execution result. @@ -235,10 +230,10 @@ func (rpc *EncryptionManager) estimateGasSinglePass(ctx context.Context, args *g gasUsedBig.Div(gasUsedBig, big.NewInt(100)) gasUsed := hexutil.Uint64(gasUsedBig.Uint64()) - return gasUsed, feeCap, nil + return gasUsed, nil, feeCap, nil } -func (rpc *EncryptionManager) normalizeFeeCapAndAdjustGasLimit(ctx context.Context, args *gethapi.TransactionArgs, blkNumber *gethrpc.BlockNumber, hi uint64) (uint64, *big.Int, error) { +func normalizeFeeCapAndAdjustGasLimit(ctx context.Context, rpc *EncryptionManager, args *gethapi.TransactionArgs, blkNumber *gethrpc.BlockNumber, hi uint64) (uint64, *big.Int, error) { // Normalize the max fee per gas the call is willing to spend. var feeCap *big.Int if args.GasPrice != nil && (args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil) { @@ -289,45 +284,13 @@ func (rpc *EncryptionManager) normalizeFeeCapAndAdjustGasLimit(ctx context.Conte // Create a helper to check if a gas allowance results in an executable transaction // isGasEnough returns whether the gaslimit should be raised, lowered, or if it was impossible to execute the message -func (rpc *EncryptionManager) isGasEnough(ctx context.Context, args *gethapi.TransactionArgs, gas uint64, blkNumber *gethrpc.BlockNumber) (bool, *gethcore.ExecutionResult, error) { +func isGasEnough(ctx context.Context, rpc *EncryptionManager, args *gethapi.TransactionArgs, gas uint64, blkNumber *gethrpc.BlockNumber) (bool, *gethcore.ExecutionResult, error) { defer core.LogMethodDuration(rpc.logger, measure.NewStopwatch(), "enclave.go:IsGasEnough") args.Gas = (*hexutil.Uint64)(&gas) result, err := rpc.chain.ObsCallAtBlock(ctx, args, blkNumber) if err != nil { - if errors.Is(err, gethcore.ErrIntrinsicGas) { - return true, nil, nil // Special case, raise gas limit - } + // since we estimate gas in a single pass, any error is just returned return true, nil, err // Bail out } return result.Failed(), result, nil } - -func newRevertError(result *gethcore.ExecutionResult) *revertError { - reason, errUnpack := abi.UnpackRevert(result.Revert()) - err := errors.New("execution reverted") - if errUnpack == nil { - err = fmt.Errorf("execution reverted: %v", reason) - } - return &revertError{ - error: err, - reason: hexutil.Encode(result.Revert()), - } -} - -// revertError is an API error that encompasses an EVM revertal with JSON error -// code and a binary data blob. -type revertError struct { - error - reason string // revert reason hex encoded -} - -// ErrorCode returns the JSON error code for a revertal. -// See: https://github.com/ethereum/wiki/wiki/JSON-RPC-Error-Codes-Improvement-Proposal -func (e *revertError) ErrorCode() int { - return 3 -} - -// ErrorData returns the hex encoded revert reason. -func (e *revertError) ErrorData() interface{} { - return e.reason -} diff --git a/go/enclave/rpc/GetLogs.go b/go/enclave/rpc/GetLogs.go index 994c32999a..6476ac4ee7 100644 --- a/go/enclave/rpc/GetLogs.go +++ b/go/enclave/rpc/GetLogs.go @@ -14,7 +14,10 @@ import ( "github.com/ten-protocol/go-ten/go/common/syserr" ) -func GetLogsValidate(reqParams []any, builder *CallBuilder[filters.FilterCriteria, []*types.Log], _ *EncryptionManager) error { +func GetLogsValidate(reqParams []any, builder *CallBuilder[filters.FilterCriteria, []*types.Log], rpc *EncryptionManager) error { + if !storeTxEnabled(rpc, builder) { + return nil + } // Parameters are [Filter] if len(reqParams) != 1 { builder.Err = fmt.Errorf("unexpected number of parameters") diff --git a/go/enclave/rpc/GetPersonalTransactions.go b/go/enclave/rpc/GetPersonalTransactions.go index f712dc256d..e3b1d06cbd 100644 --- a/go/enclave/rpc/GetPersonalTransactions.go +++ b/go/enclave/rpc/GetPersonalTransactions.go @@ -9,7 +9,11 @@ import ( "github.com/ten-protocol/go-ten/go/common/gethencoding" ) -func GetPersonalTransactionsValidate(reqParams []any, builder *CallBuilder[common.ListPrivateTransactionsQueryParams, common.PrivateTransactionsQueryResponse], _ *EncryptionManager) error { +func GetPersonalTransactionsValidate(reqParams []any, builder *CallBuilder[common.ListPrivateTransactionsQueryParams, common.PrivateTransactionsQueryResponse], rpc *EncryptionManager) error { + if !storeTxEnabled(rpc, builder) { + return nil + } + // Parameters are [PrivateTransactionListParams] if len(reqParams) != 1 { builder.Err = fmt.Errorf("unexpected number of parameters (expected %d, got %d)", 1, len(reqParams)) diff --git a/go/enclave/rpc/GetTransaction.go b/go/enclave/rpc/GetTransaction.go index 9cbb474a10..42861572c7 100644 --- a/go/enclave/rpc/GetTransaction.go +++ b/go/enclave/rpc/GetTransaction.go @@ -14,7 +14,10 @@ import ( "github.com/ten-protocol/go-ten/go/common/errutil" ) -func GetTransactionValidate(reqParams []any, builder *CallBuilder[gethcommon.Hash, RpcTransaction], _ *EncryptionManager) error { +func GetTransactionValidate(reqParams []any, builder *CallBuilder[gethcommon.Hash, RpcTransaction], rpc *EncryptionManager) error { + if !storeTxEnabled(rpc, builder) { + return nil + } // Parameters are [Hash] if len(reqParams) != 1 { builder.Err = fmt.Errorf("wrong parameters") diff --git a/go/enclave/rpc/GetTransactionReceipt.go b/go/enclave/rpc/GetTransactionReceipt.go index 5e01839e5b..783a086c3d 100644 --- a/go/enclave/rpc/GetTransactionReceipt.go +++ b/go/enclave/rpc/GetTransactionReceipt.go @@ -16,7 +16,10 @@ import ( "github.com/ten-protocol/go-ten/go/common/log" ) -func GetTransactionReceiptValidate(reqParams []any, builder *CallBuilder[gethcommon.Hash, map[string]interface{}], _ *EncryptionManager) error { +func GetTransactionReceiptValidate(reqParams []any, builder *CallBuilder[gethcommon.Hash, map[string]interface{}], rpc *EncryptionManager) error { + if !storeTxEnabled(rpc, builder) { + return nil + } // Parameters are [Hash] if len(reqParams) < 1 { builder.Err = fmt.Errorf("unexpected number of parameters") diff --git a/go/enclave/rpc/TenEthCall.go b/go/enclave/rpc/TenEthCall.go index cacaa518de..f9dcdf9c7d 100644 --- a/go/enclave/rpc/TenEthCall.go +++ b/go/enclave/rpc/TenEthCall.go @@ -1,13 +1,11 @@ package rpc import ( - "errors" "fmt" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ten-protocol/go-ten/go/common/gethencoding" "github.com/ten-protocol/go-ten/go/common/log" - "github.com/ten-protocol/go-ten/go/common/syserr" ) func TenCallValidate(reqParams []any, builder *CallBuilder[CallParamsWithBlock, string], _ *EncryptionManager) error { @@ -49,19 +47,19 @@ func TenCallExecute(builder *CallBuilder[CallParamsWithBlock, string], rpc *Encr apiArgs := builder.Param.callParams blkNumber := builder.Param.block - execResult, err := rpc.chain.ObsCall(builder.ctx, apiArgs, blkNumber) + execResult, err := rpc.chain.Call(builder.ctx, apiArgs, blkNumber) if err != nil { rpc.logger.Debug("Failed eth_call.", log.ErrKey, err) - - // return system errors to the host - if errors.Is(err, syserr.InternalError{}) { - return err - } - - builder.Err = err + return err + } + // If the result contains a revert reason, try to unpack and return it. + if len(execResult.Revert()) > 0 { + builder.Err = newRevertError(execResult.Revert()) return nil } + builder.Err = execResult.Err + var encodedResult string if len(execResult.ReturnData) != 0 { encodedResult = hexutil.Encode(execResult.ReturnData) diff --git a/go/enclave/rpc/rpc_utils.go b/go/enclave/rpc/rpc_utils.go index 8dc732b56b..e6d12771f2 100644 --- a/go/enclave/rpc/rpc_utils.go +++ b/go/enclave/rpc/rpc_utils.go @@ -3,6 +3,12 @@ package rpc import ( "fmt" + "github.com/ten-protocol/go-ten/go/common/errutil" + + "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ten-protocol/go-ten/go/common/gethapi" gethrpc "github.com/ten-protocol/go-ten/lib/gethfork/rpc" @@ -28,3 +34,26 @@ type CallParamsWithBlock struct { callParams *gethapi.TransactionArgs block *gethrpc.BlockNumber } + +func storeTxEnabled[P any, R any](rpc *EncryptionManager, builder *CallBuilder[P, R]) bool { + if !rpc.config.StoreExecutedTransactions { + builder.Err = fmt.Errorf("the current TEN enclave is not configured to respond to this query") + return false + } + return true +} + +// newRevertError creates a revertError instance with the provided revert data. +func newRevertError(revert []byte) *errutil.DataError { + err := vm.ErrExecutionReverted + + reason, errUnpack := abi.UnpackRevert(revert) + if errUnpack == nil { + err = fmt.Errorf("%w: %v", vm.ErrExecutionReverted, reason) + } + return &errutil.DataError{ + Err: err.Error(), + Code: 3, // See: https://github.com/ethereum/wiki/wiki/JSON-RPC-Error-Codes-Improvement-Proposal + Reason: hexutil.Encode(revert), + } +} diff --git a/go/enclave/storage/interfaces.go b/go/enclave/storage/interfaces.go index dea8d84815..9faf484b19 100644 --- a/go/enclave/storage/interfaces.go +++ b/go/enclave/storage/interfaces.go @@ -121,6 +121,11 @@ type EnclaveKeyStorage interface { GetEnclaveKey(ctx context.Context) ([]byte, error) } +type SystemContractAddressesStorage interface { + StoreSystemContractAddresses(ctx context.Context, addresses common.SystemContractAddresses) error + GetSystemContractAddresses(ctx context.Context) (common.SystemContractAddresses, error) +} + // Storage is the enclave's interface for interacting with the enclave's datastore type Storage interface { BlockResolver @@ -132,6 +137,7 @@ type Storage interface { CrossChainMessagesStorage EnclaveKeyStorage ScanStorage + SystemContractAddressesStorage io.Closer // HealthCheck returns whether the storage is deemed healthy or not diff --git a/go/enclave/storage/storage.go b/go/enclave/storage/storage.go index 5849eb599b..a67fd93cda 100644 --- a/go/enclave/storage/storage.go +++ b/go/enclave/storage/storage.go @@ -5,6 +5,7 @@ import ( "context" "crypto/ecdsa" "database/sql" + "encoding/json" "errors" "fmt" "math/big" @@ -41,8 +42,9 @@ import ( // these are the keys from the config table const ( // todo - this will require a dedicated table when upgrades are implemented - masterSeedCfg = "MASTER_SEED" - enclaveKeyCfg = "ENCLAVE_KEY" + masterSeedCfg = "MASTER_SEED" + enclaveKeyCfg = "ENCLAVE_KEY" + systemContractAddressesCfg = "SYSTEM_CONTRACT_ADDRESSES" ) type AttestedEnclave struct { @@ -58,6 +60,7 @@ type storageImpl struct { stateCache state.Database chainConfig *params.ChainConfig + config *enclaveconfig.EnclaveConfig logger gethlog.Logger } @@ -66,7 +69,7 @@ func NewStorageFromConfig(config *enclaveconfig.EnclaveConfig, cachingService *C if err != nil { logger.Crit("Failed to connect to backing database", log.ErrKey, err) } - return NewStorage(backingDB, cachingService, chainConfig, logger) + return NewStorage(backingDB, cachingService, config, chainConfig, logger) } var defaultCacheConfig = &gethcore.CacheConfig{ @@ -86,7 +89,7 @@ var trieDBConfig = &triedb.Config{ }, } -func NewStorage(backingDB enclavedb.EnclaveDB, cachingService *CacheService, chainConfig *params.ChainConfig, logger gethlog.Logger) Storage { +func NewStorage(backingDB enclavedb.EnclaveDB, cachingService *CacheService, config *enclaveconfig.EnclaveConfig, chainConfig *params.ChainConfig, logger gethlog.Logger) Storage { // Open trie database with provided config triedb := triedb.NewDatabase(backingDB, trieDBConfig) @@ -96,6 +99,7 @@ func NewStorage(backingDB enclavedb.EnclaveDB, cachingService *CacheService, cha db: backingDB, stateCache: stateDB, chainConfig: chainConfig, + config: config, cachingService: cachingService, eventsStorage: newEventsStorage(cachingService, backingDB, logger), logger: logger, @@ -668,12 +672,15 @@ func (s *storageImpl) StoreExecutedBatch(ctx context.Context, batch *core.Batch, return fmt.Errorf("could not write synthetic txs. Cause: %w", err) } - for _, txExecResult := range results { - err = s.eventsStorage.storeReceiptAndEventLogs(ctx, dbTx, batch.Header, txExecResult) - if err != nil { - return fmt.Errorf("could not store receipt. Cause: %w", err) + if s.config.StoreExecutedTransactions { + for _, txExecResult := range results { + err = s.eventsStorage.storeReceiptAndEventLogs(ctx, dbTx, batch.Header, txExecResult) + if err != nil { + return fmt.Errorf("could not store receipt. Cause: %w", err) + } } } + if err = dbTx.Commit(); err != nil { return fmt.Errorf("could not commit batch %w", err) } @@ -872,3 +879,40 @@ func (s *storageImpl) ReadEventType(ctx context.Context, contractAddress gethcom func (s *storageImpl) logDuration(method string, stopWatch *measure.Stopwatch) { core.LogMethodDuration(s.logger, stopWatch, fmt.Sprintf("Storage::%s completed", method)) } + +func (s *storageImpl) StoreSystemContractAddresses(ctx context.Context, addresses common.SystemContractAddresses) error { + defer s.logDuration("StoreSystemContractAddresses", measure.NewStopwatch()) + + dbTx, err := s.db.NewDBTransaction(ctx) + if err != nil { + return fmt.Errorf("could not create DB transaction - %w", err) + } + defer dbTx.Rollback() + + addressesBytes, err := json.Marshal(addresses) + if err != nil { + return fmt.Errorf("could not marshal system contract addresses - %w", err) + } + _, err = enclavedb.WriteConfig(ctx, dbTx, systemContractAddressesCfg, addressesBytes) + if err != nil { + return fmt.Errorf("could not write system contract addresses - %w", err) + } + + if err := dbTx.Commit(); err != nil { + return fmt.Errorf("could not commit system contract addresses - %w", err) + } + return nil +} + +func (s *storageImpl) GetSystemContractAddresses(ctx context.Context) (common.SystemContractAddresses, error) { + defer s.logDuration("GetSystemContractAddresses", measure.NewStopwatch()) + addressesBytes, err := enclavedb.FetchConfig(ctx, s.db.GetSQLDB(), systemContractAddressesCfg) + if err != nil { + return nil, fmt.Errorf("could not fetch system contract addresses - %w", err) + } + var addresses common.SystemContractAddresses + if err := json.Unmarshal(addressesBytes, &addresses); err != nil { + return nil, fmt.Errorf("could not unmarshal system contract addresses - %w", err) + } + return addresses, nil +} diff --git a/go/enclave/system/contracts.go b/go/enclave/system/contracts.go index a7fb17d1eb..3eed2ca747 100644 --- a/go/enclave/system/contracts.go +++ b/go/enclave/system/contracts.go @@ -15,7 +15,7 @@ func GenerateDeploymentTransaction(initCode []byte, logger gethlog.Logger) (*com tx := &types.LegacyTx{ Nonce: 0, // The first transaction of the owner identity should always be deploying the contract Value: gethcommon.Big0, - Gas: 500_000_000, // It's quite the expensive contract. + Gas: 10_000_000, // It's quite the expensive contract. GasPrice: gethcommon.Big0, // Synthetic transactions are on the house. Or the house. Data: initCode, // gethcommon.FromHex(SystemDeployer.SystemDeployerMetaData.Bin), To: nil, // Geth requires nil instead of gethcommon.Address{} which equates to zero address in order to return receipt. @@ -45,7 +45,7 @@ func VerifyLogs(receipt *types.Receipt) error { return nil } -func DeriveAddresses(receipt *types.Receipt) (SystemContractAddresses, error) { +func DeriveAddresses(receipt *types.Receipt) (common.SystemContractAddresses, error) { if receipt.Status != types.ReceiptStatusSuccessful { return nil, fmt.Errorf("cannot derive system contract addresses from failed receipt") } @@ -76,13 +76,3 @@ func DeriveAddresses(receipt *types.Receipt) (SystemContractAddresses, error) { return addresses, nil } - -type SystemContractAddresses map[string]*gethcommon.Address - -func (s *SystemContractAddresses) ToString() string { - var str string - for name, addr := range *s { - str += fmt.Sprintf("%s: %s; ", name, addr.Hex()) - } - return str -} diff --git a/go/enclave/system/hooks.go b/go/enclave/system/hooks.go index 17f7b656c4..1c3922f57f 100644 --- a/go/enclave/system/hooks.go +++ b/go/enclave/system/hooks.go @@ -35,7 +35,7 @@ type SystemContractCallbacks interface { PublicSystemContracts() map[string]*gethcommon.Address // Initialization Initialize(batch *core.Batch, receipts types.Receipt, msgBusManager SystemContractsInitializable) error - Load() error + Load(msgBusManager SystemContractsInitializable) error // Usage CreateOnBatchEndTransaction(ctx context.Context, stateDB *state.StateDB, results core.TxExecResults) (*types.Transaction, error) @@ -46,13 +46,13 @@ type SystemContractCallbacks interface { } type SystemContractsInitializable interface { - Initialize(SystemContractAddresses) error + Initialize(common.SystemContractAddresses) error } type systemContractCallbacks struct { transactionsPostProcessorAddress *gethcommon.Address storage storage.Storage - systemAddresses SystemContractAddresses + systemAddresses common.SystemContractAddresses systemContractsUpgrader *gethcommon.Address logger gethlog.Logger @@ -63,7 +63,7 @@ func NewSystemContractCallbacks(storage storage.Storage, upgrader *gethcommon.Ad transactionsPostProcessorAddress: nil, logger: logger, storage: storage, - systemAddresses: make(SystemContractAddresses), + systemAddresses: make(common.SystemContractAddresses), systemContractsUpgrader: upgrader, } } @@ -84,7 +84,7 @@ func (s *systemContractCallbacks) PublicSystemContracts() map[string]*gethcommon return s.systemAddresses } -func (s *systemContractCallbacks) Load() error { +func (s *systemContractCallbacks) Load(msgBusManager SystemContractsInitializable) error { s.logger.Info("Load: Initializing system contracts") if s.storage == nil { @@ -92,36 +92,17 @@ func (s *systemContractCallbacks) Load() error { return fmt.Errorf("storage is not set") } - batchSeqNo := uint64(2) - s.logger.Debug("Load: Fetching batch", "batchSeqNo", batchSeqNo) - batch, err := s.storage.FetchBatchBySeqNo(context.Background(), batchSeqNo) + addresses, err := s.storage.GetSystemContractAddresses(context.Background()) if err != nil { - s.logger.Error("Load: Failed fetching batch", "batchSeqNo", batchSeqNo, "error", err) - return fmt.Errorf("failed fetching batch %w", err) + s.logger.Error("Load: Failed fetching system contract addresses", "error", err) + return fmt.Errorf("failed fetching system contract addresses %w", err) } + s.logger.Info("Load: Fetched system contract addresses", "addresses", addresses) - tx, err := SystemDeployerInitTransaction(s.logger, *s.systemContractsUpgrader) - if err != nil { - s.logger.Error("Load: Failed creating system deployer init transaction", "error", err) - return fmt.Errorf("failed creating system deployer init transaction %w", err) - } - - receipt, err := s.storage.GetFilteredInternalReceipt(context.Background(), tx.Hash(), nil, true) - if err != nil { - s.logger.Error("Load: Failed fetching receipt", "transactionHash", batch.Transactions[0].Hash().Hex(), "error", err) - return fmt.Errorf("failed fetching receipt %w", err) - } - - addresses, err := DeriveAddresses(receipt.ToReceipt()) - if err != nil { - s.logger.Error("Load: Failed deriving addresses", "error", err, "receiptHash", receipt.TxHash.Hex()) - return fmt.Errorf("failed deriving addresses %w", err) - } - - return s.initializeRequiredAddresses(addresses) + return s.initializeRequiredAddresses(addresses, msgBusManager) } -func (s *systemContractCallbacks) initializeRequiredAddresses(addresses SystemContractAddresses) error { +func (s *systemContractCallbacks) initializeRequiredAddresses(addresses common.SystemContractAddresses, msgBusManager SystemContractsInitializable) error { if addresses["TransactionsPostProcessor"] == nil { return fmt.Errorf("required contract address TransactionsPostProcessor is nil") } @@ -129,30 +110,42 @@ func (s *systemContractCallbacks) initializeRequiredAddresses(addresses SystemCo s.transactionsPostProcessorAddress = addresses["TransactionsPostProcessor"] s.systemAddresses = addresses + if err := msgBusManager.Initialize(addresses); err != nil { + s.logger.Error("Initialize: Failed deriving message bus address", "error", err) + return fmt.Errorf("failed deriving message bus address %w", err) + } + return nil } +func (s *systemContractCallbacks) StoreSystemContractAddresses(addresses common.SystemContractAddresses) error { + return s.storage.StoreSystemContractAddresses(context.Background(), addresses) +} + func (s *systemContractCallbacks) Initialize(batch *core.Batch, receipt types.Receipt, msgBusManager SystemContractsInitializable) error { s.logger.Info("Initialize: Starting initialization of system contracts", "batchSeqNo", batch.SeqNo()) - if batch.SeqNo().Uint64() != common.L2SysContractGenesisSeqNo { - s.logger.Error("Initialize: Batch is not genesis", "batchSeqNo", batch.SeqNo) - return fmt.Errorf("batch is not genesis") - } - s.logger.Debug("Initialize: Deriving addresses from receipt", "transactionHash", receipt.TxHash.Hex()) - addresses, err := DeriveAddresses(&receipt) + addresses, err := verifyAndDeriveAddresses(batch, &receipt) if err != nil { - s.logger.Error("Initialize: Failed deriving addresses", "error", err, "receiptHash", receipt.TxHash.Hex()) - return fmt.Errorf("failed deriving addresses %w", err) + s.logger.Error("Initialize: Failed verifying and deriving addresses", "error", err) + return fmt.Errorf("failed verifying and deriving addresses %w", err) } - if err := msgBusManager.Initialize(addresses); err != nil { - s.logger.Error("Initialize: Failed deriving message bus address", "error", err) - return fmt.Errorf("failed deriving message bus address %w", err) + s.logger.Info("Initialize: Initializing required addresses", "addresses", addresses) + return s.initializeRequiredAddresses(addresses, msgBusManager) +} + +func verifyAndDeriveAddresses(batch *core.Batch, receipt *types.Receipt) (common.SystemContractAddresses, error) { + if batch.SeqNo().Uint64() != common.L2SysContractGenesisSeqNo { + return nil, fmt.Errorf("batch is not genesis") } - s.logger.Info("Initialize: Initializing required addresses", "addresses", addresses) - return s.initializeRequiredAddresses(addresses) + addresses, err := DeriveAddresses(receipt) + if err != nil { + return nil, fmt.Errorf("failed deriving addresses %w", err) + } + + return addresses, nil } func (s *systemContractCallbacks) CreatePublicCallbackHandlerTransaction(ctx context.Context, l2State *state.StateDB) (*types.Transaction, error) { diff --git a/go/node/cmd/cli.go b/go/node/cmd/cli.go index b8e206b8ac..a7ce886c60 100644 --- a/go/node/cmd/cli.go +++ b/go/node/cmd/cli.go @@ -23,6 +23,7 @@ type NodeConfigCLI struct { nodeAction string nodeType string isGenesis bool + numEnclaves int isSGXEnabled bool enclaveDockerImage string hostDockerImage string @@ -64,6 +65,7 @@ func ParseConfigCLI() *NodeConfigCLI { nodeName := flag.String(nodeNameFlag, "obscuronode", flagUsageMap[nodeNameFlag]) nodeType := flag.String(nodeTypeFlag, "", flagUsageMap[nodeTypeFlag]) isGenesis := flag.Bool(isGenesisFlag, false, flagUsageMap[isGenesisFlag]) + numEnclaves := flag.Int(numEnclavesFlag, 1, flagUsageMap[numEnclavesFlag]) isSGXEnabled := flag.Bool(isSGXEnabledFlag, false, flagUsageMap[isSGXEnabledFlag]) enclaveDockerImage := flag.String(enclaveDockerImageFlag, "", flagUsageMap[enclaveDockerImageFlag]) hostDockerImage := flag.String(hostDockerImageFlag, "", flagUsageMap[hostDockerImageFlag]) @@ -98,6 +100,7 @@ func ParseConfigCLI() *NodeConfigCLI { cfg.nodeName = *nodeName cfg.nodeType = *nodeType cfg.isGenesis = *isGenesis + cfg.numEnclaves = *numEnclaves cfg.isSGXEnabled = *isSGXEnabled cfg.enclaveDockerImage = *enclaveDockerImage cfg.hostDockerImage = *hostDockerImage @@ -166,6 +169,11 @@ func NodeCLIConfigToTenConfig(cliCfg *NodeConfigCLI) *config.TenConfig { fmt.Printf("Error loading default Ten config: %v\n", err) os.Exit(1) } + enclaveAddresses := make([]string, cliCfg.numEnclaves) + for i := 0; i < cliCfg.numEnclaves; i++ { + enclaveAddresses[i] = fmt.Sprintf("%s-enclave-%d:%d", + cliCfg.nodeName, i, cliCfg.enclaveWSPort) + } tenCfg.Network.L1.ChainID = int64(cliCfg.l1ChainID) tenCfg.Network.L1.L1Contracts.ManagementContract = gethcommon.HexToAddress(cliCfg.managementContractAddr) @@ -199,7 +207,7 @@ func NodeCLIConfigToTenConfig(cliCfg *NodeConfigCLI) *config.TenConfig { tenCfg.Host.DB.UseInMemory = false // these nodes always use a persistent DB tenCfg.Host.DB.PostgresHost = cliCfg.postgresDBHost tenCfg.Host.Debug.EnableDebugNamespace = cliCfg.isDebugNamespaceEnabled - tenCfg.Host.Enclave.RPCAddresses = []string{fmt.Sprintf("%s:%d", cliCfg.nodeName+"-enclave", cliCfg.enclaveWSPort)} + tenCfg.Host.Enclave.RPCAddresses = enclaveAddresses tenCfg.Host.L1.WebsocketURL = cliCfg.l1WebsocketURL tenCfg.Host.L1.L1BeaconUrl = cliCfg.l1BeaconUrl tenCfg.Host.L1.L1BlobArchiveUrl = cliCfg.l1BlobArchiveUrl @@ -209,12 +217,18 @@ func NodeCLIConfigToTenConfig(cliCfg *NodeConfigCLI) *config.TenConfig { tenCfg.Host.RPC.WSPort = uint64(cliCfg.hostWSPort) tenCfg.Host.Log.Level = cliCfg.logLevel - tenCfg.Enclave.DB.UseInMemory = false // these nodes always use a persistent DB - tenCfg.Enclave.DB.EdgelessDBHost = cliCfg.nodeName + "-edgelessdb" + tenCfg.Enclave.DB.UseInMemory = false // these nodes always use a persistent DB + tenCfg.Enclave.DB.EdgelessDBHost = cliCfg.nodeName + "-edgelessdb-" + "0" // will be dynamically set for HA tenCfg.Enclave.Debug.EnableDebugNamespace = cliCfg.isDebugNamespaceEnabled tenCfg.Enclave.EnableAttestation = cliCfg.isSGXEnabled tenCfg.Enclave.RPC.BindAddress = fmt.Sprintf("0.0.0.0:%d", cliCfg.enclaveWSPort) tenCfg.Enclave.Log.Level = cliCfg.logLevel + // the sequencer does not store the executed transactions + // todo - once we replace this launcher we'll configure this flag explicitly via an environment variable + if nodeType == common.ActiveSequencer { + tenCfg.Enclave.StoreExecutedTransactions = false + } + return tenCfg } diff --git a/go/node/cmd/cli_flags.go b/go/node/cmd/cli_flags.go index eff07fdc9f..c65e9fb5d7 100644 --- a/go/node/cmd/cli_flags.go +++ b/go/node/cmd/cli_flags.go @@ -5,6 +5,7 @@ const ( nodeNameFlag = "node_name" nodeTypeFlag = "node_type" isGenesisFlag = "is_genesis" + numEnclavesFlag = "num_enclaves" hostIDFlag = "host_id" isSGXEnabledFlag = "is_sgx_enabled" enclaveDockerImageFlag = "enclave_docker_image" @@ -43,7 +44,8 @@ func getFlagUsageMap() map[string]string { return map[string]string{ nodeNameFlag: "Specifies the node base name", nodeTypeFlag: "The node's type (e.g. sequencer, validator)", - isGenesisFlag: "Wether the node is the genesis node of the network", + isGenesisFlag: "Whether the node is the genesis node of the network", + numEnclavesFlag: "The number of enclaves to run as an HA setup (default 1, no HA pool)", hostIDFlag: "The 20 bytes of the address of the Obscuro host this enclave serves", isSGXEnabledFlag: "Whether the it should run on an SGX is enabled CPU", enclaveDockerImageFlag: "Docker image for the enclave", diff --git a/go/node/cmd/main.go b/go/node/cmd/main.go index 918ec841a7..5c3a9c7a19 100644 --- a/go/node/cmd/main.go +++ b/go/node/cmd/main.go @@ -9,7 +9,7 @@ func main() { tenCfg := NodeCLIConfigToTenConfig(cliConfig) - dockerNode := node.NewDockerNode(tenCfg, cliConfig.hostDockerImage, cliConfig.enclaveDockerImage, cliConfig.edgelessDBImage, false, cliConfig.pccsAddr) + dockerNode := node.NewDockerNode(tenCfg, cliConfig.hostDockerImage, cliConfig.enclaveDockerImage, cliConfig.edgelessDBImage, false, cliConfig.pccsAddr, cliConfig.numEnclaves) var err error switch cliConfig.nodeAction { case startAction: diff --git a/go/node/docker_node.go b/go/node/docker_node.go index ec9df21e30..9f4f22166a 100644 --- a/go/node/docker_node.go +++ b/go/node/docker_node.go @@ -20,9 +20,10 @@ type DockerNode struct { edgelessDBImage string enclaveDebugMode bool pccsAddr string // optional specified PCCS address + numEnclaves int // number of enclaves to start for the node as an HA setup } -func NewDockerNode(cfg *config.TenConfig, hostImage, enclaveImage, edgelessDBImage string, enclaveDebug bool, pccsAddr string) *DockerNode { +func NewDockerNode(cfg *config.TenConfig, hostImage, enclaveImage, edgelessDBImage string, enclaveDebug bool, pccsAddr string, numEnclaves int) *DockerNode { return &DockerNode{ cfg: cfg, hostImage: hostImage, @@ -30,6 +31,7 @@ func NewDockerNode(cfg *config.TenConfig, hostImage, enclaveImage, edgelessDBIma edgelessDBImage: edgelessDBImage, enclaveDebugMode: enclaveDebug, pccsAddr: pccsAddr, + numEnclaves: numEnclaves, } } @@ -37,14 +39,17 @@ func (d *DockerNode) Start() error { // todo (@pedro) - this should probably be removed in the future d.cfg.PrettyPrint() // dump config to stdout - err := d.startEdgelessDB() - if err != nil { - return fmt.Errorf("failed to start edgelessdb: %w", err) - } + var err error + for i := 0; i < d.numEnclaves; i++ { + err = d.startEdgelessDB(i) + if err != nil { + return fmt.Errorf("failed to start edgelessdb: %w", err) + } - err = d.startEnclave() - if err != nil { - return fmt.Errorf("failed to start enclave: %w", err) + err = d.startEnclave(i) + if err != nil { + return fmt.Errorf("failed to start enclave: %w", err) + } } err = d.startHost() @@ -62,9 +67,11 @@ func (d *DockerNode) Stop() error { return err } - err = docker.StopAndRemove(d.cfg.Node.Name + "-enclave") - if err != nil { - return err + for i := 0; i < d.numEnclaves; i++ { + err = docker.StopAndRemove(d.cfg.Node.Name + "-enclave-" + strconv.Itoa(i)) + if err != nil { + return err + } } return nil @@ -84,10 +91,12 @@ func (d *DockerNode) Upgrade(networkCfg *NetworkConfig) error { d.cfg.Network.L1.L1Contracts.MessageBusContract = common.HexToAddress(networkCfg.MessageBusAddress) d.cfg.Network.L1.StartHash = common.HexToHash(networkCfg.L1StartHash) - fmt.Println("Starting upgraded host and enclave") - err = d.startEnclave() - if err != nil { - return err + fmt.Println("Starting upgraded host and enclaves") + for i := 0; i < d.numEnclaves; i++ { + err = d.startEnclave(i) + if err != nil { + return err + } } err = d.startHost() @@ -124,7 +133,7 @@ func (d *DockerNode) startHost() error { return err } -func (d *DockerNode) startEnclave() error { +func (d *DockerNode) startEnclave(enclaveIdx int) error { devices := map[string]string{} exposedPorts := []int{} @@ -134,6 +143,9 @@ func (d *DockerNode) startEnclave() error { } if d.enclaveDebugMode { + if d.numEnclaves > 1 { + return fmt.Errorf("cannot run multiple enclaves in debug mode") + } cmd = []string{ "dlv", "--listen=:2345", @@ -147,6 +159,9 @@ func (d *DockerNode) startEnclave() error { exposedPorts = append(exposedPorts, 2345) } + // we set the edgeless DB address dynamically for each enclave + d.cfg.Enclave.DB.EdgelessDBHost = fmt.Sprintf("%s-edgelessdb-%d", d.cfg.Node.Name, enclaveIdx) + envVariables := d.cfg.ToEnvironmentVariables() if d.cfg.Enclave.EnableAttestation { @@ -163,16 +178,21 @@ func (d *DockerNode) startEnclave() error { cmd = append(cmd, "-willAttest=false") } + volumeName := fmt.Sprintf("%s-enclave-volume-%d", d.cfg.Node.Name, enclaveIdx) + containerName := fmt.Sprintf("%s-enclave-%d", d.cfg.Node.Name, enclaveIdx) + // we need the enclave volume to store the db credentials - enclaveVolume := map[string]string{d.cfg.Node.Name + "-enclave-volume": _enclaveDataDir} - _, err := docker.StartNewContainer(d.cfg.Node.Name+"-enclave", d.enclaveImage, cmd, exposedPorts, envVariables, devices, enclaveVolume, true) + enclaveVolume := map[string]string{volumeName: _enclaveDataDir} + _, err := docker.StartNewContainer(containerName, d.enclaveImage, cmd, exposedPorts, envVariables, devices, enclaveVolume, true) return err } -func (d *DockerNode) startEdgelessDB() error { +func (d *DockerNode) startEdgelessDB(enclaveIdx int) error { + containerName := fmt.Sprintf("%s-edgelessdb-%d", d.cfg.Node.Name, enclaveIdx) + volumeName := fmt.Sprintf("%s-db-volume-%d", d.cfg.Node.Name, enclaveIdx) envs := map[string]string{ - "EDG_EDB_CERT_DNS": d.cfg.Node.Name + "-edgelessdb", + "EDG_EDB_CERT_DNS": containerName, } devices := map[string]string{} @@ -188,11 +208,8 @@ func (d *DockerNode) startEdgelessDB() error { envs["PCCS_ADDR"] = d.pccsAddr } - // todo - do we need this volume? - //dbVolume := map[string]string{d.cfg.Node.Name + "-db-volume": "/data"} - //_, err := docker.StartNewContainer(d.cfg.Node.Name+"-edgelessdb", d.cfg.edgelessDBImage, nil, nil, envs, devices, dbVolume) - - _, err := docker.StartNewContainer(d.cfg.Node.Name+"-edgelessdb", d.edgelessDBImage, nil, nil, envs, devices, nil, true) + dbVolume := map[string]string{volumeName: "/data"} + _, err := docker.StartNewContainer(containerName, d.edgelessDBImage, nil, nil, envs, devices, dbVolume, true) return err } diff --git a/go/responses/responses.go b/go/responses/responses.go index f231083bd9..534a2c8b53 100644 --- a/go/responses/responses.go +++ b/go/responses/responses.go @@ -98,25 +98,6 @@ func AsEncryptedResponse[T any](data *T, encryptHandler Encryptor) *EnclaveRespo return AsPlaintextResponse(encrypted) } -// AsEncryptedEmptyResponse - encrypts an empty message -func AsEncryptedEmptyResponse(encryptHandler Encryptor) *EnclaveResponse { - userResp := UserResponse[any]{ - Result: nil, - } - - encoded, err := json.Marshal(userResp) - if err != nil { - return AsPlaintextError(err) - } - - encrypted, err := encryptHandler.Encrypt(encoded) - if err != nil { - return AsPlaintextError(err) - } - - return AsPlaintextResponse(encrypted) -} - // AsEncryptedError - Encodes and encrypts an error to be returned for a concrete user. func AsEncryptedError(err error, encrypt Encryptor) *EnclaveResponse { userResp := UserResponse[string]{ @@ -161,7 +142,7 @@ func DecodeResponse[T any](encoded []byte) (*T, error) { resp := UserResponse[T]{} err := json.Unmarshal(encoded, &resp) if err != nil { - return nil, err + return nil, fmt.Errorf("could not decode response. Cause: %w", err) } if resp.Err != nil { return nil, resp.Err diff --git a/integration/common/constants.go b/integration/common/constants.go index dcc11a05ce..67b631f3f2 100644 --- a/integration/common/constants.go +++ b/integration/common/constants.go @@ -88,8 +88,9 @@ func DefaultEnclaveConfig() *enclaveconfig.EnclaveConfig { // Due to hiding L1 costs in the gas quantity, the gas limit needs to be huge // Arbitrum with the same approach has gas limit of 1,125,899,906,842,624, // whilst the usage is small. Should be ok since execution is paid for anyway. - GasLocalExecutionCapFlag: 300_000_000_000, - GasBatchExecutionLimit: 300_000_000_000, - RPCTimeout: 5 * time.Second, + GasLocalExecutionCapFlag: 300_000_000_000, + GasBatchExecutionLimit: 30_000_000, + RPCTimeout: 5 * time.Second, + StoreExecutedTransactions: true, } } diff --git a/integration/simulation/devnetwork/node.go b/integration/simulation/devnetwork/node.go index 5f9bc5f9f5..dbb6a952b1 100644 --- a/integration/simulation/devnetwork/node.go +++ b/integration/simulation/devnetwork/node.go @@ -227,6 +227,7 @@ func (n *InMemNodeOperator) createEnclaveContainer(idx int) *enclavecontainer.En GasPaymentAddress: defaultCfg.GasPaymentAddress, RPCTimeout: 5 * time.Second, SystemContractOwner: gethcommon.HexToAddress("0xA58C60cc047592DE97BF1E8d2f225Fc5D959De77"), + StoreExecutedTransactions: true, } return enclavecontainer.NewEnclaveContainerWithLogger(enclaveConfig, enclaveLogger) } diff --git a/integration/simulation/network/geth_utils.go b/integration/simulation/network/geth_utils.go index 99a03954c9..f0803a5806 100644 --- a/integration/simulation/network/geth_utils.go +++ b/integration/simulation/network/geth_utils.go @@ -149,6 +149,30 @@ func DeployTenNetworkContracts(client ethadapter.EthClient, wallets *params.SimW }, nil } +func PermissionTenSequencerEnclave(mcOwner wallet.Wallet, client ethadapter.EthClient, mcAddress common.Address, seqEnclaveID common.Address) error { + ctr, err := ManagementContract.NewManagementContract(mcAddress, client.EthClient()) + if err != nil { + return err + } + + opts, err := bind.NewKeyedTransactorWithChainID(mcOwner.PrivateKey(), mcOwner.ChainID()) + if err != nil { + return err + } + + tx, err := ctr.GrantSequencerEnclave(opts, seqEnclaveID) + if err != nil { + return err + } + + _, err = integrationCommon.AwaitReceiptEth(context.Background(), client.EthClient(), tx.Hash(), 25*time.Second) + if err != nil { + return err + } + + return nil +} + func StopEth2Network(clients []ethadapter.EthClient, network eth2network.PosEth2Network) { // Stop the clients first for _, c := range clients { diff --git a/integration/simulation/network/network_utils.go b/integration/simulation/network/network_utils.go index d333eb934f..49cbc8ac16 100644 --- a/integration/simulation/network/network_utils.go +++ b/integration/simulation/network/network_utils.go @@ -101,8 +101,9 @@ func createInMemTenNode( MaxRollupSize: 1024 * 128, BaseFee: big.NewInt(1), // todo @siliev:: fix test transaction builders so this can be different GasLocalExecutionCapFlag: params.MaxGasLimit / 2, - GasBatchExecutionLimit: params.MaxGasLimit / 2, + GasBatchExecutionLimit: 30_000_000, RPCTimeout: 5 * time.Second, + StoreExecutedTransactions: true, } enclaveLogger := testlog.Logger().New(log.NodeIDKey, id, log.CmpKey, log.EnclaveCmp) diff --git a/integration/simulation/network/socket.go b/integration/simulation/network/socket.go index 20afad5572..2e79d4fca5 100644 --- a/integration/simulation/network/socket.go +++ b/integration/simulation/network/socket.go @@ -147,6 +147,20 @@ func (n *networkOfSocketNodes) Create(simParams *params.SimParams, _ *stats.Stat } walletClients := createAuthClientsPerWallet(n.l2Clients, simParams.Wallets) + // permission the sequencer enclaveID + seqHealth, err := n.tenClients[0].Health() + if err != nil { + return nil, fmt.Errorf("unable to get sequencer enclaveID: %w", err) + } + if seqHealth.Enclaves == nil || len(seqHealth.Enclaves) == 0 { + return nil, fmt.Errorf("no enclaves found in health response") + } + seqEnclaveID := seqHealth.Enclaves[0].EnclaveID + err = PermissionTenSequencerEnclave(n.wallets.MCOwnerWallet, n.gethClients[0], simParams.L1TenData.MgmtContractAddress, seqEnclaveID) + if err != nil { + return nil, fmt.Errorf("unable to permission sequencer enclaveID: %w", err) + } + return &RPCHandles{ EthClients: n.gethClients, TenClients: n.tenClients, diff --git a/integration/simulation/transaction_injector.go b/integration/simulation/transaction_injector.go index 6888055def..cf200134e9 100644 --- a/integration/simulation/transaction_injector.go +++ b/integration/simulation/transaction_injector.go @@ -447,7 +447,7 @@ func (ti *TransactionInjector) issueRandomWithdrawals() { tx := &types.LegacyTx{ Nonce: fromWallet.GetNonceAndIncrement(), Value: gethcommon.Big1, - Gas: uint64(1_000_000_000), + Gas: uint64(1_000_000), GasPrice: price, Data: nil, To: &msgBusAddr, @@ -530,7 +530,7 @@ func (ti *TransactionInjector) newTx(data []byte, nonce uint64, ercType testcomm return &types.LegacyTx{ Nonce: nonce, Value: gethcommon.Big0, - Gas: uint64(1_000_000_000), + Gas: uint64(1_000_000), GasPrice: gethcommon.Big1, Data: data, To: ti.wallets.Tokens[ercType].L2ContractAddress, diff --git a/integration/tengateway/tengateway_test.go b/integration/tengateway/tengateway_test.go index 322bc1b4c7..3d9afd8d28 100644 --- a/integration/tengateway/tengateway_test.go +++ b/integration/tengateway/tengateway_test.go @@ -116,7 +116,6 @@ func TestTenGateway(t *testing.T) { "testSubscriptionTopics": testSubscriptionTopics, "testDifferentMessagesOnRegister": testDifferentMessagesOnRegister, "testInvokeNonSensitiveMethod": testInvokeNonSensitiveMethod, - "testGetStorageAtForReturningUserID": testGetStorageAtForReturningUserID, // "testRateLimiter": testRateLimiter, "testSessionKeys": testSessionKeys, } { @@ -263,7 +262,7 @@ func interactWithSmartContractUnsigned(client *ethclient.Client, sendRaw bool, n interactionTx := types.LegacyTx{ Nonce: nonce, To: &contractAddress, - Gas: uint64(10_000_000), + Gas: uint64(1_000_000), GasPrice: result.ToInt(), Data: contractInteractionData, Value: value, @@ -769,7 +768,7 @@ func testUnsubscribe(t *testing.T, _ int, httpURL, wsURL string, w wallet.Wallet // deploy events contract deployTx := &types.LegacyTx{ Nonce: w.GetNonceAndIncrement(), - Gas: uint64(10_000_000), + Gas: uint64(1_000_000), GasPrice: gethcommon.Big1, Data: gethcommon.FromHex(eventsContractBytecode), } @@ -899,54 +898,6 @@ func testInvokeNonSensitiveMethod(t *testing.T, _ int, httpURL, wsURL string, w } } -func testGetStorageAtForReturningUserID(t *testing.T, _ int, httpURL, wsURL string, w wallet.Wallet) { - user, err := NewGatewayUser([]wallet.Wallet{w}, httpURL, wsURL) - require.NoError(t, err) - - type JSONResponse struct { - Result string `json:"result"` - } - var response JSONResponse - - // make a request to GetStorageAt with correct parameters to get userID that exists in the database - respBody := makeHTTPEthJSONReq(httpURL, "eth_getStorageAt", user.tgClient.UserID(), []interface{}{common.UserIDRequestCQMethod, "0", nil}) - if err = json.Unmarshal(respBody, &response); err != nil { - t.Error("Unable to unmarshal response") - } - if !bytes.Equal(gethcommon.FromHex(response.Result), user.tgClient.UserIDBytes()) { - t.Errorf("Wrong ID returned. Expected: %s, received: %s", user.tgClient.UserID(), response.Result) - } - - // make a request to GetStorageAt with correct parameters to get userID, but with wrong userID - respBody2 := makeHTTPEthJSONReq(httpURL, "eth_getStorageAt", "0x0000000000000000000000000000000000000001", []interface{}{common.UserIDRequestCQMethod, "0", nil}) - if !strings.Contains(string(respBody2), "not found") { - t.Error("eth_getStorageAt did not respond with not found error") - } - - err = user.RegisterAccounts() - if err != nil { - t.Errorf("Failed to register accounts: %s", err) - return - } - - // make a request to GetStorageAt with wrong parameters to get userID, but correct userID - respBody3 := makeHTTPEthJSONReq(httpURL, "eth_getStorageAt", user.tgClient.UserID(), []interface{}{"0x0000000000000000000000000000000000000007", "0", nil}) - expectedErr := "not supported" - if !strings.Contains(string(respBody3), expectedErr) { - t.Errorf("eth_getStorageAt did not respond with error: %s, it was: %s", expectedErr, string(respBody3)) - } - - privateTxs, _ := json.Marshal(common.ListPrivateTransactionsQueryParams{ - Address: gethcommon.HexToAddress("0xA58C60cc047592DE97BF1E8d2f225Fc5D959De77"), - Pagination: common.QueryPagination{Size: 10}, - }) - - respBody4 := makeHTTPEthJSONReq(httpURL, "eth_getStorageAt", user.tgClient.UserID(), []interface{}{common.ListPrivateTransactionsCQMethod, string(privateTxs), nil}) - if err = json.Unmarshal(respBody4, &response); err != nil { - t.Error("Unable to unmarshal response") - } -} - func makeRequestHTTP(url string, body []byte) []byte { generateViewingKeyBody := bytes.NewBuffer(body) resp, err := http.Post(url, "application/json", generateViewingKeyBody) //nolint:noctx,gosec diff --git a/packages/ui/routes/index.ts b/packages/ui/routes/index.ts index ddb30d872c..a2d5d960ad 100644 --- a/packages/ui/routes/index.ts +++ b/packages/ui/routes/index.ts @@ -2,7 +2,6 @@ export const requestMethods = { requestAccounts: "eth_requestAccounts", switchNetwork: "wallet_switchEthereumChain", addNetwork: "wallet_addEthereumChain", - getStorageAt: "eth_getStorageAt", signTypedData: "eth_signTypedData_v4", getChainId: "eth_chainId", }; diff --git a/testnet/launcher/docker.go b/testnet/launcher/docker.go index 9f985eb979..07e1a0f916 100644 --- a/testnet/launcher/docker.go +++ b/testnet/launcher/docker.go @@ -43,11 +43,6 @@ func (t *Testnet) Start() error { return fmt.Errorf("unable to deploy l1 contracts - %w", err) } - println("MANAGEMENT CONTRACT: ", networkConfig.ManagementContractAddress) - println("MANAGEMENT CONTRACT: ", networkConfig.ManagementContractAddress) - println("MANAGEMENT CONTRACT: ", networkConfig.ManagementContractAddress) - println("MANAGEMENT CONTRACT: ", networkConfig.ManagementContractAddress) - println("MANAGEMENT CONTRACT: ", networkConfig.ManagementContractAddress) println("MANAGEMENT CONTRACT: ", networkConfig.ManagementContractAddress) edgelessDBImage := "ghcr.io/edgelesssys/edgelessdb-sgx-4gb:v0.3.2" @@ -67,13 +62,7 @@ func (t *Testnet) Start() error { sequencerCfg.Network.L1.L1Contracts.ManagementContract = common.HexToAddress(networkConfig.ManagementContractAddress) sequencerCfg.Network.L1.L1Contracts.MessageBusContract = common.HexToAddress(networkConfig.MessageBusAddress) - sequencerNode := node.NewDockerNode(sequencerCfg, - "testnetobscuronet.azurecr.io/obscuronet/host:latest", - "testnetobscuronet.azurecr.io/obscuronet/enclave:latest", - edgelessDBImage, - false, - "", // no PCCS override - ) + sequencerNode := node.NewDockerNode(sequencerCfg, "testnetobscuronet.azurecr.io/obscuronet/host:latest", "testnetobscuronet.azurecr.io/obscuronet/enclave:latest", edgelessDBImage, false, "", 1) err = sequencerNode.Start() if err != nil { @@ -98,13 +87,7 @@ func (t *Testnet) Start() error { validatorNodeCfg.Network.L1.L1Contracts.ManagementContract = common.HexToAddress(networkConfig.ManagementContractAddress) validatorNodeCfg.Network.L1.L1Contracts.MessageBusContract = common.HexToAddress(networkConfig.MessageBusAddress) - validatorNode := node.NewDockerNode(validatorNodeCfg, - "testnetobscuronet.azurecr.io/obscuronet/host:latest", - "testnetobscuronet.azurecr.io/obscuronet/enclave:latest", - edgelessDBImage, - false, - "", // no PCCS override - ) + validatorNode := node.NewDockerNode(validatorNodeCfg, "testnetobscuronet.azurecr.io/obscuronet/host:latest", "testnetobscuronet.azurecr.io/obscuronet/enclave:latest", edgelessDBImage, false, "", 1) err = validatorNode.Start() if err != nil { return fmt.Errorf("unable to start the obscuro node - %w", err) diff --git a/tools/edbconnect/Dockerfile b/tools/edbconnect/Dockerfile index b117a9154f..8eae91e84d 100644 --- a/tools/edbconnect/Dockerfile +++ b/tools/edbconnect/Dockerfile @@ -4,7 +4,7 @@ # deploy = copies over only the enclave executable without the source # in a lightweight base image specialized for deployment and prepares the /data/ folder. -FROM ghcr.io/edgelesssys/ego-dev:v1.5.3 AS build-base +FROM ghcr.io/edgelesssys/ego-dev:v1.6.0 AS build-base # setup container data structure RUN mkdir -p /home/ten/go-ten @@ -32,7 +32,7 @@ RUN ego sign edb-enclave.json # Trigger a new build stage and use the smaller ego version: -FROM ghcr.io/edgelesssys/ego-deploy:v1.5.3 +FROM ghcr.io/edgelesssys/ego-deploy:v1.6.0 # Copy the binary and the entrypoint script COPY --from=sign-built-enclave \ diff --git a/tools/edbconnect/edb-connect.sh b/tools/edbconnect/edb-connect.sh index 37d14c740a..36b0ea2575 100644 --- a/tools/edbconnect/edb-connect.sh +++ b/tools/edbconnect/edb-connect.sh @@ -5,11 +5,12 @@ IMAGE_NAME="testnetobscuronet.azurecr.io/obscuronet/edbconnect:latest" CONTAINER_BASE_NAME="edb-connect" UNIQUE_ID=$(date +%s%3N) # Using milliseconds for uniqueness CONTAINER_NAME="${CONTAINER_BASE_NAME}-${UNIQUE_ID}" -VOLUME_NAME="obscuronode-enclave-volume" +VOLUME_NAME="obscuronode-enclave-volume-0" +DB_HOST="obscuronode-edgelessdb-0" NETWORK_NAME="node_network" SGX_ENCLAVE_DEVICE="/dev/sgx_enclave" SGX_PROVISION_DEVICE="/dev/sgx_provision" -COMMAND="ego run /home/ten/go-ten/tools/edbconnect/main/main" +COMMAND="ego run /home/ten/go-ten/tools/edbconnect/main/main $DB_HOST" # Function to destroy exited containers matching the base name destroy_exited_containers() { diff --git a/tools/edbconnect/main/main.go b/tools/edbconnect/main/main.go index 42b2c265b8..28f1513a1b 100644 --- a/tools/edbconnect/main/main.go +++ b/tools/edbconnect/main/main.go @@ -12,6 +12,16 @@ import ( ) func main() { + // get edbHost from first command line arg + var edbHost string + if len(os.Args) > 1 { + edbHost = os.Args[1] + } else { + fmt.Println("Usage: edbconnect ") + fmt.Println("Ensure you have the latest copy of the ./edb-connect.sh launch script if you see this error.") + os.Exit(1) + } + fmt.Println("Retrieving Edgeless DB credentials...") creds, found, err := edgelessdb.LoadCredentialsFromFile() if err != nil { @@ -29,9 +39,9 @@ func main() { } fmt.Println("TLS config created. Connecting to Edgeless DB...") testlog.SetupSysOut() - db, err := edgelessdb.ConnectToEdgelessDB("obscuronode-edgelessdb", cfg, testlog.Logger()) + db, err := edgelessdb.ConnectToEdgelessDB(edbHost, cfg, testlog.Logger()) if err != nil { - fmt.Println("Error connecting to Edgeless DB:", err) + fmt.Printf("Error connecting to Edgeless DB at %s: %v\n", edbHost, err) panic(err) } fmt.Println("Connected to Edgeless DB.") diff --git a/tools/hardhatdeployer/contract_deployer.go b/tools/hardhatdeployer/contract_deployer.go index 59684be16e..6604b8ce42 100644 --- a/tools/hardhatdeployer/contract_deployer.go +++ b/tools/hardhatdeployer/contract_deployer.go @@ -93,7 +93,7 @@ func (cd *contractDeployer) run() (string, error) { deployContractTx := types.LegacyTx{ Nonce: cd.wallet.GetNonceAndIncrement(), GasPrice: big.NewInt(params.InitialBaseFee), - Gas: uint64(500_000_000), + Gas: uint64(5_000_000), Data: cd.contractCode, } diff --git a/tools/walletextension/enclave.Dockerfile b/tools/walletextension/enclave.Dockerfile index dd2893ed0c..5f4285d29c 100644 --- a/tools/walletextension/enclave.Dockerfile +++ b/tools/walletextension/enclave.Dockerfile @@ -9,7 +9,7 @@ # /data persistent volume mount point # Trigger new build stage for compiling the enclave -FROM ghcr.io/edgelesssys/ego-dev:v1.5.3 AS build-base +FROM ghcr.io/edgelesssys/ego-dev:v1.6.0 AS build-base # Install ca-certificates package and update it RUN apt-get update && apt-get install -y \ @@ -39,7 +39,7 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ # Sign the enclave executable RUN ego sign enclave.json -FROM ghcr.io/edgelesssys/ego-deploy:v1.5.3 +FROM ghcr.io/edgelesssys/ego-deploy:v1.6.0 # Create data directory that will be used for persistence RUN mkdir -p /data && chmod 777 /data diff --git a/tools/walletextension/frontend/src/api/ethRequests.ts b/tools/walletextension/frontend/src/api/ethRequests.ts index ed8583be6e..695e50346c 100644 --- a/tools/walletextension/frontend/src/api/ethRequests.ts +++ b/tools/walletextension/frontend/src/api/ethRequests.ts @@ -82,27 +82,20 @@ export const getSignature = async (account: string, data: any) => { } }; -export const getToken = async (provider: ethers.providers.Web3Provider) => { - if (!provider.send) { - return null; - } +export const getToken = async () => { try { - if (await isTenChain()) { - const token = await provider.send(requestMethods.getStorageAt, [ - userStorageAddress, - getRandomIntAsString(0, 1000), - null, - ]); - return token; - } else { - return null; - } + const token = localStorage.getItem('ten_token') || ''; + return token; } catch (e: any) { console.error(e); throw e; } }; +export const clearToken = () => { + localStorage.removeItem('ten_token'); +}; + export async function addNetworkToMetaMask(rpcUrls: string[]) { if (!ethereum) { throw "No ethereum object found"; @@ -150,7 +143,7 @@ export async function authenticateAccountWithTenGatewayEIP712( ...typedData, message: { ...typedData.message, - "Encryption Token": token, + "Encryption Token": token, }, }; const signature = await getSignature(account, data); diff --git a/tools/walletextension/frontend/src/components/providers/wallet-provider.tsx b/tools/walletextension/frontend/src/components/providers/wallet-provider.tsx index 11c77be0af..24e7d6b04f 100644 --- a/tools/walletextension/frontend/src/components/providers/wallet-provider.tsx +++ b/tools/walletextension/frontend/src/components/providers/wallet-provider.tsx @@ -15,6 +15,7 @@ import { ToastType } from "@/types/interfaces"; import { authenticateAccountWithTenGatewayEIP712, getToken, + clearToken } from "@/api/ethRequests"; import { ethers } from "ethers"; import ethService from "@/services/ethService"; @@ -56,7 +57,7 @@ export const WalletConnectionProvider = ({ try { await ethService.checkIfMetamaskIsLoaded(providerInstance); - const fetchedToken = await getToken(providerInstance); + const fetchedToken = await getToken(); setToken(fetchedToken); const status = await ethService.isUserConnectedToTenChain(fetchedToken); @@ -124,6 +125,7 @@ export const WalletConnectionProvider = ({ setAccounts(null); setWalletConnected(false); setToken(""); + clearToken(); } }; @@ -135,7 +137,7 @@ export const WalletConnectionProvider = ({ ); return; } - const token = await getToken(provider); + const token = await getToken(); if (!isValidTokenFormat(token)) { showToast( @@ -194,6 +196,7 @@ export const WalletConnectionProvider = ({ setAccounts(null); setWalletConnected(false); setToken(""); + clearToken(); } else { window.location.reload(); } diff --git a/tools/walletextension/frontend/src/services/ethService.ts b/tools/walletextension/frontend/src/services/ethService.ts index 92d749b881..5b5f39feef 100644 --- a/tools/walletextension/frontend/src/services/ethService.ts +++ b/tools/walletextension/frontend/src/services/ethService.ts @@ -127,7 +127,7 @@ const ethService = { return; } - const token = await getToken(provider); + const token = await getToken(); if (!token || !isValidTokenFormat(token)) { return; diff --git a/tools/walletextension/frontend/src/services/useGatewayService.ts b/tools/walletextension/frontend/src/services/useGatewayService.ts index 5e25ea27ea..23a792b03f 100644 --- a/tools/walletextension/frontend/src/services/useGatewayService.ts +++ b/tools/walletextension/frontend/src/services/useGatewayService.ts @@ -52,10 +52,14 @@ const useGatewayService = () => { // SWITCHED_CODE=4902; error 4902 means that the chain does not exist if ( switched === SWITCHED_CODE || - !isValidTokenFormat(await getToken(provider)) + !isValidTokenFormat(await getToken()) ) { showToast(ToastType.INFO, "Adding TEN Testnet..."); const user = await joinTestnet(); + + // Store the token in localStorage + localStorage.setItem("ten_token", "0x" + user); + const rpcUrls = [ `${tenGatewayAddress}/${tenGatewayVersion}/?token=${user}`, ]; diff --git a/tools/walletextension/rpcapi/blockchain_api.go b/tools/walletextension/rpcapi/blockchain_api.go index dbb085c650..9ae0a86e8e 100644 --- a/tools/walletextension/rpcapi/blockchain_api.go +++ b/tools/walletextension/rpcapi/blockchain_api.go @@ -190,8 +190,6 @@ func (api *BlockChainAPI) GetStorageAt(ctx context.Context, address gethcommon.A } switch address.Hex() { - case common.UserIDRequestCQMethod: // todo - review whether we need this endpoint - return user.ID, nil case common.ListPrivateTransactionsCQMethod: // sensitive CustomQuery methods use the convention of having "address" at the top level of the params json userAddr, err := extractCustomQueryAddress(params) diff --git a/tools/walletextension/storage/database/sqlite/sqlite.go b/tools/walletextension/storage/database/sqlite/sqlite.go index 832fc1117c..32f47e784d 100644 --- a/tools/walletextension/storage/database/sqlite/sqlite.go +++ b/tools/walletextension/storage/database/sqlite/sqlite.go @@ -1,11 +1,19 @@ package sqlite /* - SQLite database implementation of the Storage interface + SQLite database implementation of the Storage interface. - SQLite is used for local deployments and testing without the need for a cloud database. - To make sure to see similar behaviour as in production using CosmosDB we use SQLite database in a similar way as comosDB (as key-value database). + This implementation mimics the CosmosDB approach where we store the entire user record (including accounts and session keys) + in a single JSON object within the 'users' table. There are no separate tables for accounts or session keys. + + Each user record: + { + "user_data": + } + + This simplifies the schema and keeps it similar to the CosmosDB container-based storage. */ + import ( "database/sql" "encoding/json" @@ -15,15 +23,14 @@ import ( "path/filepath" "github.com/ethereum/go-ethereum/crypto" + _ "github.com/mattn/go-sqlite3" // sqlite driver for sql.Open() dbcommon "github.com/ten-protocol/go-ten/tools/walletextension/storage/database/common" - "github.com/ten-protocol/go-ten/go/common/viewingkey" - "github.com/ten-protocol/go-ten/tools/walletextension/common" - - _ "github.com/mattn/go-sqlite3" // sqlite driver for sql.Open() obscurocommon "github.com/ten-protocol/go-ten/go/common" "github.com/ten-protocol/go-ten/go/common/errutil" + "github.com/ten-protocol/go-ten/go/common/viewingkey" + "github.com/ten-protocol/go-ten/tools/walletextension/common" ) type SqliteDB struct { @@ -33,7 +40,7 @@ type SqliteDB struct { const sqliteCfg = "_foreign_keys=on&_journal_mode=wal&_txlock=immediate&_synchronous=normal" func NewSqliteDatabase(dbPath string) (*SqliteDB, error) { - // load the db file + // load or create the db file dbFilePath, err := createOrLoad(dbPath) if err != nil { return nil, err @@ -43,17 +50,16 @@ func NewSqliteDatabase(dbPath string) (*SqliteDB, error) { path := fmt.Sprintf("file:%s?%s", dbFilePath, sqliteCfg) db, err := sql.Open("sqlite3", path) if err != nil { - fmt.Println("Error opening database: ", err) - return nil, err + return nil, fmt.Errorf("error opening database: %w", err) } - // enable foreign keys in sqlite + // Enable foreign keys in SQLite (harmless, even though we don't use them now) _, err = db.Exec("PRAGMA foreign_keys = ON;") if err != nil { return nil, err } - // Modify the users table to store the entire GWUserDB as JSON + // Create the users table if it doesn't exist. We store entire user as JSON. _, err = db.Exec(`CREATE TABLE IF NOT EXISTS users ( id TEXT PRIMARY KEY, user_data TEXT @@ -62,7 +68,9 @@ func NewSqliteDatabase(dbPath string) (*SqliteDB, error) { return nil, err } - // Remove the accounts table as it will be stored within the user_data JSON + // If there was an old 'accounts' table from a previous implementation, drop it. + // This ensures no leftover foreign key constraints cause issues. + _, _ = db.Exec("DROP TABLE IF EXISTS accounts;") return &SqliteDB{db: db}, nil } @@ -73,23 +81,23 @@ func (s *SqliteDB) AddUser(userID []byte, privateKey []byte) error { PrivateKey: privateKey, Accounts: []dbcommon.GWAccountDB{}, } + userJSON, err := json.Marshal(user) if err != nil { - return err + return fmt.Errorf("failed to marshal user data: %w", err) } return s.withTx(func(dbTx *sql.Tx) error { stmt, err := dbTx.Prepare("INSERT OR REPLACE INTO users(id, user_data) VALUES (?, ?)") if err != nil { - return err + return fmt.Errorf("failed to prepare insert statement: %w", err) } defer stmt.Close() _, err = stmt.Exec(string(user.UserId), string(userJSON)) if err != nil { - return err + return fmt.Errorf("failed to insert user: %w", err) } - return nil }) } @@ -98,7 +106,7 @@ func (s *SqliteDB) DeleteUser(userID []byte) error { return s.withTx(func(dbTx *sql.Tx) error { stmt, err := dbTx.Prepare("DELETE FROM users WHERE id = ?") if err != nil { - return err + return fmt.Errorf("failed to prepare delete statement: %w", err) } defer stmt.Close() @@ -106,18 +114,24 @@ func (s *SqliteDB) DeleteUser(userID []byte) error { if err != nil { return fmt.Errorf("failed to delete user: %w", err) } - return nil }) } -func (s *SqliteDB) ActivateSessionKey(userID []byte, active bool) error { +func (s *SqliteDB) AddAccount(userID []byte, accountAddress []byte, signature []byte, signatureType viewingkey.SignatureType) error { return s.withTx(func(dbTx *sql.Tx) error { user, err := s.readUser(dbTx, userID) if err != nil { return err } - user.ActiveSK = active + + newAccount := dbcommon.GWAccountDB{ + AccountAddress: accountAddress, + Signature: signature, + SignatureType: int(signatureType), + } + + user.Accounts = append(user.Accounts, newAccount) return s.updateUser(dbTx, user) }) } @@ -140,32 +154,24 @@ func (s *SqliteDB) AddSessionKey(userID []byte, key common.GWSessionKey) error { }) } -func (s *SqliteDB) RemoveSessionKey(userID []byte) error { +func (s *SqliteDB) ActivateSessionKey(userID []byte, active bool) error { return s.withTx(func(dbTx *sql.Tx) error { user, err := s.readUser(dbTx, userID) if err != nil { return err } - user.SessionKey = nil + user.ActiveSK = active return s.updateUser(dbTx, user) }) } -func (s *SqliteDB) AddAccount(userID []byte, accountAddress []byte, signature []byte, signatureType viewingkey.SignatureType) error { +func (s *SqliteDB) RemoveSessionKey(userID []byte) error { return s.withTx(func(dbTx *sql.Tx) error { user, err := s.readUser(dbTx, userID) if err != nil { return err } - - newAccount := dbcommon.GWAccountDB{ - AccountAddress: accountAddress, - Signature: signature, - SignatureType: int(signatureType), - } - - user.Accounts = append(user.Accounts, newAccount) - + user.SessionKey = nil return s.updateUser(dbTx, user) }) } @@ -175,10 +181,7 @@ func (s *SqliteDB) GetUser(userID []byte) (*common.GWUser, error) { var err error err = s.withTx(func(dbTx *sql.Tx) error { user, err = s.readUser(dbTx, userID) - if err != nil { - return err - } - return nil + return err }) if err != nil { return nil, err @@ -212,41 +215,19 @@ func (s *SqliteDB) updateUser(dbTx *sql.Tx, user dbcommon.GWUserDB) error { stmt, err := dbTx.Prepare("UPDATE users SET user_data = ? WHERE id = ?") if err != nil { - return err + return fmt.Errorf("failed to prepare update statement: %w", err) } defer stmt.Close() _, err = stmt.Exec(string(updatedUserJSON), string(user.UserId)) if err != nil { - return fmt.Errorf("failed to update user with new account: %w", err) + return fmt.Errorf("failed to update user: %w", err) } return nil } -func createOrLoad(dbPath string) (string, error) { - // If path is empty we create a random throwaway temp file, otherwise we use the path to the database - if dbPath == "" { - tempDir := filepath.Join("/tmp", "obscuro_gateway", obscurocommon.RandomStr(8)) - err := os.MkdirAll(tempDir, os.ModePerm) - if err != nil { - fmt.Println("Error creating directory: ", tempDir, err) - return "", err - } - dbPath = filepath.Join(tempDir, "gateway_databse.db") - } else { - dir := filepath.Dir(dbPath) - err := os.MkdirAll(dir, 0o755) - if err != nil { - fmt.Println("Error creating directories:", err) - return "", err - } - } - - return dbPath, nil -} - -// GetEncryptionKey returns nil for SQLite as it doesn't use encryption +// GetEncryptionKey returns nil for SQLite as it doesn't use encryption directly in this implementation. func (s *SqliteDB) GetEncryptionKey() []byte { return nil } @@ -254,14 +235,33 @@ func (s *SqliteDB) GetEncryptionKey() []byte { func (s *SqliteDB) withTx(fn func(*sql.Tx) error) error { tx, err := s.db.Begin() if err != nil { - return err + return fmt.Errorf("failed to begin transaction: %w", err) } defer tx.Rollback() - err = fn(tx) - if err != nil { + if err := fn(tx); err != nil { return err } return tx.Commit() } + +func createOrLoad(dbPath string) (string, error) { + // If path is empty we create a random temporary file, otherwise we use the provided path + if dbPath == "" { + tempDir := filepath.Join("/tmp", "obscuro_gateway", obscurocommon.RandomStr(8)) + err := os.MkdirAll(tempDir, os.ModePerm) + if err != nil { + return "", fmt.Errorf("error creating directory %s: %w", tempDir, err) + } + dbPath = filepath.Join(tempDir, "gateway_database.db") + } else { + dir := filepath.Dir(dbPath) + err := os.MkdirAll(dir, 0o755) + if err != nil { + return "", fmt.Errorf("error creating directories: %w", err) + } + } + + return dbPath, nil +}