From 0e48362262d7a1410c469ddbe7c494727b1b1bc7 Mon Sep 17 00:00:00 2001 From: Tudor Malene Date: Mon, 25 Sep 2023 09:29:05 +0100 Subject: [PATCH 1/3] small doc changes (#1547) * small doc changes * address pr comments --- docs/_data/navigation.yml | 4 ++++ docs/_docs/testnet/security.md | 10 ++++++++ docs/_docs/what-is-obscuro/quick-start.md | 29 +++++++++++++++++++++++ docs/index.md | 18 ++++++++++++-- 4 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 docs/_docs/testnet/security.md create mode 100644 docs/_docs/what-is-obscuro/quick-start.md diff --git a/docs/_data/navigation.yml b/docs/_data/navigation.yml index 46fb912d5a..1c7f50e2a6 100644 --- a/docs/_data/navigation.yml +++ b/docs/_data/navigation.yml @@ -14,6 +14,8 @@ sidebar-list: children: - title: Encrypted Ethereum url: what-is-obscuro/encrypted-ethereum + - title: Developer quick start + url: what-is-obscuro/quick-start - title: Technology url: what-is-obscuro/technology - title: User Experience @@ -39,6 +41,8 @@ sidebar-list: url: testnet/deploying-a-smart-contract-programmatically - title: Change Log url: testnet/changelog + - title: Security + url: testnet/security - title: On Chain Capabilities children: diff --git a/docs/_docs/testnet/security.md b/docs/_docs/testnet/security.md new file mode 100644 index 0000000000..530b22b6a6 --- /dev/null +++ b/docs/_docs/testnet/security.md @@ -0,0 +1,10 @@ +--- +--- +# Testnet Security + +The first Obscuro Testnet is focused on functionality and the User and Developer experience. + +Privacy features require special attention from the core and security audit team and will be finalised in a +future version of Testnet. + +As a user of the "Obscuro Testnet", do not expect the data you are loading to be 100% private. diff --git a/docs/_docs/what-is-obscuro/quick-start.md b/docs/_docs/what-is-obscuro/quick-start.md new file mode 100644 index 0000000000..ea420665d8 --- /dev/null +++ b/docs/_docs/what-is-obscuro/quick-start.md @@ -0,0 +1,29 @@ +--- +--- +# Developer quick start + +The only difference between an Obscuro and an Ethereum (or Arbitrum) dApp is that on Obscuro you can hide the internal +state of the contract. + +The most obvious example is that an ERC20 token deployed on Obscuro will not respond to balance requests unless you are +the account owner. + +In Obscuro, the internal node database is encrypted, and the contract execution is also encrypted inside the TEE. +The calls to [getStorageAt](https://docs.alchemy.com/reference/eth-getstorageat) are disabled, so all data access +requests will be performed through view functions which are under the control of the smart contract developer. + +Nobody (which includes node operators and the sequencer) can access the internal state of a contract. + +**The only thing you have to do when porting a dApp to Obscuro is to add a check in your view functions comparing +the `tx.origing` and `msg.sender` against the accounts allowed to access that data.** + +The snippet below illustrates this for an [ERC20 token](https://github.com/obscuronet/sample-applications/blob/main/number-guessing-game/contracts/ERC20.sol#L25). + +```solidity +function balanceOf(address tokenOwner) public view override returns (uint256) { + require(tx.origin == tokenOwner || msg.sender == tokenOwner, "Only the token owner can see the balance."); + return balances[tokenOwner]; +} +``` + +_Note that this works because in Obscuro all calls to view functions are authenticated._ \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 5374841c2d..f78efcb44a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -2,6 +2,20 @@

-Welcome to Obscuro. Obscuro hyper-scales and encrypts Ethereum. No SDKs, 100% EVM. +Welcome to Obscuro - the first Ethereum L2 with private smart contract state. -On this docsite you will find useful guidance on Obscuro, how to participate in the Testnet and, if you want to go deeper, read the whitepaper using the menu above. A PDF version of the whitepaper is [available](https://whitepaper.obscu.ro/assets/images/obscuro-whitepaper-0-10-0.pdf). +Obscuro hyper-scales and encrypts Ethereum. 100% EVM, 100% Solidity. + +On this docsite you will find useful guidance on Obscuro, how to participate in the Testnet and, if you want to go deeper, read the whitepaper using the menu above. + +The Litepaper is available to view [here](https://obscu.ro/litepaper). + +A PDF version of the whitepaper is available [here](https://whitepaper.obscu.ro/assets/images/obscuro-whitepaper-0-10-0.pdf). + + +## Useful Resources + +1. [Github](https://github.com/obscuronet/go-obscuro) +2. [Twitter](https://twitter.com/obscuronet/) +3. [Discord](https://discord.gg/7pkKv2Tyfn) +4. [Blog](https://medium.com/obscuro-labs) From d5c19310c1772aa116ec02d9c0601ec347d75c83 Mon Sep 17 00:00:00 2001 From: Pedro Gomes Date: Mon, 25 Sep 2023 11:34:15 +0100 Subject: [PATCH 2/3] fix (#1549) * fix * fix * fix fix * added loose fixes --- ...manual-deploy-obscuro-gateway-database.yml | 4 +- integration/manualtests/connection_test.go | 129 ++++++++++++++++++ tools/walletextension/userconn/user_conn.go | 10 +- 3 files changed, 137 insertions(+), 6 deletions(-) create mode 100644 integration/manualtests/connection_test.go diff --git a/.github/workflows/manual-deploy-obscuro-gateway-database.yml b/.github/workflows/manual-deploy-obscuro-gateway-database.yml index a2c7b566bb..8ab3b8ebaa 100644 --- a/.github/workflows/manual-deploy-obscuro-gateway-database.yml +++ b/.github/workflows/manual-deploy-obscuro-gateway-database.yml @@ -117,4 +117,6 @@ jobs: -e MARIADB_USER=obscurouser \ -e MARIADB_PASSWORD=${{ secrets.OBSCURO_GATEWAY_MARIADB_USER_PWD }} \ -v /home/obscuro/go-obscuro/tools/walletextension/storage/database/001_init.sql:/docker-entrypoint-initdb.d/schema.sql \ - mariadb:11.1.2-jammy' + mariadb:11.1.2-jammy \ + --max_password_errors=2' + diff --git a/integration/manualtests/connection_test.go b/integration/manualtests/connection_test.go new file mode 100644 index 0000000000..d7ba3d5f6a --- /dev/null +++ b/integration/manualtests/connection_test.go @@ -0,0 +1,129 @@ +package manualtests + +import ( + "context" + "encoding/hex" + "fmt" + "io" + "log" + "net/http" + "strings" + "testing" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/obscuronet/go-obscuro/tools/walletextension/common" + "github.com/stretchr/testify/require" + "github.com/valyala/fasthttp" + + gethcommon "github.com/ethereum/go-ethereum/common" +) + +func TestSubscribeToOG(t *testing.T) { + t.Skip("skip manual tests") + + // Using http + ogHTTPAddress := "https://dev-testnet.obscu.ro:443" + ogWSAddress := "wss://dev-testnet.obscu.ro:81" + // ogWSAddress := "ws://51.132.131.47:81" + + // join the network + statusCode, userID, err := fasthttp.Get(nil, fmt.Sprintf("%s/v1/join/", ogHTTPAddress)) + require.NoError(t, err) // dialing to the given TCP address timed out + fmt.Println(statusCode) + fmt.Println(userID) + + // sign the message + messagePayload := signMessage(string(userID)) + + // register an account + var regAccountResp []byte + regAccountResp, err = registerAccount(ogHTTPAddress, string(userID), messagePayload) + require.NoError(t, err) + fmt.Println(string(regAccountResp)) + fmt.Println(hex.EncodeToString(regAccountResp)) + + // Using WS -> + + // Connect to WebSocket server using the standard geth client + client, err := ethclient.Dial(ogWSAddress) + require.NoError(t, err) + + // Create a simple request + at, err := client.BalanceAt(context.Background(), l2Wallet.Address(), nil) + require.NoError(t, err) + + fmt.Println("Balance for account ", l2Wallet.Address().Hex(), " - ", at.String()) + + // Create a subscription + query := ethereum.FilterQuery{ + Addresses: []gethcommon.Address{l2Wallet.Address()}, + } + + logs := make(chan types.Log) + sub, err := client.SubscribeFilterLogs(context.Background(), query, logs) + if err != nil { + log.Fatalf("Failed to subscribe: %v", err) + } + defer sub.Unsubscribe() + + // Listen for events from the contract + for { + select { + case err := <-sub.Err(): + log.Fatalf("Subscription error: %v", err) + case vLog := <-logs: + // Process the contract event + // This is just a simple example printing the block number; you'll want to decode and handle the logs according to your contract's ABI + log.Printf("Received log in block number: %v", vLog.BlockNumber) + } + } +} + +func registerAccount(baseAddress, userID, payload string) ([]byte, error) { + req, err := http.NewRequestWithContext( + context.Background(), + http.MethodPost, + baseAddress+"/v1/authenticate/?u="+userID, + strings.NewReader(payload), + ) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json; charset=UTF-8") + + client := &http.Client{} + response, err := client.Do(req) + if err != nil { + return nil, err + } + + defer response.Body.Close() + return io.ReadAll(response.Body) +} + +// { +// "signature": "0xc784adea83ed3ec60528f4747418c85abe553b35a47fd2c95425de654bb9d0d40ede24aec182e6a2ec65c0c7c6aedab7823f21a9b9f7ff5db3a77a9f90dc97b41c", +// "message": "Register e097c4a10d4285d13b377985834b4c57e069b5856cc6c2cd4a038f62da4bc459 for 0x06ed49a32fcc5094abee51a4ffd46dd23b62a191" +// } +func signMessage(userID string) string { + pk := l2Wallet.PrivateKey() + address := l2Wallet.Address() + hexAddress := address.Hex() + + message := fmt.Sprintf("Register %s for %s", userID, strings.ToLower(hexAddress)) + prefixedMessage := fmt.Sprintf(common.PersonalSignMessagePrefix, len(message), message) + + messageHash := crypto.Keccak256([]byte(prefixedMessage)) + sig, err := crypto.Sign(messageHash, pk) + if err != nil { + log.Fatalf("Failed to sign message: %v", err) + } + sig[64] += 27 + signature := "0x" + hex.EncodeToString(sig) + payload := fmt.Sprintf("{\"signature\": \"%s\", \"message\": \"%s\"}", signature, message) + fmt.Println(payload) + return payload +} diff --git a/tools/walletextension/userconn/user_conn.go b/tools/walletextension/userconn/user_conn.go index b40e0ed28e..e2d2790883 100644 --- a/tools/walletextension/userconn/user_conn.go +++ b/tools/walletextension/userconn/user_conn.go @@ -57,9 +57,9 @@ func NewUserConnWS(resp http.ResponseWriter, req *http.Request, logger gethlog.L // We search all the request's headers. If there's a websocket upgrade header, we upgrade to a websocket connection. conn, err := upgrader.Upgrade(resp, req, nil) if err != nil { - err = fmt.Errorf("unable to upgrade to websocket connection") - logger.Error("unable to upgrade to websocket connection") - httpLogAndSendErr(resp, err.Error()) // todo (@ziga) - Handle error properly for websockets. + err = fmt.Errorf("unable to upgrade to websocket connection - %w", err) + logger.Error("unable to upgrade to websocket connection", log.ErrKey, err) + httpLogAndSendErr(resp, err.Error()) return nil, err } @@ -91,7 +91,7 @@ func (h *userConnHTTP) WriteResponse(msg []byte) error { } func (h *userConnHTTP) HandleError(msg string) { - h.logger.Error(msg) + h.logger.Error(fmt.Sprintf("Handling HTTP user error - %s", msg)) httpLogAndSendErr(h.resp, msg) } @@ -139,7 +139,7 @@ func (w *userConnWS) WriteResponse(msg []byte) error { // HandleError logs and prints the error, and writes it to the websocket as a JSON object with a single key, "error". func (w *userConnWS) HandleError(msg string) { - w.logger.Error(msg) + w.logger.Error(fmt.Sprintf("Handling WS user error - %s", msg)) errMsg, err := json.Marshal(map[string]interface{}{ common.JSONKeyErr: msg, From cc741c33c8204241471403da8db36971671f8251 Mon Sep 17 00:00:00 2001 From: Tudor Malene Date: Mon, 25 Sep 2023 13:23:29 +0100 Subject: [PATCH 3/3] fix rollup limiter & improve "slow query" logging (#1551) * fix rollup limiter. Improve "slow query" logging * add fix for empty rollups * add fix for empty rollups --- go/enclave/components/rollup_compression.go | 2 +- go/enclave/limiters/interfaces.go | 4 +- go/enclave/limiters/rolluplimiter.go | 26 +++++------ go/enclave/storage/storage.go | 19 ++++++-- integration/simulation/validate_chain.go | 48 +++++++++++++-------- 5 files changed, 62 insertions(+), 37 deletions(-) diff --git a/go/enclave/components/rollup_compression.go b/go/enclave/components/rollup_compression.go index 382089ac35..84ab2bca62 100644 --- a/go/enclave/components/rollup_compression.go +++ b/go/enclave/components/rollup_compression.go @@ -161,7 +161,7 @@ func (rc *RollupCompression) createRollupHeader(batches []*core.Batch) (*common. isReorg := false for i, batch := range batches { - rc.logger.Info("Add batch to rollup", log.BatchSeqNoKey, batch.SeqNo(), log.BatchHeightKey, batch.Number(), log.BatchHashKey, batch.Hash()) + rc.logger.Debug("Compressing batch to rollup", log.BatchSeqNoKey, batch.SeqNo(), log.BatchHeightKey, batch.Number(), log.BatchHashKey, batch.Hash()) // determine whether the batch is canonical can, err := rc.storage.FetchBatchByHeight(batch.NumberU64()) if err != nil { diff --git a/go/enclave/limiters/interfaces.go b/go/enclave/limiters/interfaces.go index 0729617814..5a93a2ff17 100644 --- a/go/enclave/limiters/interfaces.go +++ b/go/enclave/limiters/interfaces.go @@ -3,6 +3,8 @@ package limiters import ( "errors" + "github.com/obscuronet/go-obscuro/go/enclave/core" + "github.com/ethereum/go-ethereum/core/types" ) @@ -14,5 +16,5 @@ type BatchSizeLimiter interface { var ErrInsufficientSpace = errors.New("insufficient space in BatchSizeLimiter") type RollupLimiter interface { - AcceptBatch(encodable interface{}) (bool, error) + AcceptBatch(batch *core.Batch) (bool, error) } diff --git a/go/enclave/limiters/rolluplimiter.go b/go/enclave/limiters/rolluplimiter.go index 05800a10b9..36cc10e4b5 100644 --- a/go/enclave/limiters/rolluplimiter.go +++ b/go/enclave/limiters/rolluplimiter.go @@ -1,20 +1,19 @@ package limiters import ( - "errors" "fmt" + "github.com/obscuronet/go-obscuro/go/enclave/core" + "github.com/ethereum/go-ethereum/rlp" ) -var ErrFailedToEncode = errors.New("failed to encode data") - -// MaxTransactionSizeLimiter - configured to be close to what the ethereum clients -// have configured as the maximum size a transaction can have. Note that this isn't -// a protocol limit, but a miner imposed limit and it might be hard to find someone -// to include a transaction if it goes above it -// todo - figure out the best number, optimism uses 132KB -const MaxTransactionSize = 64 * 1024 +const ( + // 85% is a very conservative number. It will most likely be 66% in practice. + // We can lower it, once we have a mechanism in place to handle batches that don't actually compress to that. + txCompressionFactor = 0.85 + compressedHeaderSize = 1 +) type rollupLimiter struct { remainingSize uint64 @@ -27,13 +26,14 @@ func NewRollupLimiter(size uint64) RollupLimiter { } // todo (@stefan) figure out how to optimize the serialization out of the limiter -func (rl *rollupLimiter) AcceptBatch(encodable interface{}) (bool, error) { - encodedData, err := rlp.EncodeToBytes(encodable) +func (rl *rollupLimiter) AcceptBatch(batch *core.Batch) (bool, error) { + encodedData, err := rlp.EncodeToBytes(batch.Transactions) if err != nil { - return false, fmt.Errorf("%w: %s", ErrFailedToEncode, err.Error()) + return false, fmt.Errorf("failed to encode data. Cause: %w", err) } - encodedSize := uint64(len(encodedData)) + // adjust with a compression factor and add the size of a compressed batch header + encodedSize := uint64(float64(len(encodedData))*txCompressionFactor) + compressedHeaderSize if encodedSize > rl.remainingSize { return false, nil } diff --git a/go/enclave/storage/storage.go b/go/enclave/storage/storage.go index 7fe1eaaf8b..e8d627c65b 100644 --- a/go/enclave/storage/storage.go +++ b/go/enclave/storage/storage.go @@ -40,8 +40,11 @@ import ( // todo - this will require a dedicated table when updates are implemented const ( - masterSeedCfg = "MASTER_SEED" - _slowCallThresholdMillis = 50 // requests that take longer than this will be logged + masterSeedCfg = "MASTER_SEED" + _slowCallDebugThresholdMillis = 50 // requests that take longer than this will be logged with DEBUG + _slowCallInfoThresholdMillis = 100 // requests that take longer than this will be logged with INFO + _slowCallWarnThresholdMillis = 200 // requests that take longer than this will be logged with WARN + _slowCallErrorThresholdMillis = 500 // requests that take longer than this will be logged with ERROR ) type storageImpl struct { @@ -551,8 +554,16 @@ func (s *storageImpl) GetPublicTransactionCount() (uint64, error) { func (s *storageImpl) logDuration(method string, callStart time.Time) { durationMillis := time.Since(callStart).Milliseconds() + msg := fmt.Sprintf("Storage::%s completed", method) // we only log 'slow' calls to reduce noise - if durationMillis > _slowCallThresholdMillis { - s.logger.Info(fmt.Sprintf("Storage::%s completed", method), log.DurationMilliKey, durationMillis) + switch { + case durationMillis > _slowCallErrorThresholdMillis: + s.logger.Error(msg, log.DurationMilliKey, durationMillis) + case durationMillis > _slowCallWarnThresholdMillis: + s.logger.Warn(msg, log.DurationMilliKey, durationMillis) + case durationMillis > _slowCallInfoThresholdMillis: + s.logger.Info(msg, log.DurationMilliKey, durationMillis) + case durationMillis > _slowCallDebugThresholdMillis: + s.logger.Debug(msg, log.DurationMilliKey, durationMillis) } } diff --git a/integration/simulation/validate_chain.go b/integration/simulation/validate_chain.go index 8621b341c2..c2f3ab5d67 100644 --- a/integration/simulation/validate_chain.go +++ b/integration/simulation/validate_chain.go @@ -127,32 +127,44 @@ func checkObscuroBlockchainValidity(t *testing.T, s *Simulation, maxL1Height uin } } -func checkCollectedL1Fees(t *testing.T, node ethadapter.EthClient, s *Simulation, nodeIdx int, rollupReceipts types.Receipts) { - costOfRollups := big.NewInt(0) +// the cost of an empty rollup - adjust if the management contract changes. This is the rollup overhead. +const emptyRollupGas = 110_000 - if !s.Params.IsInMem { - for _, receipt := range rollupReceipts { - block, err := node.EthClient().BlockByHash(context.Background(), receipt.BlockHash) - if err != nil { - panic(err) - } +func checkCollectedL1Fees(t *testing.T, node ethadapter.EthClient, s *Simulation, nodeIdx int, rollupReceipts types.Receipts) { + costOfRollupsWithTransactions := big.NewInt(0) + costOfEmptyRollups := big.NewInt(0) - txCost := big.NewInt(0).Mul(block.BaseFee(), big.NewInt(0).SetUint64(receipt.GasUsed)) - costOfRollups.Add(costOfRollups, txCost) - } + if s.Params.IsInMem { + // not supported for in memory tests + return + } - l2FeesWallet := s.Params.Wallets.L2FeesWallet - obsClients := network.CreateAuthClients(s.RPCHandles.RPCClients, l2FeesWallet) - feeBalance, err := obsClients[nodeIdx].BalanceAt(context.Background(), nil) + for _, receipt := range rollupReceipts { + block, err := node.EthClient().BlockByHash(context.Background(), receipt.BlockHash) if err != nil { - panic(fmt.Errorf("failed getting balance for bridge transfer receiver. Cause: %w", err)) + panic(err) } - // if balance of collected fees is less than cost of published rollups fail - if feeBalance.Cmp(costOfRollups) == -1 { - t.Errorf("Node %d: Sequencer has collected insufficient fees. Has: %d, needs: %d", nodeIdx, feeBalance, costOfRollups) + txCost := big.NewInt(0).Mul(block.BaseFee(), big.NewInt(0).SetUint64(receipt.GasUsed)) + // only calculate the fees collected for non-empty rollups, because the empty ones are subsidized + if receipt.GasUsed > emptyRollupGas { + costOfRollupsWithTransactions.Add(costOfRollupsWithTransactions, txCost) + } else { + costOfEmptyRollups.Add(costOfEmptyRollups, txCost) } } + + l2FeesWallet := s.Params.Wallets.L2FeesWallet + obsClients := network.CreateAuthClients(s.RPCHandles.RPCClients, l2FeesWallet) + feeBalance, err := obsClients[nodeIdx].BalanceAt(context.Background(), nil) + if err != nil { + panic(fmt.Errorf("failed getting balance for bridge transfer receiver. Cause: %w", err)) + } + + // if balance of collected fees is less than cost of published rollups fail + if feeBalance.Cmp(costOfRollupsWithTransactions) == -1 { + t.Errorf("Node %d: Sequencer has collected insufficient fees. Has: %d, needs: %d", nodeIdx, feeBalance, costOfRollupsWithTransactions) + } } func checkBlockchainOfEthereumNode(t *testing.T, node ethadapter.EthClient, minHeight uint64, s *Simulation, nodeIdx int) uint64 {