Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor and parallelize integration tests #216

Merged
merged 15 commits into from
Jun 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/soroban-rpc.yml
Original file line number Diff line number Diff line change
Expand Up @@ -183,4 +183,4 @@ jobs:

- name: Run Soroban RPC Integration Tests
run: |
go test -race -timeout 60m -v ./cmd/soroban-rpc/internal/test/...
go test -race -timeout 20m ./cmd/soroban-rpc/internal/integrationtest/...
39 changes: 31 additions & 8 deletions cmd/soroban-rpc/internal/daemon/daemon.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package daemon
import (
"context"
"errors"
"net"
"net/http"
"net/http/pprof" //nolint:gosec
"os"
Expand Down Expand Up @@ -50,7 +51,9 @@ type Daemon struct {
jsonRPCHandler *internal.Handler
logger *supportlog.Entry
preflightWorkerPool *preflight.PreflightWorkerPool
listener net.Listener
server *http.Server
adminListener net.Listener
adminServer *http.Server
closeOnce sync.Once
closeError error
Expand All @@ -62,6 +65,15 @@ func (d *Daemon) GetDB() *db.DB {
return d.db
}

func (d *Daemon) GetEndpointAddrs() (net.TCPAddr, *net.TCPAddr) {
var addr = d.listener.Addr().(*net.TCPAddr)
var adminAddr *net.TCPAddr
if d.adminListener != nil {
adminAddr = d.adminListener.Addr().(*net.TCPAddr)
}
return *addr, adminAddr
}

func (d *Daemon) close() {
shutdownCtx, shutdownRelease := context.WithTimeout(context.Background(), defaultShutdownGracePeriod)
defer shutdownRelease()
Expand Down Expand Up @@ -133,8 +145,7 @@ func newCaptiveCore(cfg *config.Config, logger *supportlog.Entry) (*ledgerbacken

}

func MustNew(cfg *config.Config) *Daemon {
logger := supportlog.New()
func MustNew(cfg *config.Config, logger *supportlog.Entry) *Daemon {
logger.SetLevel(cfg.LogLevel)
if cfg.LogFormat == config.LogFormatJSON {
logger.UseJSONFormatter()
Expand All @@ -157,6 +168,7 @@ func MustNew(cfg *config.Config) *Daemon {
historyArchive, err := historyarchive.NewArchivePool(
cfg.HistoryArchiveURLs,
historyarchive.ArchiveOptions{
Logger: logger,
NetworkPassphrase: cfg.NetworkPassphrase,
CheckpointFrequency: cfg.CheckpointFrequency,
ConnectOptions: storage.ConnectOptions{
Expand Down Expand Up @@ -251,8 +263,13 @@ func MustNew(cfg *config.Config) *Daemon {
daemon.ingestService = ingestService
daemon.jsonRPCHandler = &jsonRPCHandler

// Use a separate listener in order to obtain the actual TCP port
// when using dynamic ports during testing (e.g. endpoint="localhost:0")
daemon.listener, err = net.Listen("tcp", cfg.Endpoint)
if err != nil {
daemon.logger.WithError(err).WithField("endpoint", cfg.Endpoint).Fatal("cannot listen on endpoint")
}
daemon.server = &http.Server{
Addr: cfg.Endpoint,
Handler: httpHandler,
ReadTimeout: defaultReadTimeout,
}
Expand All @@ -269,7 +286,11 @@ func MustNew(cfg *config.Config) *Daemon {
adminMux.Handle("/debug/pprof/"+profile.Name(), pprof.Handler(profile.Name()))
}
adminMux.Handle("/metrics", promhttp.HandlerFor(metricsRegistry, promhttp.HandlerOpts{}))
daemon.adminServer = &http.Server{Addr: cfg.AdminEndpoint, Handler: adminMux}
daemon.adminListener, err = net.Listen("tcp", cfg.AdminEndpoint)
if err != nil {
daemon.logger.WithError(err).WithField("endpoint", cfg.Endpoint).Fatal("cannot listen on admin endpoint")
}
daemon.adminServer = &http.Server{Handler: adminMux}
}
daemon.registerMetrics()
return daemon
Expand Down Expand Up @@ -340,20 +361,22 @@ func (d *Daemon) mustInitializeStorage(cfg *config.Config) (*feewindow.FeeWindow

func (d *Daemon) Run() {
d.logger.WithFields(supportlog.F{
"addr": d.server.Addr,
"addr": d.listener.Addr().String(),
}).Info("starting HTTP server")

panicGroup := util.UnrecoverablePanicGroup.Log(d.logger)
panicGroup.Go(func() {
if err := d.server.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
// Error starting or closing listener:
if err := d.server.Serve(d.listener); !errors.Is(err, http.ErrServerClosed) {
d.logger.WithError(err).Fatal("soroban JSON RPC server encountered fatal error")
}
})

if d.adminServer != nil {
d.logger.WithFields(supportlog.F{
"addr": d.adminListener.Addr().String(),
}).Info("starting Admin HTTP server")
panicGroup.Go(func() {
if err := d.adminServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
if err := d.adminServer.Serve(d.adminListener); !errors.Is(err, http.ErrServerClosed) {
d.logger.WithError(err).Error("soroban admin server encountered fatal error")
}
})
Expand Down
58 changes: 58 additions & 0 deletions cmd/soroban-rpc/internal/integrationtest/archive_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
package integrationtest

import (
"net"
"net/http"
"net/http/httptest"
"sync"
"testing"
"time"

"github.com/stretchr/testify/require"

"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/integrationtest/infrastructure"
)

func TestArchiveUserAgent(t *testing.T) {
userAgents := sync.Map{}
historyArchive := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
agent := r.Header["User-Agent"][0]
t.Log("agent", agent)
userAgents.Store(agent, "")
if r.URL.Path == "/.well-known/stellar-history.json" || r.URL.Path == "/history/00/00/00/history-0000001f.json" {
w.Write([]byte(`{
"version": 1,
"server": "stellar-core 21.0.1 (dfd3dbff1d9cad4dc31e022de6ac2db731b4b326)",
"currentLedger": 31,
"networkPassphrase": "Standalone Network ; February 2017",
"currentBuckets": []
}`))
return
}
// emulate a problem with the archive
w.WriteHeader(http.StatusInternalServerError)
}))
defer historyArchive.Close()
historyPort := historyArchive.Listener.Addr().(*net.TCPAddr).Port

cfg := &infrastructure.TestConfig{
OnlyRPC: &infrastructure.TestOnlyRPCConfig{
CorePorts: infrastructure.TestCorePorts{
CoreArchivePort: uint16(historyPort),
},
DontWait: true,
},
}

infrastructure.NewTest(t, cfg)

require.Eventually(t,
func() bool {
_, ok1 := userAgents.Load("soroban-rpc/0.0.0")
_, ok2 := userAgents.Load("soroban-rpc/0.0.0/captivecore")
return ok1 && ok2
},
5*time.Second,
time.Second,
)
}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package test
package integrationtest

import (
"bytes"
Expand All @@ -7,15 +7,17 @@ import (
"testing"

"github.com/stretchr/testify/require"

"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/integrationtest/infrastructure"
)

// TestCORS ensures that we receive the correct CORS headers as a response to an HTTP request.
// Specifically, when we include an Origin header in the request, a soroban-rpc should response
// with a corresponding Access-Control-Allow-Origin.
func TestCORS(t *testing.T) {
test := NewTest(t, nil)
test := infrastructure.NewTest(t, nil)

request, err := http.NewRequest("POST", test.sorobanRPCURL(), bytes.NewBufferString("{\"jsonrpc\": \"2.0\", \"id\": 1, \"method\": \"getHealth\"}"))
request, err := http.NewRequest("POST", test.GetSorobanRPCURL(), bytes.NewBufferString("{\"jsonrpc\": \"2.0\", \"id\": 1, \"method\": \"getHealth\"}"))
require.NoError(t, err)
request.Header.Set("Content-Type", "application/json")
origin := "testorigin.com"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,42 +1,22 @@
package test
package integrationtest

import (
"context"
"testing"

"github.com/stellar/go/keypair"
"github.com/stellar/go/txnbuild"
"github.com/stellar/go/xdr"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/integrationtest/infrastructure"
"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/methods"
)

func TestGetFeeStats(t *testing.T) {
test := NewTest(t, nil)
test := infrastructure.NewTest(t, nil)

client := test.GetRPCLient()
sourceAccount := keypair.Root(StandaloneNetworkPassphrase)
address := sourceAccount.Address()
account := txnbuild.NewSimpleAccount(address, 0)

// Submit soroban transaction
contractBinary := getHelloWorldContract(t)
params := preflightTransactionParams(t, client, txnbuild.TransactionParams{
SourceAccount: &account,
IncrementSequenceNum: true,
Operations: []txnbuild.Operation{
createInstallContractCodeOperation(account.AccountID, contractBinary),
},
BaseFee: txnbuild.MinBaseFee,
Preconditions: txnbuild.Preconditions{
TimeBounds: txnbuild.NewInfiniteTimeout(),
},
})
tx, err := txnbuild.NewTransaction(params)
assert.NoError(t, err)
sorobanTxResponse := sendSuccessfulTransaction(t, client, sourceAccount, tx)
sorobanTxResponse, _ := test.UploadHelloWorldContract()
var sorobanTxResult xdr.TransactionResult
require.NoError(t, xdr.SafeUnmarshalBase64(sorobanTxResponse.ResultXdr, &sorobanTxResult))
sorobanTotalFee := sorobanTxResult.FeeCharged
Expand All @@ -46,28 +26,18 @@ func TestGetFeeStats(t *testing.T) {
sorobanResourceFeeCharged := sorobanFees.TotalRefundableResourceFeeCharged + sorobanFees.TotalNonRefundableResourceFeeCharged
sorobanInclusionFee := uint64(sorobanTotalFee - sorobanResourceFeeCharged)

seq, err := test.MasterAccount().GetSequenceNumber()
require.NoError(t, err)
// Submit classic transaction
params = txnbuild.TransactionParams{
SourceAccount: &account,
IncrementSequenceNum: true,
Operations: []txnbuild.Operation{
&txnbuild.BumpSequence{BumpTo: account.Sequence + 100},
},
BaseFee: txnbuild.MinBaseFee,
Memo: nil,
Preconditions: txnbuild.Preconditions{
TimeBounds: txnbuild.NewInfiniteTimeout(),
},
}
tx, err = txnbuild.NewTransaction(params)
assert.NoError(t, err)
classicTxResponse := sendSuccessfulTransaction(t, client, sourceAccount, tx)
classicTxResponse := test.SendMasterOperation(
&txnbuild.BumpSequence{BumpTo: seq + 100},
)
var classicTxResult xdr.TransactionResult
require.NoError(t, xdr.SafeUnmarshalBase64(classicTxResponse.ResultXdr, &classicTxResult))
classicFee := uint64(classicTxResult.FeeCharged)

var result methods.GetFeeStatsResult
if err := client.CallResult(context.Background(), "getFeeStats", nil, &result); err != nil {
if err := test.GetRPCLient().CallResult(context.Background(), "getFeeStats", nil, &result); err != nil {
t.Fatalf("rpc call failed: %v", err)
}
expectedResult := methods.GetFeeStatsResult{
Expand Down
Original file line number Diff line number Diff line change
@@ -1,35 +1,30 @@
package test
package integrationtest

import (
"context"
"crypto/sha256"
"testing"

"github.com/creachadair/jrpc2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/stellar/go/keypair"
"github.com/stellar/go/txnbuild"
"github.com/stellar/go/xdr"

"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/integrationtest/infrastructure"
"github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/methods"
)

func TestGetLedgerEntriesNotFound(t *testing.T) {
test := NewTest(t, nil)

test := infrastructure.NewTest(t, nil)
client := test.GetRPCLient()

sourceAccount := keypair.Root(StandaloneNetworkPassphrase).Address()
contractID := getContractID(t, sourceAccount, testSalt, StandaloneNetworkPassphrase)
contractIDHash := xdr.Hash(contractID)
hash := xdr.Hash{0xa, 0xb}
keyB64, err := xdr.MarshalBase64(xdr.LedgerKey{
Type: xdr.LedgerEntryTypeContractData,
ContractData: &xdr.LedgerKeyContractData{
Contract: xdr.ScAddress{
Type: xdr.ScAddressTypeScAddressTypeContract,
ContractId: &contractIDHash,
ContractId: &hash,
},
Key: xdr.ScVal{
Type: xdr.ScValTypeScvLedgerKeyContractInstance,
Expand All @@ -54,7 +49,7 @@ func TestGetLedgerEntriesNotFound(t *testing.T) {
}

func TestGetLedgerEntriesInvalidParams(t *testing.T) {
test := NewTest(t, nil)
test := infrastructure.NewTest(t, nil)

client := test.GetRPCLient()

Expand All @@ -71,48 +66,9 @@ func TestGetLedgerEntriesInvalidParams(t *testing.T) {
}

func TestGetLedgerEntriesSucceeds(t *testing.T) {
test := NewTest(t, nil)

client := test.GetRPCLient()

sourceAccount := keypair.Root(StandaloneNetworkPassphrase)
address := sourceAccount.Address()
account := txnbuild.NewSimpleAccount(address, 0)
test := infrastructure.NewTest(t, nil)
_, contractID, contractHash := test.CreateHelloWorldContract()

contractBinary := getHelloWorldContract(t)
params := preflightTransactionParams(t, client, txnbuild.TransactionParams{
SourceAccount: &account,
IncrementSequenceNum: true,
Operations: []txnbuild.Operation{
createInstallContractCodeOperation(account.AccountID, contractBinary),
},
BaseFee: txnbuild.MinBaseFee,
Preconditions: txnbuild.Preconditions{
TimeBounds: txnbuild.NewInfiniteTimeout(),
},
})
tx, err := txnbuild.NewTransaction(params)
assert.NoError(t, err)
sendSuccessfulTransaction(t, client, sourceAccount, tx)

params = preflightTransactionParams(t, client, txnbuild.TransactionParams{
SourceAccount: &account,
IncrementSequenceNum: true,
Operations: []txnbuild.Operation{
createCreateContractOperation(address, contractBinary),
},
BaseFee: txnbuild.MinBaseFee,
Preconditions: txnbuild.Preconditions{
TimeBounds: txnbuild.NewInfiniteTimeout(),
},
})
tx, err = txnbuild.NewTransaction(params)
assert.NoError(t, err)
sendSuccessfulTransaction(t, client, sourceAccount, tx)

contractID := getContractID(t, address, testSalt, StandaloneNetworkPassphrase)

contractHash := sha256.Sum256(contractBinary)
contractCodeKeyB64, err := xdr.MarshalBase64(xdr.LedgerKey{
Type: xdr.LedgerEntryTypeContractCode,
ContractCode: &xdr.LedgerKeyContractCode{
Expand Down Expand Up @@ -146,7 +102,7 @@ func TestGetLedgerEntriesSucceeds(t *testing.T) {
}

var result methods.GetLedgerEntriesResponse
err = client.CallResult(context.Background(), "getLedgerEntries", request, &result)
err = test.GetRPCLient().CallResult(context.Background(), "getLedgerEntries", request, &result)
require.NoError(t, err)
require.Equal(t, 2, len(result.Entries))
require.Greater(t, result.LatestLedger, uint32(0))
Expand All @@ -159,7 +115,7 @@ func TestGetLedgerEntriesSucceeds(t *testing.T) {
var firstEntry xdr.LedgerEntryData
require.NoError(t, xdr.SafeUnmarshalBase64(result.Entries[0].XDR, &firstEntry))
require.Equal(t, xdr.LedgerEntryTypeContractCode, firstEntry.Type)
require.Equal(t, contractBinary, firstEntry.MustContractCode().Code)
require.Equal(t, infrastructure.GetHelloWorldContract(), firstEntry.MustContractCode().Code)

require.Greater(t, result.Entries[1].LastModifiedLedger, uint32(0))
require.LessOrEqual(t, result.Entries[1].LastModifiedLedger, result.LatestLedger)
Expand Down
Loading
Loading