-
Notifications
You must be signed in to change notification settings - Fork 39
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add New Node Flow fixes #1712
Add New Node Flow fixes #1712
Changes from 16 commits
29140ba
4720e32
bda3bcf
60a45ee
e79e913
3792c75
7aaafdd
fdb8c05
57dcce5
a58c598
d12a540
30f5fd8
3bde2ac
6f9c00e
5d1fd19
96003ff
b6677bf
cbde85f
db662be
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change | ||||||||||||||||||
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
@@ -23,18 +23,14 @@ on: | |||||||||||||||||||
required: true | ||||||||||||||||||||
default: 3 | ||||||||||||||||||||
type: number | ||||||||||||||||||||
node_private_key: | ||||||||||||||||||||
description: 'Node Private Key' | ||||||||||||||||||||
node_id: | ||||||||||||||||||||
description: 'Node id' | ||||||||||||||||||||
required: true | ||||||||||||||||||||
type: string | ||||||||||||||||||||
node_account_address: | ||||||||||||||||||||
description: 'Node Account Address' | ||||||||||||||||||||
required: true | ||||||||||||||||||||
type: string | ||||||||||||||||||||
node_l1_ws_url: | ||||||||||||||||||||
description: 'Node L1 Connection String' | ||||||||||||||||||||
required: true | ||||||||||||||||||||
type: string | ||||||||||||||||||||
MGMT_CONTRACT_ADDR: | ||||||||||||||||||||
description: 'Management Contract Addr' | ||||||||||||||||||||
required: true | ||||||||||||||||||||
|
@@ -47,14 +43,7 @@ on: | |||||||||||||||||||
description: 'L1 Starting Hash' | ||||||||||||||||||||
required: true | ||||||||||||||||||||
type: string | ||||||||||||||||||||
HOC_ERC20_ADDR: | ||||||||||||||||||||
description: 'HOC ERC20 Contract Addr' | ||||||||||||||||||||
required: true | ||||||||||||||||||||
type: string | ||||||||||||||||||||
POC_ERC20_ADDR: | ||||||||||||||||||||
description: 'POC ERC20 Contract Addr' | ||||||||||||||||||||
required: true | ||||||||||||||||||||
type: string | ||||||||||||||||||||
|
||||||||||||||||||||
|
||||||||||||||||||||
jobs: | ||||||||||||||||||||
build: | ||||||||||||||||||||
|
@@ -115,34 +104,34 @@ jobs: | |||||||||||||||||||
with: | ||||||||||||||||||||
creds: ${{ secrets.AZURE_CREDENTIALS }} | ||||||||||||||||||||
|
||||||||||||||||||||
- name: 'Create VM for Obscuro node-${{ matrix.host_id }} on Azure' | ||||||||||||||||||||
- name: 'Create VM for Obscuro node-${{ github.event.inputs.node_id }} on Azure' | ||||||||||||||||||||
uses: azure/CLI@v1 | ||||||||||||||||||||
with: | ||||||||||||||||||||
inlineScript: | | ||||||||||||||||||||
az vm create -g Testnet -n "${{ vars.AZURE_RESOURCE_PREFIX }}-${{ matrix.host_id }}-${{ GITHUB.RUN_NUMBER }}" \ | ||||||||||||||||||||
az vm create -g Testnet -n "${{ vars.AZURE_RESOURCE_PREFIX }}-${{ github.event.inputs.node_id }}-${{ GITHUB.RUN_NUMBER }}" \ | ||||||||||||||||||||
--admin-username obscurouser --admin-password "${{ secrets.OBSCURO_NODE_VM_PWD }}" \ | ||||||||||||||||||||
--public-ip-address-dns-name "obscuronode-${{ matrix.host_id }}-${{ github.event.inputs.testnet_type }}-${{ GITHUB.RUN_NUMBER }}" \ | ||||||||||||||||||||
--public-ip-address-dns-name "obscuronode-${{ github.event.inputs.node_id }}-${{ github.event.inputs.testnet_type }}-${{ GITHUB.RUN_NUMBER }}" \ | ||||||||||||||||||||
--tags deploygroup=ObscuroNode-${{ github.event.inputs.testnet_type }}-${{ GITHUB.RUN_NUMBER }} ${{ vars.AZURE_DEPLOY_GROUP_L2 }}=true \ | ||||||||||||||||||||
--vnet-name ${{ github.event.inputs.testnet_type }}-virtual-network --subnet ${{ github.event.inputs.testnet_type }}-sub-network \ | ||||||||||||||||||||
--size Standard_DC8_v2 --storage-sku StandardSSD_LRS --image ObscuroConfUbuntu \ | ||||||||||||||||||||
--public-ip-sku Standard --authentication-type password | ||||||||||||||||||||
|
||||||||||||||||||||
- name: 'Open Obscuro node-${{ matrix.host_id }} ports on Azure' | ||||||||||||||||||||
- name: 'Open Obscuro node-${{ github.event.inputs.host_id }} ports on Azure' | ||||||||||||||||||||
uses: azure/CLI@v1 | ||||||||||||||||||||
with: | ||||||||||||||||||||
inlineScript: | | ||||||||||||||||||||
az vm open-port -g Testnet -n "${{ vars.AZURE_RESOURCE_PREFIX }}-${{ matrix.host_id }}-${{ GITHUB.RUN_NUMBER }}" --port 80,81,6060,6061,10000 | ||||||||||||||||||||
az vm open-port -g Testnet -n "${{ vars.AZURE_RESOURCE_PREFIX }}-${{ github.event.inputs.node_id }}-${{ GITHUB.RUN_NUMBER }}" --port 80,81,6060,6061,10000 | ||||||||||||||||||||
|
||||||||||||||||||||
# To overcome issues with critical VM resources being unavailable, we need to wait for the VM to be ready | ||||||||||||||||||||
- name: 'Allow time for VM initialization' | ||||||||||||||||||||
shell: bash | ||||||||||||||||||||
run: sleep 60 | ||||||||||||||||||||
Comment on lines
141
to
143
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Using - sleep 60
+ while ! nc -z $VM_IP $VM_PORT; do
+ sleep 1 # wait for 1 second before check again
+ done Committable suggestion
Suggested change
|
||||||||||||||||||||
|
||||||||||||||||||||
- name: 'Start Obscuro node-${{ matrix.host_id }} on Azure' | ||||||||||||||||||||
- name: 'Start Obscuro node-${{ github.event.inputs.node_id }} on Azure' | ||||||||||||||||||||
uses: azure/CLI@v1 | ||||||||||||||||||||
with: | ||||||||||||||||||||
inlineScript: | | ||||||||||||||||||||
az vm run-command invoke -g Testnet -n "${{ vars.AZURE_RESOURCE_PREFIX }}-${{ matrix.host_id }}-${{ GITHUB.RUN_NUMBER }}" \ | ||||||||||||||||||||
az vm run-command invoke -g Testnet -n "${{ vars.AZURE_RESOURCE_PREFIX }}-${{ github.event.inputs.node_id }}-${{ GITHUB.RUN_NUMBER }}" \ | ||||||||||||||||||||
--command-id RunShellScript \ | ||||||||||||||||||||
--scripts 'mkdir -p /home/obscuro \ | ||||||||||||||||||||
&& git clone --depth 1 -b ${{ env.BRANCH_NAME }} https://github.com/ten-protocol/go-ten.git /home/obscuro/go-obscuro \ | ||||||||||||||||||||
|
@@ -166,13 +155,13 @@ jobs: | |||||||||||||||||||
-node_type=validator \ | ||||||||||||||||||||
-is_sgx_enabled=true \ | ||||||||||||||||||||
-host_id=${{ github.event.inputs.node_account_address }} \ | ||||||||||||||||||||
-l1_ws_url=${{ github.event.inputs.node_l1_ws_url }} \ | ||||||||||||||||||||
-l1_ws_url=${{ secrets.ADD_NEW_NODE_L1_WS_URL }} \ | ||||||||||||||||||||
-management_contract_addr=${{ github.event.inputs.MGMT_CONTRACT_ADDR }} \ | ||||||||||||||||||||
-message_bus_contract_addr=${{ github.event.inputs.MSG_BUS_CONTRACT_ADDR }} \ | ||||||||||||||||||||
-l1_start=${{ github.event.inputs.L1_START_HASH }} \ | ||||||||||||||||||||
-private_key=${{ github.event.inputs.node_private_key }} \ | ||||||||||||||||||||
-private_key=${{ secrets.ADD_NEW_NODE_PRIVATE_KEY }} \ | ||||||||||||||||||||
-sequencer_id=${{ vars.ACCOUNT_ADDR_NODE_0 }} \ | ||||||||||||||||||||
-host_public_p2p_addr=obscuronode-${{ matrix.host_id }}-${{ github.event.inputs.testnet_type }}-${{ GITHUB.RUN_NUMBER }}.uksouth.cloudapp.azure.com:10000 \ | ||||||||||||||||||||
-host_public_p2p_addr=obscuronode-${{ github.event.inputs.node_id }}-${{ github.event.inputs.testnet_type }}-${{ GITHUB.RUN_NUMBER }}.uksouth.cloudapp.azure.com:10000 \ | ||||||||||||||||||||
-host_p2p_port=10000 \ | ||||||||||||||||||||
-enclave_docker_image=${{ vars.L2_ENCLAVE_DOCKER_BUILD_TAG }} \ | ||||||||||||||||||||
-host_docker_image=${{ vars.L2_HOST_DOCKER_BUILD_TAG }} \ | ||||||||||||||||||||
|
@@ -209,8 +198,8 @@ jobs: | |||||||||||||||||||
inlineScript: | | ||||||||||||||||||||
az network nic ip-config address-pool add \ | ||||||||||||||||||||
--address-pool ${{ github.event.inputs.testnet_type }}-backend-pool \ | ||||||||||||||||||||
--ip-config-name ipconfig${{ vars.AZURE_RESOURCE_PREFIX }}-1-${{ GITHUB.RUN_NUMBER }} \ | ||||||||||||||||||||
--nic-name ${{ vars.AZURE_RESOURCE_PREFIX }}-1-${{ GITHUB.RUN_NUMBER }}VMNic \ | ||||||||||||||||||||
--ip-config-name ipconfig${{ vars.AZURE_RESOURCE_PREFIX }}-${{ github.event.inputs.node_id }}-${{ GITHUB.RUN_NUMBER }} \ | ||||||||||||||||||||
--nic-name ${{ vars.AZURE_RESOURCE_PREFIX }}-${{ github.event.inputs.node_id }}-${{ GITHUB.RUN_NUMBER }}VMNic \ | ||||||||||||||||||||
--resource-group Testnet \ | ||||||||||||||||||||
--lb-name ${{ github.event.inputs.testnet_type }}-loadbalancer | ||||||||||||||||||||
|
||||||||||||||||||||
|
@@ -227,5 +216,4 @@ jobs: | |||||||||||||||||||
- name: "Wait until obscuro node is healthy" | ||||||||||||||||||||
shell: bash | ||||||||||||||||||||
run: | | ||||||||||||||||||||
./.github/workflows/runner-scripts/wait-node-healthy.sh --host=obscuronode-0-${{ github.event.inputs.testnet_type }}-${{ GITHUB.RUN_NUMBER }}.uksouth.cloudapp.azure.com | ||||||||||||||||||||
./.github/workflows/runner-scripts/wait-node-healthy.sh --host=obscuronode-1-${{ github.event.inputs.testnet_type }}-${{ GITHUB.RUN_NUMBER }}.uksouth.cloudapp.azure.com | ||||||||||||||||||||
./.github/workflows/runner-scripts/wait-node-healthy.sh --host=obscuronode-${{ github.event.inputs.node_id }}-${{ github.event.inputs.testnet_type }}-${{ GITHUB.RUN_NUMBER }}.uksouth.cloudapp.azure.com |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -43,12 +43,12 @@ if [[ -z "${ipconfig_id}" ]]; then | |
exit 0 | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Consider using a non-zero exit status when the script exits due to not finding the |
||
fi | ||
|
||
az network nic ip-config address-pool remove \ | ||
--address-pool ${pool} \ | ||
--ip-config-name "${ipconfig_id}" \ | ||
--nic-name "${nic_id}" \ | ||
--resource-group Testnet \ | ||
--lb-name ${lb} \ | ||
#az network nic ip-config address-pool remove \ | ||
# --address-pool ${pool} \ | ||
# --ip-config-name "${ipconfig_id}" \ | ||
# --nic-name "${nic_id}" \ | ||
# --resource-group Testnet \ | ||
# --lb-name ${lb} \ | ||
Comment on lines
+46
to
+51
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The block of code that removes an address pool from the network interface configuration has been commented out. If this is intentional to prevent the removal for now, ensure that there is a tracking mechanism (like a TODO or a ticket) to revisit this if the functionality needs to be restored in the future. |
||
|
||
|
||
echo 'Load balancer removed successfully' | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The message 'Load balancer removed successfully' is misleading because the code that performs the removal is commented out. This message should be updated to reflect the current behavior of the script. |
Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||||||||||||||||||||||||||||
---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
@@ -685,15 +685,18 @@ func (e *enclaveImpl) GetTransactionCount(encryptedParams common.EncryptedParams | |||||||||||||||||||||||||||||||||||||||||||||||||||
} | ||||||||||||||||||||||||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||||||||||||||||||||||||
var nonce uint64 | ||||||||||||||||||||||||||||||||||||||||||||||||||||
l2Head, err := e.storage.FetchBatchBySeqNo(e.registry.HeadBatchSeq().Uint64()) | ||||||||||||||||||||||||||||||||||||||||||||||||||||
if err == nil { | ||||||||||||||||||||||||||||||||||||||||||||||||||||
// todo - we should return an error when head state is not available, but for current test situations with race | ||||||||||||||||||||||||||||||||||||||||||||||||||||
// conditions we allow it to return zero while head state is uninitialized | ||||||||||||||||||||||||||||||||||||||||||||||||||||
s, err := e.storage.CreateStateDB(l2Head.Hash()) | ||||||||||||||||||||||||||||||||||||||||||||||||||||
if err != nil { | ||||||||||||||||||||||||||||||||||||||||||||||||||||
return nil, responses.ToInternalError(err) | ||||||||||||||||||||||||||||||||||||||||||||||||||||
headBatch := e.registry.HeadBatchSeq() | ||||||||||||||||||||||||||||||||||||||||||||||||||||
if headBatch != nil { | ||||||||||||||||||||||||||||||||||||||||||||||||||||
l2Head, err := e.storage.FetchBatchBySeqNo(headBatch.Uint64()) | ||||||||||||||||||||||||||||||||||||||||||||||||||||
if err == nil { | ||||||||||||||||||||||||||||||||||||||||||||||||||||
// todo - we should return an error when head state is not available, but for current test situations with race | ||||||||||||||||||||||||||||||||||||||||||||||||||||
// conditions we allow it to return zero while head state is uninitialized | ||||||||||||||||||||||||||||||||||||||||||||||||||||
s, err := e.storage.CreateStateDB(l2Head.Hash()) | ||||||||||||||||||||||||||||||||||||||||||||||||||||
if err != nil { | ||||||||||||||||||||||||||||||||||||||||||||||||||||
return nil, responses.ToInternalError(err) | ||||||||||||||||||||||||||||||||||||||||||||||||||||
} | ||||||||||||||||||||||||||||||||||||||||||||||||||||
nonce = s.GetNonce(address) | ||||||||||||||||||||||||||||||||||||||||||||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The refactoring of the logic for fetching the head batch sequence and creating a state DB includes a new conditional check for However, the error handling inside the conditional block could be improved. The comment on line 692 suggests that there are known race conditions that are being temporarily tolerated by allowing a zero nonce to be returned when the head state is uninitialized. This could lead to unexpected behavior and should be addressed as soon as possible. Additionally, the error from l2Head, err := e.storage.FetchBatchBySeqNo(headBatch.Uint64())
if err == nil {
// TODO: We should return an error when head state is not available, but for current test situations with race
// conditions we allow it to return zero while head state is uninitialized
s, err := e.storage.CreateStateDB(l2Head.Hash())
if err != nil {
return nil, responses.ToInternalError(err)
}
nonce = s.GetNonce(address) Committable suggestion
Suggested change
|
||||||||||||||||||||||||||||||||||||||||||||||||||||
} | ||||||||||||||||||||||||||||||||||||||||||||||||||||
nonce = s.GetNonce(address) | ||||||||||||||||||||||||||||||||||||||||||||||||||||
} | ||||||||||||||||||||||||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||||||||||||||||||||||||
encoded := hexutil.EncodeUint64(nonce) | ||||||||||||||||||||||||||||||||||||||||||||||||||||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,18 +2,18 @@ package integration | |
|
||
// Tracks the start ports handed out to different tests, in a bid to minimise conflicts. | ||
const ( | ||
StartPortEth2NetworkTests = 31000 | ||
StartPortNodeRunnerTest = 32000 | ||
StartPortSimulationGethInMem = 34000 | ||
StartPortSimulationInMem = 35000 | ||
StartPortSimulationFullNetwork = 37000 | ||
StartPortSmartContractTests = 38000 | ||
StartPortContractDeployerTest = 39000 | ||
StartPortWalletExtensionUnitTest = 40000 | ||
StartPortFaucetUnitTest = 41000 | ||
StartPortFaucetHTTPUnitTest = 42000 | ||
StartPortTenscanUnitTest = 43000 | ||
StartPortTenGatewayUnitTest = 44000 | ||
StartPortEth2NetworkTests = 10000 | ||
StartPortNodeRunnerTest = 14000 | ||
StartPortSimulationGethInMem = 18000 | ||
StartPortSimulationInMem = 22000 | ||
StartPortSimulationFullNetwork = 26000 | ||
StartPortSmartContractTests = 30000 | ||
StartPortContractDeployerTest = 34000 | ||
StartPortWalletExtensionUnitTest = 38000 | ||
StartPortFaucetUnitTest = 42000 | ||
StartPortFaucetHTTPUnitTest = 48000 | ||
StartPortTenscanUnitTest = 52000 | ||
StartPortTenGatewayUnitTest = 56000 | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Consider adding a comment explaining the rationale for choosing these specific port numbers, as they are now in a range that might be more commonly used by other applications. |
||
|
||
DefaultGethWSPortOffset = 100 | ||
DefaultGethAUTHPortOffset = 200 | ||
|
@@ -24,9 +24,9 @@ const ( | |
DefaultEnclaveOffset = 700 // The default offset between a Geth nodes port and the enclave ports. Used in Socket Simulations. | ||
DefaultHostRPCHTTPOffset = 800 // The default offset for the host's RPC HTTP port | ||
DefaultHostRPCWSOffset = 900 // The default offset for the host's RPC websocket port | ||
DefaultTenscanHTTPPortOffset = 910 | ||
DefaultTenGatewayHTTPPortOffset = 930 | ||
DefaultTenGatewayWSPortOffset = 940 | ||
DefaultTenscanHTTPPortOffset = 1000 | ||
DefaultTenGatewayHTTPPortOffset = 1001 | ||
DefaultTenGatewayWSPortOffset = 1002 | ||
) | ||
|
||
const ( | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -18,6 +18,8 @@ import ( | |
"github.com/ethereum/go-ethereum/ethclient" | ||
"github.com/ten-protocol/go-ten/integration/datagenerator" | ||
"golang.org/x/sync/errgroup" | ||
|
||
gethcommon "github.com/ethereum/go-ethereum/common" | ||
) | ||
|
||
const ( | ||
|
@@ -206,6 +208,10 @@ func (n *Impl) Start() error { | |
startTime := time.Now() | ||
var eg errgroup.Group | ||
|
||
if err := n.ensureNoDuplicatedNetwork(); err != nil { | ||
return err | ||
} | ||
|
||
// initialize the genesis data on the nodes | ||
for _, nodeDataDir := range n.dataDirs { | ||
dataDir := nodeDataDir | ||
|
@@ -506,6 +512,11 @@ func (n *Impl) waitForMergeEvent(startTime time.Time) error { | |
} | ||
|
||
fmt.Printf("Reached the merge block after %s\n", time.Since(startTime)) | ||
|
||
if err = n.prefundedBalancesActive(dial); err != nil { | ||
fmt.Printf("Error prefunding accounts %s\n", err.Error()) | ||
return err | ||
} | ||
return nil | ||
} | ||
|
||
|
@@ -589,6 +600,31 @@ func (n *Impl) gethImportEnodes(enodes []string) error { | |
return nil | ||
} | ||
|
||
func (n *Impl) prefundedBalancesActive(client *ethclient.Client) error { | ||
for _, addr := range n.preFundedMinerAddrs { | ||
balance, err := client.BalanceAt(context.Background(), gethcommon.HexToAddress(addr), nil) | ||
if err != nil { | ||
return fmt.Errorf("unable to check balance for account %s - %w", addr, err) | ||
} | ||
if balance.Cmp(gethcommon.Big0) == 0 { | ||
return fmt.Errorf("unexpected %s balance for account %s", balance.String(), addr) | ||
} | ||
fmt.Printf("Account %s prefunded with %s\n", addr, balance.String()) | ||
} | ||
|
||
return nil | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The if balance.Cmp(expectedMinimumBalance) < 0 {
return fmt.Errorf("insufficient balance for account %s: %s", addr, balance.String())
} |
||
} | ||
|
||
func (n *Impl) ensureNoDuplicatedNetwork() error { | ||
for nodeIdx, port := range n.gethWSPorts { | ||
_, err := ethclient.Dial(fmt.Sprintf("ws://127.0.0.1:%d", port)) | ||
if err == nil { | ||
return fmt.Errorf("unexpected geth node %d is active before the network is started", nodeIdx) | ||
} | ||
} | ||
return nil | ||
} | ||
|
||
func min(a, b int) int { | ||
if a < b { | ||
return a | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The removal of
HOC_ERC20_ADDR
andPOC_ERC20_ADDR
frommanual-deploy-testnet-validator.yml
is confirmed. However, these variables are still referenced in.github/workflows/manual-deploy-testnet-l2.yml
. This could be an oversight or intentional, depending on whether these workflows are interconnected or independent.HOC_ERC20_ADDR
andPOC_ERC20_ADDR
are still used in.github/workflows/manual-deploy-testnet-l2.yml
.Analysis chain
The declarations for
node_l1_ws_url
,HOC_ERC20_ADDR
, andPOC_ERC20_ADDR
have been removed. Confirm that there are no lingering references to these variables in the workflow.Scripts Executed
The following scripts were executed for the analysis:
Script:
Length of output: 394