From 2013be455289b6a33a6b28bc1c29715ebd35c96c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bla=C5=BE=20Hrastnik?= Date: Thu, 8 Aug 2024 20:21:37 +0900 Subject: [PATCH] scripts: keystone: Allow overriding PublicKeys.json/NodeList.txt location --- core/scripts/keystone/.gitignore | 1 + .../keystone/src/01_deploy_contracts_cmd.go | 49 +++++++++++++------ .../keystone/src/02_deploy_jobspecs_cmd.go | 29 +++++++++-- .../src/03_gen_crib_cluster_overrides_cmd.go | 29 ++++++++--- .../03_gen_crib_cluster_overrides_cmd_test.go | 2 +- .../keystone/src/04_delete_ocr3_jobs_cmd.go | 25 ++++++++-- .../keystone/src/06_deploy_workflows_cmd.go | 9 ++-- .../keystone/src/07_delete_workflows_cmd.go | 19 +++++-- core/scripts/keystone/src/88_gen_jobspecs.go | 2 +- .../keystone/src/88_gen_ocr3_config.go | 4 +- .../keystone/src/88_gen_ocr3_config_test.go | 2 +- core/scripts/keystone/src/99_fetch_keys.go | 11 +---- core/scripts/keystone/src/99_files.go | 4 +- 13 files changed, 133 insertions(+), 53 deletions(-) diff --git a/core/scripts/keystone/.gitignore b/core/scripts/keystone/.gitignore index 4af4a42a015..92bf9aabc58 100644 --- a/core/scripts/keystone/.gitignore +++ b/core/scripts/keystone/.gitignore @@ -3,3 +3,4 @@ !*-sample.sh keystone .cache/ +artefacts/ diff --git a/core/scripts/keystone/src/01_deploy_contracts_cmd.go b/core/scripts/keystone/src/01_deploy_contracts_cmd.go index 2ca60bdfaf2..b3049737956 100644 --- a/core/scripts/keystone/src/01_deploy_contracts_cmd.go +++ b/core/scripts/keystone/src/01_deploy_contracts_cmd.go @@ -52,8 +52,12 @@ func (g *deployContracts) Run(args []string) { skipFunding := fs.Bool("skipfunding", false, "skip funding the transmitters") onlySetConfig := fs.Bool("onlysetconfig", false, "set the config on the OCR3 contract without deploying the contracts or funding transmitters") dryRun := fs.Bool("dryrun", false, "dry run, don't actually deploy the contracts and do not fund transmitters") + publicKeys := fs.String("publickeys", "", "Custom public keys json location") + nodeList := fs.String("nodes", "", "Custom node list location") + artefactsDir := fs.String("artefacts", "", "Custom artefacts directory location") err := fs.Parse(args) + if err != nil || *ocrConfigFile == "" || ocrConfigFile == nil || *ethUrl == "" || ethUrl == nil || @@ -63,11 +67,21 @@ func (g *deployContracts) Run(args []string) { os.Exit(1) } + if *artefactsDir == "" { + *artefactsDir = defaultArtefactsDir + } + if *publicKeys == "" { + *publicKeys = defaultPublicKeys + } + if *nodeList == "" { + *nodeList = defaultNodeList + } + os.Setenv("ETH_URL", *ethUrl) os.Setenv("ETH_CHAIN_ID", fmt.Sprintf("%d", *chainID)) os.Setenv("ACCOUNT_KEY", *accountKey) - deploy(*ocrConfigFile, *skipFunding, *dryRun, *onlySetConfig) + deploy(*nodeList, *publicKeys, *ocrConfigFile, *skipFunding, *dryRun, *onlySetConfig, *artefactsDir) } // deploy does the following: @@ -77,16 +91,20 @@ func (g *deployContracts) Run(args []string) { // 4. Writes the deployed contract addresses to a file // 5. Funds the transmitters func deploy( + nodeList string, + publicKeys string, configFile string, skipFunding bool, dryRun bool, onlySetConfig bool, + artefacts string, ) { env := helpers.SetupEnv(false) ocrConfig := generateOCR3Config( + nodeList, configFile, env.ChainID, - ".cache/PublicKeys.json", + publicKeys, ) if dryRun { @@ -96,11 +114,11 @@ func deploy( if onlySetConfig { fmt.Println("Skipping deployment of contracts and skipping funding transmitters, only setting config") - setOCR3Config(env, ocrConfig) + setOCR3Config(env, ocrConfig, artefacts) return } - if ContractsAlreadyDeployed() { + if ContractsAlreadyDeployed(artefacts) { fmt.Println("Contracts already deployed") return } @@ -118,10 +136,10 @@ func deploy( jsonBytes, err := json.Marshal(contracts) PanicErr(err) - err = os.WriteFile(DeployedContractsFilePath(), jsonBytes, 0600) + err = os.WriteFile(DeployedContractsFilePath(artefacts), jsonBytes, 0600) PanicErr(err) - setOCR3Config(env, ocrConfig) + setOCR3Config(env, ocrConfig, artefacts) if skipFunding { fmt.Println("Skipping funding transmitters") @@ -139,8 +157,9 @@ func deploy( func setOCR3Config( env helpers.Environment, ocrConfig orc2drOracleConfig, + artefacts string, ) { - loadedContracts, err := LoadDeployedContracts() + loadedContracts, err := LoadDeployedContracts(artefacts) PanicErr(err) ocrContract, err := ocr3_capability.NewOCR3Capability(loadedContracts.OCRContract, env.Ec) @@ -161,16 +180,16 @@ func setOCR3Config( loadedContracts.SetConfigTxBlock = receipt.BlockNumber.Uint64() jsonBytes, err := json.Marshal(loadedContracts) PanicErr(err) - err = os.WriteFile(DeployedContractsFilePath(), jsonBytes, 0600) + err = os.WriteFile(DeployedContractsFilePath(artefacts), jsonBytes, 0600) PanicErr(err) } -func LoadDeployedContracts() (deployedContracts, error) { - if !ContractsAlreadyDeployed() { +func LoadDeployedContracts(artefacts string) (deployedContracts, error) { + if !ContractsAlreadyDeployed(artefacts) { return deployedContracts{}, fmt.Errorf("no deployed contracts found, run deploy first") } - jsonBytes, err := os.ReadFile(DeployedContractsFilePath()) + jsonBytes, err := os.ReadFile(DeployedContractsFilePath(artefacts)) if err != nil { return deployedContracts{}, err } @@ -180,13 +199,13 @@ func LoadDeployedContracts() (deployedContracts, error) { return contracts, err } -func ContractsAlreadyDeployed() bool { - _, err := os.Stat(DeployedContractsFilePath()) +func ContractsAlreadyDeployed(artefacts string) bool { + _, err := os.Stat(DeployedContractsFilePath(artefacts)) return err == nil } -func DeployedContractsFilePath() string { - return filepath.Join(artefactsDir, deployedContractsJSON) +func DeployedContractsFilePath(artefacts string) string { + return filepath.Join(artefacts, deployedContractsJSON) } func DeployForwarder(e helpers.Environment) *forwarder.KeystoneForwarder { diff --git a/core/scripts/keystone/src/02_deploy_jobspecs_cmd.go b/core/scripts/keystone/src/02_deploy_jobspecs_cmd.go index 5918650cf88..275943d6388 100644 --- a/core/scripts/keystone/src/02_deploy_jobspecs_cmd.go +++ b/core/scripts/keystone/src/02_deploy_jobspecs_cmd.go @@ -16,8 +16,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/cmd" ) -type deployJobSpecs struct { -} +type deployJobSpecs struct{} func NewDeployJobSpecsCommand() *deployJobSpecs { return &deployJobSpecs{} @@ -32,6 +31,11 @@ func (g *deployJobSpecs) Run(args []string) { chainID := fs.Int64("chainid", 11155111, "chain id") p2pPort := fs.Int64("p2pport", 6690, "p2p port") onlyReplay := fs.Bool("onlyreplay", false, "only replay the block from the OCR3 contract setConfig transaction") + templatesLocation := fs.String("templates", "", "Custom templates location") + nodeList := fs.String("nodes", "", "Custom node list location") + publicKeys := fs.String("publickeys", "", "Custom public keys json location") + artefactsDir := fs.String("artefacts", "", "Custom artefacts directory location") + err := fs.Parse(args) if err != nil || chainID == nil || *chainID == 0 || p2pPort == nil || *p2pPort == 0 || onlyReplay == nil { fs.Usage() @@ -43,12 +47,27 @@ func (g *deployJobSpecs) Run(args []string) { fmt.Println("Deploying OCR3 job specs") } - nodes := downloadNodeAPICredentialsDefault() - deployedContracts, err := LoadDeployedContracts() + if *artefactsDir == "" { + *artefactsDir = defaultArtefactsDir + } + if *publicKeys == "" { + *publicKeys = defaultPublicKeys + } + if *nodeList == "" { + *nodeList = defaultNodeList + } + if *templatesLocation == "" { + *templatesLocation = "templates" + } + + nodes := downloadNodeAPICredentials(*nodeList) + deployedContracts, err := LoadDeployedContracts(*artefactsDir) PanicErr(err) jobspecs := genSpecs( - ".cache/PublicKeys.json", ".cache/NodeList.txt", "templates", + *publicKeys, + *nodeList, + *templatesLocation, *chainID, *p2pPort, deployedContracts.OCRContract.Hex(), ) flattenedSpecs := []hostSpec{jobspecs.bootstrap} diff --git a/core/scripts/keystone/src/03_gen_crib_cluster_overrides_cmd.go b/core/scripts/keystone/src/03_gen_crib_cluster_overrides_cmd.go index cb3acf903b3..6b98951459e 100644 --- a/core/scripts/keystone/src/03_gen_crib_cluster_overrides_cmd.go +++ b/core/scripts/keystone/src/03_gen_crib_cluster_overrides_cmd.go @@ -9,8 +9,7 @@ import ( helpers "github.com/smartcontractkit/chainlink/core/scripts/common" ) -type generateCribClusterOverrides struct { -} +type generateCribClusterOverrides struct{} func NewGenerateCribClusterOverridesCommand() *generateCribClusterOverrides { return &generateCribClusterOverrides{} @@ -24,25 +23,39 @@ func (g *generateCribClusterOverrides) Run(args []string) { fs := flag.NewFlagSet(g.Name(), flag.ContinueOnError) chainID := fs.Int64("chainid", 11155111, "chain id") outputPath := fs.String("outpath", "../crib", "the path to output the generated overrides") + publicKeys := fs.String("publickeys", "", "Custom public keys json location") + nodeList := fs.String("nodes", "", "Custom node list location") + artefactsDir := fs.String("artefacts", "", "Custom artefacts directory location") - deployedContracts, err := LoadDeployedContracts() - helpers.PanicErr(err) templatesDir := "templates" - err = fs.Parse(args) + err := fs.Parse(args) if err != nil || outputPath == nil || *outputPath == "" || chainID == nil || *chainID == 0 { fs.Usage() os.Exit(1) } - lines := generateCribConfig(".cache/PublicKeys.json", chainID, templatesDir, deployedContracts.ForwarderContract.Hex()) + if *artefactsDir == "" { + *artefactsDir = defaultArtefactsDir + } + if *publicKeys == "" { + *publicKeys = defaultPublicKeys + } + if *nodeList == "" { + *nodeList = defaultNodeList + } + + deployedContracts, err := LoadDeployedContracts(*artefactsDir) + helpers.PanicErr(err) + + lines := generateCribConfig(*nodeList, *publicKeys, chainID, templatesDir, deployedContracts.ForwarderContract.Hex()) cribOverridesStr := strings.Join(lines, "\n") err = os.WriteFile(filepath.Join(*outputPath, "crib-cluster-overrides.yaml"), []byte(cribOverridesStr), 0600) helpers.PanicErr(err) } -func generateCribConfig(pubKeysPath string, chainID *int64, templatesDir string, forwarderAddress string) []string { - nca := downloadNodePubKeys(*chainID, pubKeysPath) +func generateCribConfig(nodeList string, pubKeysPath string, chainID *int64, templatesDir string, forwarderAddress string) []string { + nca := downloadNodePubKeys(nodeList, *chainID, pubKeysPath) nodeAddresses := []string{} for _, node := range nca[1:] { diff --git a/core/scripts/keystone/src/03_gen_crib_cluster_overrides_cmd_test.go b/core/scripts/keystone/src/03_gen_crib_cluster_overrides_cmd_test.go index 722b01e91c3..53d43c2342f 100644 --- a/core/scripts/keystone/src/03_gen_crib_cluster_overrides_cmd_test.go +++ b/core/scripts/keystone/src/03_gen_crib_cluster_overrides_cmd_test.go @@ -13,7 +13,7 @@ func TestGenerateCribConfig(t *testing.T) { forwarderAddress := "0x1234567890abcdef" publicKeysPath := "./testdata/PublicKeys.json" - lines := generateCribConfig(publicKeysPath, &chainID, templatesDir, forwarderAddress) + lines := generateCribConfig(defaultNodeList, publicKeysPath, &chainID, templatesDir, forwarderAddress) snaps.MatchSnapshot(t, strings.Join(lines, "\n")) } diff --git a/core/scripts/keystone/src/04_delete_ocr3_jobs_cmd.go b/core/scripts/keystone/src/04_delete_ocr3_jobs_cmd.go index 65c494d5766..136691962dd 100644 --- a/core/scripts/keystone/src/04_delete_ocr3_jobs_cmd.go +++ b/core/scripts/keystone/src/04_delete_ocr3_jobs_cmd.go @@ -5,14 +5,14 @@ import ( "encoding/json" "flag" "fmt" + "os" "github.com/urfave/cli" helpers "github.com/smartcontractkit/chainlink/core/scripts/common" ) -type deleteJobs struct { -} +type deleteJobs struct{} type OCRSpec struct { ContractID string @@ -43,9 +43,26 @@ func (g *deleteJobs) Name() string { } func (g *deleteJobs) Run(args []string) { - deployedContracts, err := LoadDeployedContracts() + fs := flag.NewFlagSet(g.Name(), flag.ContinueOnError) + nodeList := fs.String("nodes", "", "Custom node list location") + artefactsDir := fs.String("artefacts", "", "Custom artefacts directory location") + + err := fs.Parse(args) + if err != nil { + fs.Usage() + os.Exit(1) + } + + if *artefactsDir == "" { + *artefactsDir = defaultArtefactsDir + } + if *nodeList == "" { + *nodeList = defaultNodeList + } + + deployedContracts, err := LoadDeployedContracts(*artefactsDir) helpers.PanicErr(err) - nodes := downloadNodeAPICredentialsDefault() + nodes := downloadNodeAPICredentials(*nodeList) for _, node := range nodes { output := &bytes.Buffer{} diff --git a/core/scripts/keystone/src/06_deploy_workflows_cmd.go b/core/scripts/keystone/src/06_deploy_workflows_cmd.go index ea6be88da88..0ca8e5d4a7b 100644 --- a/core/scripts/keystone/src/06_deploy_workflows_cmd.go +++ b/core/scripts/keystone/src/06_deploy_workflows_cmd.go @@ -12,8 +12,7 @@ import ( helpers "github.com/smartcontractkit/chainlink/core/scripts/common" ) -type deployWorkflows struct { -} +type deployWorkflows struct{} func NewDeployWorkflowsCommand() *deployWorkflows { return &deployWorkflows{} @@ -26,15 +25,19 @@ func (g *deployWorkflows) Name() string { func (g *deployWorkflows) Run(args []string) { fs := flag.NewFlagSet(g.Name(), flag.ContinueOnError) workflowFile := fs.String("workflow", "workflow.yml", "path to workflow file") + nodeList := fs.String("nodes", "", "Custom node list location") err := fs.Parse(args) if err != nil || workflowFile == nil || *workflowFile == "" { fs.Usage() os.Exit(1) } + if *nodeList == "" { + *nodeList = defaultNodeList + } fmt.Println("Deploying workflows") // use a separate list - nodes := downloadNodeAPICredentialsDefault() + nodes := downloadNodeAPICredentials(*nodeList) if _, err = os.Stat(*workflowFile); err != nil { PanicErr(errors.New("toml file does not exist")) diff --git a/core/scripts/keystone/src/07_delete_workflows_cmd.go b/core/scripts/keystone/src/07_delete_workflows_cmd.go index 0490f6352c2..cccedaf9e70 100644 --- a/core/scripts/keystone/src/07_delete_workflows_cmd.go +++ b/core/scripts/keystone/src/07_delete_workflows_cmd.go @@ -5,14 +5,14 @@ import ( "encoding/json" "flag" "fmt" + "os" "github.com/urfave/cli" helpers "github.com/smartcontractkit/chainlink/core/scripts/common" ) -type deleteWorkflows struct { -} +type deleteWorkflows struct{} func NewDeleteWorkflowsCommand() *deleteWorkflows { return &deleteWorkflows{} @@ -23,7 +23,20 @@ func (g *deleteWorkflows) Name() string { } func (g *deleteWorkflows) Run(args []string) { - nodes := downloadNodeAPICredentialsDefault() + fs := flag.NewFlagSet(g.Name(), flag.ExitOnError) + nodeList := fs.String("nodes", "", "Custom node list location") + + err := fs.Parse(args) + if err != nil { + fs.Usage() + os.Exit(1) + } + + if *nodeList == "" { + *nodeList = defaultNodeList + } + + nodes := downloadNodeAPICredentials(*nodeList) for _, node := range nodes { output := &bytes.Buffer{} diff --git a/core/scripts/keystone/src/88_gen_jobspecs.go b/core/scripts/keystone/src/88_gen_jobspecs.go index 046906a15e2..5f0b9097d2f 100644 --- a/core/scripts/keystone/src/88_gen_jobspecs.go +++ b/core/scripts/keystone/src/88_gen_jobspecs.go @@ -34,7 +34,7 @@ func genSpecs( ocrConfigContractAddress string, ) donHostSpec { nodes := downloadNodeAPICredentials(nodeListPath) - nca := downloadNodePubKeys(chainID, pubkeysPath) + nca := downloadNodePubKeys(nodeListPath, chainID, pubkeysPath) bootstrapNode := nca[0] bootstrapSpecLines, err := readLines(filepath.Join(templatesDir, bootstrapSpecTemplate)) diff --git a/core/scripts/keystone/src/88_gen_ocr3_config.go b/core/scripts/keystone/src/88_gen_ocr3_config.go index fe9241a2bde..1107df57ca1 100644 --- a/core/scripts/keystone/src/88_gen_ocr3_config.go +++ b/core/scripts/keystone/src/88_gen_ocr3_config.go @@ -96,10 +96,10 @@ func mustReadConfig(fileName string) (output TopLevelConfigSource) { return mustParseJSON[TopLevelConfigSource](fileName) } -func generateOCR3Config(configFile string, chainID int64, pubKeysPath string) orc2drOracleConfig { +func generateOCR3Config(nodeList string, configFile string, chainID int64, pubKeysPath string) orc2drOracleConfig { topLevelCfg := mustReadConfig(configFile) cfg := topLevelCfg.OracleConfig - nca := downloadNodePubKeys(chainID, pubKeysPath) + nca := downloadNodePubKeys(nodeList, chainID, pubKeysPath) onchainPubKeys := []common.Address{} for _, n := range nca { diff --git a/core/scripts/keystone/src/88_gen_ocr3_config_test.go b/core/scripts/keystone/src/88_gen_ocr3_config_test.go index 185354ec2fe..10cdc07b204 100644 --- a/core/scripts/keystone/src/88_gen_ocr3_config_test.go +++ b/core/scripts/keystone/src/88_gen_ocr3_config_test.go @@ -10,7 +10,7 @@ import ( func TestGenerateOCR3Config(t *testing.T) { // Generate OCR3 config - config := generateOCR3Config("./testdata/SampleConfig.json", 11155111, "./testdata/PublicKeys.json") + config := generateOCR3Config(".cache/NodeList.txt", "./testdata/SampleConfig.json", 11155111, "./testdata/PublicKeys.json") matchOffchainConfig := match.Custom("OffchainConfig", func(s any) (any, error) { // coerce the value to a string diff --git a/core/scripts/keystone/src/99_fetch_keys.go b/core/scripts/keystone/src/99_fetch_keys.go index 7a3967fc2c8..b115a7bb94d 100644 --- a/core/scripts/keystone/src/99_fetch_keys.go +++ b/core/scripts/keystone/src/99_fetch_keys.go @@ -17,14 +17,14 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/web/presenters" ) -func downloadNodePubKeys(chainID int64, pubKeysPath string) []NodeKeys { +func downloadNodePubKeys(nodeList string, chainID int64, pubKeysPath string) []NodeKeys { // Check if file exists already, and if so, return the keys if _, err := os.Stat(pubKeysPath); err == nil { fmt.Println("Loading existing public keys at:", pubKeysPath) return mustParseJSON[[]NodeKeys](pubKeysPath) } - nodes := downloadNodeAPICredentialsDefault() + nodes := downloadNodeAPICredentials(nodeList) nodesKeys := mustFetchNodesKeys(chainID, nodes) marshalledNodeKeys, err := json.MarshalIndent(nodesKeys, "", " ") @@ -40,13 +40,6 @@ func downloadNodePubKeys(chainID int64, pubKeysPath string) []NodeKeys { return nodesKeys } -// downloadNodeAPICredentialsDefault downloads the node API credentials, or loads them from disk if they already exist -// -// The nodes are sorted by URL. In the case of crib, the bootstrap node is the first node in the list. -func downloadNodeAPICredentialsDefault() []*node { - return downloadNodeAPICredentials(".cache/NodeList.local.txt") -} - // downloadNodeAPICredentials downloads the node API credentials, or loads them from disk if they already exist // // The nodes are sorted by URL. In the case of crib, the bootstrap node is the first node in the list. diff --git a/core/scripts/keystone/src/99_files.go b/core/scripts/keystone/src/99_files.go index d334b0fd56c..08ba12e4194 100644 --- a/core/scripts/keystone/src/99_files.go +++ b/core/scripts/keystone/src/99_files.go @@ -11,7 +11,9 @@ import ( ) const ( - artefactsDir = "artefacts" + defaultArtefactsDir = "artefacts" + defaultPublicKeys = ".cache/PublicKeys.json" + defaultNodeList = ".cache/NodeList.txt" deployedContractsJSON = "deployed_contracts.json" bootstrapSpecTemplate = "bootstrap.toml" cribOverrideTemplate = "crib-overrides.yaml"