core/scripts/keystone improvements (smartcontractkit#14075)

* scripts: keystone: Use Hostname() over Host

Host can contain host:port which will then result in invalid bootstrap
addresses (host:port:port)

* scripts: keystone: Add command to deploy workflow spec

* scripts: keystone: oracle spec used the wrong provider type

* scripts: keystone: Add utility to delete a workflow spec

* Simplify the changes to scripts a bit

* scripts: keystone: Allow overriding PublicKeys.json/NodeList.txt location

core/scripts/keystone/.gitignore

@@ -3,3 +3,4 @@
 !*-sample.sh
 keystone
 .cache/
+artefacts/

core/scripts/keystone/main.go

@@ -20,6 +20,8 @@ func main() {
 		src.NewGenerateCribClusterOverridesCommand(),
 		src.NewDeleteJobsCommand(),
 		src.NewDeployAndInitializeCapabilitiesRegistryCommand(),
+		src.NewDeployWorkflowsCommand(),
+		src.NewDeleteWorkflowsCommand(),
 	}
 
 	commandsList := func(commands []command) string {

core/scripts/keystone/src/01_deploy_contracts_cmd.go

@@ -52,8 +52,12 @@ func (g *deployContracts) Run(args []string) {
 	skipFunding := fs.Bool("skipfunding", false, "skip funding the transmitters")
 	onlySetConfig := fs.Bool("onlysetconfig", false, "set the config on the OCR3 contract without deploying the contracts or funding transmitters")
 	dryRun := fs.Bool("dryrun", false, "dry run, don't actually deploy the contracts and do not fund transmitters")
+	publicKeys := fs.String("publickeys", "", "Custom public keys json location")
+	nodeList := fs.String("nodes", "", "Custom node list location")
+	artefactsDir := fs.String("artefacts", "", "Custom artefacts directory location")
 
 	err := fs.Parse(args)
+
 	if err != nil ||
 		*ocrConfigFile == "" || ocrConfigFile == nil ||
 		*ethUrl == "" || ethUrl == nil ||
@@ -63,11 +67,21 @@ func (g *deployContracts) Run(args []string) {
 		os.Exit(1)
 	}
 
+	if *artefactsDir == "" {
+		*artefactsDir = defaultArtefactsDir
+	}
+	if *publicKeys == "" {
+		*publicKeys = defaultPublicKeys
+	}
+	if *nodeList == "" {
+		*nodeList = defaultNodeList
+	}
+
 	os.Setenv("ETH_URL", *ethUrl)
 	os.Setenv("ETH_CHAIN_ID", fmt.Sprintf("%d", *chainID))
 	os.Setenv("ACCOUNT_KEY", *accountKey)
 
-	deploy(*ocrConfigFile, *skipFunding, *dryRun, *onlySetConfig)
+	deploy(*nodeList, *publicKeys, *ocrConfigFile, *skipFunding, *dryRun, *onlySetConfig, *artefactsDir)
 }
 
 // deploy does the following:
@@ -77,16 +91,20 @@ func (g *deployContracts) Run(args []string) {
 //  4. Writes the deployed contract addresses to a file
 //  5. Funds the transmitters
 func deploy(
+	nodeList string,
+	publicKeys string,
 	configFile string,
 	skipFunding bool,
 	dryRun bool,
 	onlySetConfig bool,
+	artefacts string,
 ) {
 	env := helpers.SetupEnv(false)
 	ocrConfig := generateOCR3Config(
+		nodeList,
 		configFile,
 		env.ChainID,
-		".cache/PublicKeys.json",
+		publicKeys,
 	)
 
 	if dryRun {
@@ -96,11 +114,11 @@ func deploy(
 
 	if onlySetConfig {
 		fmt.Println("Skipping deployment of contracts and skipping funding transmitters, only setting config")
-		setOCR3Config(env, ocrConfig)
+		setOCR3Config(env, ocrConfig, artefacts)
 		return
 	}
 
-	if ContractsAlreadyDeployed() {
+	if ContractsAlreadyDeployed(artefacts) {
 		fmt.Println("Contracts already deployed")
 		return
 	}
@@ -118,10 +136,10 @@ func deploy(
 	jsonBytes, err := json.Marshal(contracts)
 	PanicErr(err)
 
-	err = os.WriteFile(DeployedContractsFilePath(), jsonBytes, 0600)
+	err = os.WriteFile(DeployedContractsFilePath(artefacts), jsonBytes, 0600)
 	PanicErr(err)
 
-	setOCR3Config(env, ocrConfig)
+	setOCR3Config(env, ocrConfig, artefacts)
 
 	if skipFunding {
 		fmt.Println("Skipping funding transmitters")
@@ -139,8 +157,9 @@ func deploy(
 func setOCR3Config(
 	env helpers.Environment,
 	ocrConfig orc2drOracleConfig,
+	artefacts string,
 ) {
-	loadedContracts, err := LoadDeployedContracts()
+	loadedContracts, err := LoadDeployedContracts(artefacts)
 	PanicErr(err)
 
 	ocrContract, err := ocr3_capability.NewOCR3Capability(loadedContracts.OCRContract, env.Ec)
@@ -161,16 +180,16 @@ func setOCR3Config(
 	loadedContracts.SetConfigTxBlock = receipt.BlockNumber.Uint64()
 	jsonBytes, err := json.Marshal(loadedContracts)
 	PanicErr(err)
-	err = os.WriteFile(DeployedContractsFilePath(), jsonBytes, 0600)
+	err = os.WriteFile(DeployedContractsFilePath(artefacts), jsonBytes, 0600)
 	PanicErr(err)
 }
 
-func LoadDeployedContracts() (deployedContracts, error) {
-	if !ContractsAlreadyDeployed() {
+func LoadDeployedContracts(artefacts string) (deployedContracts, error) {
+	if !ContractsAlreadyDeployed(artefacts) {
 		return deployedContracts{}, fmt.Errorf("no deployed contracts found, run deploy first")
 	}
 
-	jsonBytes, err := os.ReadFile(DeployedContractsFilePath())
+	jsonBytes, err := os.ReadFile(DeployedContractsFilePath(artefacts))
 	if err != nil {
 		return deployedContracts{}, err
 	}
@@ -180,13 +199,13 @@ func LoadDeployedContracts() (deployedContracts, error) {
 	return contracts, err
 }
 
-func ContractsAlreadyDeployed() bool {
-	_, err := os.Stat(DeployedContractsFilePath())
+func ContractsAlreadyDeployed(artefacts string) bool {
+	_, err := os.Stat(DeployedContractsFilePath(artefacts))
 	return err == nil
 }
 
-func DeployedContractsFilePath() string {
-	return filepath.Join(artefactsDir, deployedContractsJSON)
+func DeployedContractsFilePath(artefacts string) string {
+	return filepath.Join(artefacts, deployedContractsJSON)
 }
 
 func DeployForwarder(e helpers.Environment) *forwarder.KeystoneForwarder {

core/scripts/keystone/src/02_deploy_jobspecs_cmd.go

@@ -16,8 +16,7 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/cmd"
 )
 
-type deployJobSpecs struct {
-}
+type deployJobSpecs struct{}
 
 func NewDeployJobSpecsCommand() *deployJobSpecs {
 	return &deployJobSpecs{}
@@ -32,6 +31,11 @@ func (g *deployJobSpecs) Run(args []string) {
 	chainID := fs.Int64("chainid", 11155111, "chain id")
 	p2pPort := fs.Int64("p2pport", 6690, "p2p port")
 	onlyReplay := fs.Bool("onlyreplay", false, "only replay the block from the OCR3 contract setConfig transaction")
+	templatesLocation := fs.String("templates", "", "Custom templates location")
+	nodeList := fs.String("nodes", "", "Custom node list location")
+	publicKeys := fs.String("publickeys", "", "Custom public keys json location")
+	artefactsDir := fs.String("artefacts", "", "Custom artefacts directory location")
+
 	err := fs.Parse(args)
 	if err != nil || chainID == nil || *chainID == 0 || p2pPort == nil || *p2pPort == 0 || onlyReplay == nil {
 		fs.Usage()
@@ -43,12 +47,27 @@ func (g *deployJobSpecs) Run(args []string) {
 		fmt.Println("Deploying OCR3 job specs")
 	}
 
-	nodes := downloadNodeAPICredentialsDefault()
-	deployedContracts, err := LoadDeployedContracts()
+	if *artefactsDir == "" {
+		*artefactsDir = defaultArtefactsDir
+	}
+	if *publicKeys == "" {
+		*publicKeys = defaultPublicKeys
+	}
+	if *nodeList == "" {
+		*nodeList = defaultNodeList
+	}
+	if *templatesLocation == "" {
+		*templatesLocation = "templates"
+	}
+
+	nodes := downloadNodeAPICredentials(*nodeList)
+	deployedContracts, err := LoadDeployedContracts(*artefactsDir)
 	PanicErr(err)
 
 	jobspecs := genSpecs(
-		".cache/PublicKeys.json", ".cache/NodeList.txt", "templates",
+		*publicKeys,
+		*nodeList,
+		*templatesLocation,
 		*chainID, *p2pPort, deployedContracts.OCRContract.Hex(),
 	)
 	flattenedSpecs := []hostSpec{jobspecs.bootstrap}

core/scripts/keystone/src/03_gen_crib_cluster_overrides_cmd.go

@@ -9,8 +9,7 @@ import (
 	helpers "github.com/smartcontractkit/chainlink/core/scripts/common"
 )
 
-type generateCribClusterOverrides struct {
-}
+type generateCribClusterOverrides struct{}
 
 func NewGenerateCribClusterOverridesCommand() *generateCribClusterOverrides {
 	return &generateCribClusterOverrides{}
@@ -24,25 +23,39 @@ func (g *generateCribClusterOverrides) Run(args []string) {
 	fs := flag.NewFlagSet(g.Name(), flag.ContinueOnError)
 	chainID := fs.Int64("chainid", 11155111, "chain id")
 	outputPath := fs.String("outpath", "../crib", "the path to output the generated overrides")
+	publicKeys := fs.String("publickeys", "", "Custom public keys json location")
+	nodeList := fs.String("nodes", "", "Custom node list location")
+	artefactsDir := fs.String("artefacts", "", "Custom artefacts directory location")
 
-	deployedContracts, err := LoadDeployedContracts()
-	helpers.PanicErr(err)
 	templatesDir := "templates"
-	err = fs.Parse(args)
+	err := fs.Parse(args)
 	if err != nil || outputPath == nil || *outputPath == "" || chainID == nil || *chainID == 0 {
 		fs.Usage()
 		os.Exit(1)
 	}
 
-	lines := generateCribConfig(".cache/PublicKeys.json", chainID, templatesDir, deployedContracts.ForwarderContract.Hex())
+	if *artefactsDir == "" {
+		*artefactsDir = defaultArtefactsDir
+	}
+	if *publicKeys == "" {
+		*publicKeys = defaultPublicKeys
+	}
+	if *nodeList == "" {
+		*nodeList = defaultNodeList
+	}
+
+	deployedContracts, err := LoadDeployedContracts(*artefactsDir)
+	helpers.PanicErr(err)
+
+	lines := generateCribConfig(*nodeList, *publicKeys, chainID, templatesDir, deployedContracts.ForwarderContract.Hex())
 
 	cribOverridesStr := strings.Join(lines, "\n")
 	err = os.WriteFile(filepath.Join(*outputPath, "crib-cluster-overrides.yaml"), []byte(cribOverridesStr), 0600)
 	helpers.PanicErr(err)
 }
 
-func generateCribConfig(pubKeysPath string, chainID *int64, templatesDir string, forwarderAddress string) []string {
-	nca := downloadNodePubKeys(*chainID, pubKeysPath)
+func generateCribConfig(nodeList string, pubKeysPath string, chainID *int64, templatesDir string, forwarderAddress string) []string {
+	nca := downloadNodePubKeys(nodeList, *chainID, pubKeysPath)
 	nodeAddresses := []string{}
 
 	for _, node := range nca[1:] {

core/scripts/keystone/src/03_gen_crib_cluster_overrides_cmd_test.go

@@ -13,7 +13,7 @@ func TestGenerateCribConfig(t *testing.T) {
 	forwarderAddress := "0x1234567890abcdef"
 	publicKeysPath := "./testdata/PublicKeys.json"
 
-	lines := generateCribConfig(publicKeysPath, &chainID, templatesDir, forwarderAddress)
+	lines := generateCribConfig(defaultNodeList, publicKeysPath, &chainID, templatesDir, forwarderAddress)
 
 	snaps.MatchSnapshot(t, strings.Join(lines, "\n"))
 }

core/scripts/keystone/src/04_delete_ocr3_jobs_cmd.go

@@ -5,14 +5,14 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"os"
 
 	"github.com/urfave/cli"
 
 	helpers "github.com/smartcontractkit/chainlink/core/scripts/common"
 )
 
-type deleteJobs struct {
-}
+type deleteJobs struct{}
 
 type OCRSpec struct {
 	ContractID string
@@ -22,11 +22,16 @@ type BootSpec struct {
 	ContractID string
 }
 
+type WorkflowSpec struct {
+	WorkflowID string
+}
+
 type JobSpec struct {
 	Id                           string
 	Name                         string
 	BootstrapSpec                BootSpec
 	OffChainReporting2OracleSpec OCRSpec
+	WorkflowSpec                 WorkflowSpec
 }
 
 func NewDeleteJobsCommand() *deleteJobs {
@@ -38,9 +43,26 @@ func (g *deleteJobs) Name() string {
 }
 
 func (g *deleteJobs) Run(args []string) {
-	deployedContracts, err := LoadDeployedContracts()
+	fs := flag.NewFlagSet(g.Name(), flag.ContinueOnError)
+	nodeList := fs.String("nodes", "", "Custom node list location")
+	artefactsDir := fs.String("artefacts", "", "Custom artefacts directory location")
+
+	err := fs.Parse(args)
+	if err != nil {
+		fs.Usage()
+		os.Exit(1)
+	}
+
+	if *artefactsDir == "" {
+		*artefactsDir = defaultArtefactsDir
+	}
+	if *nodeList == "" {
+		*nodeList = defaultNodeList
+	}
+
+	deployedContracts, err := LoadDeployedContracts(*artefactsDir)
 	helpers.PanicErr(err)
-	nodes := downloadNodeAPICredentialsDefault()
+	nodes := downloadNodeAPICredentials(*nodeList)
 
 	for _, node := range nodes {
 		output := &bytes.Buffer{}