From 73966ef5dd383aca7e97747d75f6c58a20f84cce Mon Sep 17 00:00:00 2001 From: Jordan Krage Date: Mon, 18 Sep 2023 16:29:48 -0500 Subject: [PATCH 1/3] core/services/chainlink: log warn instead of error if CSA key not available for feeds service (#10543) --- core/services/chainlink/application.go | 42 ++++++++++++++------------ 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/core/services/chainlink/application.go b/core/services/chainlink/application.go index 89070cbf96b..ba8e242ea8e 100644 --- a/core/services/chainlink/application.go +++ b/core/services/chainlink/application.go @@ -419,26 +419,30 @@ func NewApplication(opts ApplicationOpts) (Application, error) { } } - var feedsService feeds.Service + var feedsService feeds.Service = &feeds.NullService{} if cfg.Feature().FeedsManager() { - feedsORM := feeds.NewORM(db, opts.Logger, cfg.Database()) - feedsService = feeds.NewService( - feedsORM, - jobORM, - db, - jobSpawner, - keyStore, - cfg.Insecure(), - cfg.JobPipeline(), - cfg.OCR(), - cfg.OCR2(), - cfg.Database(), - legacyEVMChains, - globalLogger, - opts.Version, - ) - } else { - feedsService = &feeds.NullService{} + if keys, err := opts.KeyStore.CSA().GetAll(); err != nil { + globalLogger.Warn("[Feeds Service] Unable to start without CSA key", "err", err) + } else if len(keys) == 0 { + globalLogger.Warn("[Feeds Service] Unable to start without CSA key") + } else { + feedsORM := feeds.NewORM(db, opts.Logger, cfg.Database()) + feedsService = feeds.NewService( + feedsORM, + jobORM, + db, + jobSpawner, + keyStore, + cfg.Insecure(), + cfg.JobPipeline(), + cfg.OCR(), + cfg.OCR2(), + cfg.Database(), + legacyEVMChains, + globalLogger, + opts.Version, + ) + } } healthChecker := services.NewChecker() From c8762ee4ee3fc7b2ea5e5c75c3a917ff3d5a4aa8 Mon Sep 17 00:00:00 2001 From: Adam Hamrick Date: Mon, 18 Sep 2023 18:51:39 -0400 Subject: [PATCH 2/3] Improve Testnet Notifications (#10681) * Improve Testnet Notifications * Use cancelled() * Better check --- .github/workflows/integration-tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index a51f1c6997b..8455df90aa2 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -168,6 +168,7 @@ jobs: MATRIX_JSON_KEEPER=$(./scripts/buildTestMatrixList.sh ./smoke/keeper_test.go keeper ubuntu20.04-8cores-32GB 1) COMBINED_ARRAY=$(jq -c -n "$MATRIX_JSON_AUTOMATION + $MATRIX_JSON_KEEPER") echo "MATRIX_JSON=${COMBINED_ARRAY}" >> $GITHUB_ENV + eth-smoke-tests-matrix-automation: if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }} environment: integration @@ -235,6 +236,7 @@ jobs: this-job-name: ETH Smoke Tests ${{ matrix.product.name }} test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}' continue-on-error: true + eth-smoke-tests-matrix: if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }} environment: integration @@ -805,7 +807,7 @@ jobs: testnet-smoke-tests-notify: name: Live Testnet Start Slack Thread - if: success() || failure() + if: ${{ needs.*.result != 'skipped' }} environment: integration outputs: thread_ts: ${{ steps.slack.outputs.thread_ts }} @@ -857,6 +859,7 @@ jobs: testnet-smoke-tests-results: name: Post Live Testnet Smoke Test Results + if: ${{ needs.*.result != 'skipped' }} environment: integration permissions: checks: write From eba9a270f6c5cf71a606c4fa8c90f05cf88ea354 Mon Sep 17 00:00:00 2001 From: ilija42 <57732589+ilija42@users.noreply.github.com> Date: Tue, 19 Sep 2023 01:00:42 +0200 Subject: [PATCH 3/3] BCF-2637 Add evm chain id removal migrations (#10669) * Add evm chain id removal migrations * Remove _test db url check for 0195 evm chain id migration * Improve 0195 migration, improve error msg and sql string constant names * Update 0195 migration to only add/remove evmChainID not nil constraints * Remove dangling err handling (leftover from evmChainID removal) * Update 0195 migration naming * Add helper function for not null evmChainID migration * Update changelog for evm chain id not null migration helper function * Update error message for sql open in shell MigrateDatabase * Fix job_orm_test test that set evm_chain_id to null * Fix evmChainID migration helper err when db is init for the first time * Move goose migration ver check for evmChainID migration in helper func --- core/cmd/shell.go | 44 +++++++ core/cmd/shell_local.go | 21 +++- core/internal/cltest/factories.go | 4 +- core/services/job/job_orm_test.go | 113 +++--------------- ...d_not_null_to_evm_chain_id_in_job_specs.go | 49 ++++++++ docs/CHANGELOG.md | 4 + 6 files changed, 131 insertions(+), 104 deletions(-) create mode 100644 core/store/migrate/migrations/0195_add_not_null_to_evm_chain_id_in_job_specs.go diff --git a/core/cmd/shell.go b/core/cmd/shell.go index 2f4da835f24..b9c031fd8b7 100644 --- a/core/cmd/shell.go +++ b/core/cmd/shell.go @@ -133,6 +133,12 @@ type ChainlinkAppFactory struct{} func (n ChainlinkAppFactory) NewApplication(ctx context.Context, cfg chainlink.GeneralConfig, appLggr logger.Logger, db *sqlx.DB) (app chainlink.Application, err error) { initGlobals(cfg.Prometheus()) + // TODO TO BE REMOVED IN v2.7.0 + err = evmChainIDMigration(cfg, db.DB, appLggr) + if err != nil { + return nil, err + } + err = handleNodeVersioning(db, appLggr, cfg.RootDir(), cfg.Database(), cfg.WebServer().HTTPPort()) if err != nil { return nil, err @@ -295,6 +301,44 @@ func takeBackupIfVersionUpgrade(dbUrl url.URL, rootDir string, cfg periodicbacku return err } +// evmChainIDMigration TODO TO BE REMOVED IN v2.7.0. This is a helper function for evmChainID 0195 migration in v2.6.0 only, so that we don't have to inject evmChainID into goose. +func evmChainIDMigration(generalConfig chainlink.GeneralConfig, db *sql.DB, lggr logger.Logger) error { + migrationVer, err := migrate.Current(db, lggr) + if err != nil { + return err + } + if migrationVer != 194 { + return nil + } + + if generalConfig.EVMEnabled() { + if generalConfig.EVMConfigs() == nil { + return errors.New("evm configs are missing") + } + if generalConfig.EVMConfigs()[0] == nil { + return errors.New("evm config is nil") + } + updateQueries := []string{ + `UPDATE direct_request_specs SET evm_chain_id = $1 WHERE evm_chain_id IS NULL;`, + `UPDATE flux_monitor_specs SET evm_chain_id = $1 WHERE evm_chain_id IS NULL;`, + `UPDATE ocr_oracle_specs SET evm_chain_id = $1 WHERE evm_chain_id IS NULL;`, + `UPDATE keeper_specs SET evm_chain_id = $1 WHERE evm_chain_id IS NULL;`, + `UPDATE vrf_specs SET evm_chain_id = $1 WHERE evm_chain_id IS NULL;`, + `UPDATE blockhash_store_specs SET evm_chain_id = $1 WHERE evm_chain_id IS NULL;`, + `UPDATE block_header_feeder_specs SET evm_chain_id = $1 WHERE evm_chain_id IS NULL;`, + } + + chainID := generalConfig.EVMConfigs()[0].ChainID.String() + for i := range updateQueries { + _, err := db.Exec(updateQueries[i], chainID) + if err != nil { + return errors.Wrap(err, "failed to set missing evm chain ids") + } + } + } + return nil +} + // Runner implements the Run method. type Runner interface { Run(context.Context, chainlink.Application) error diff --git a/core/cmd/shell_local.go b/core/cmd/shell_local.go index db4d0476fb3..26c0b5ed802 100644 --- a/core/cmd/shell_local.go +++ b/core/cmd/shell_local.go @@ -376,9 +376,6 @@ func (s *Shell) runNode(c *cli.Context) error { legacyEVMChains := app.GetRelayers().LegacyEVMChains() if s.Config.EVMEnabled() { - if err != nil { - return errors.Wrap(err, "error migrating keystore") - } chainList, err := legacyEVMChains.List() if err != nil { return fmt.Errorf("error listing legacy evm chains: %w", err) @@ -843,6 +840,23 @@ func (s *Shell) MigrateDatabase(_ *cli.Context) error { return s.errorOut(errDBURLMissing) } + // TODO TO BE REMOVED IN v2.7.0 + db, err := sql.Open(string(dialects.Postgres), parsed.String()) + if err != nil { + return fmt.Errorf("unable to open postgres database for evmChainID helper migration: %+v", err) + } + defer func() { + if cerr := db.Close(); cerr != nil { + err = multierr.Append(err, cerr) + } + }() + + // TODO TO BE REMOVED IN v2.7.0 + err = evmChainIDMigration(s.Config, db, s.Logger) + if err != nil { + return err + } + s.Logger.Infof("Migrating database: %#v", parsed.String()) if err := migrateDB(cfg, s.Logger); err != nil { return s.errorOut(err) @@ -1026,6 +1040,7 @@ func migrateDB(config dbConfig, lggr logger.Logger) error { if err != nil { return fmt.Errorf("failed to initialize orm: %v", err) } + if err = migrate.Migrate(db.DB, lggr); err != nil { return fmt.Errorf("migrateDB failed: %v", err) } diff --git a/core/internal/cltest/factories.go b/core/internal/cltest/factories.go index 36a89971ca6..cb0b51743e4 100644 --- a/core/internal/cltest/factories.go +++ b/core/internal/cltest/factories.go @@ -624,8 +624,8 @@ func MustInsertOffchainreportingOracleSpec(t *testing.T, db *sqlx.DB, transmitte ocrKeyID := models.MustSha256HashFromHex(DefaultOCRKeyBundleID) spec := job.OCROracleSpec{} - require.NoError(t, db.Get(&spec, `INSERT INTO ocr_oracle_specs (created_at, updated_at, contract_address, p2p_bootstrap_peers, is_bootstrap_peer, encrypted_ocr_key_bundle_id, transmitter_address, observation_timeout, blockchain_timeout, contract_config_tracker_subscribe_interval, contract_config_tracker_poll_interval, contract_config_confirmations, database_timeout, observation_grace_period, contract_transmitter_transmit_timeout) VALUES ( -NOW(),NOW(),$1,'{}',false,$2,$3,0,0,0,0,0,0,0,0 + require.NoError(t, db.Get(&spec, `INSERT INTO ocr_oracle_specs (created_at, updated_at, contract_address, p2p_bootstrap_peers, is_bootstrap_peer, encrypted_ocr_key_bundle_id, transmitter_address, observation_timeout, blockchain_timeout, contract_config_tracker_subscribe_interval, contract_config_tracker_poll_interval, contract_config_confirmations, database_timeout, observation_grace_period, contract_transmitter_transmit_timeout, evm_chain_id) VALUES ( +NOW(),NOW(),$1,'{}',false,$2,$3,0,0,0,0,0,0,0,0,0 ) RETURNING *`, NewEIP55Address(), &ocrKeyID, &transmitterAddress)) return spec } diff --git a/core/services/job/job_orm_test.go b/core/services/job/job_orm_test.go index fb34f92a060..b885bf4f83f 100644 --- a/core/services/job/job_orm_test.go +++ b/core/services/job/job_orm_test.go @@ -721,33 +721,9 @@ func TestORM_CreateJob_OCR_DuplicatedContractAddress(t *testing.T) { _, bridge := cltest.MustCreateBridge(t, db, cltest.BridgeOpts{}, config.Database()) _, bridge2 := cltest.MustCreateBridge(t, db, cltest.BridgeOpts{}, config.Database()) - spec := testspecs.GenerateOCRSpec(testspecs.OCRSpecParams{ - Name: "job1", - EVMChainID: testutils.FixtureChainID.String(), - DS1BridgeName: bridge.Name.String(), - DS2BridgeName: bridge2.Name.String(), - TransmitterAddress: address.Hex(), - }) - - jb, err := ocr.ValidatedOracleSpecToml(legacyChains, spec.Toml()) - require.NoError(t, err) - - // Default Chain Job - externalJobID := uuid.NullUUID{UUID: uuid.New(), Valid: true} - spec2 := testspecs.GenerateOCRSpec(testspecs.OCRSpecParams{ - Name: "job2", - EVMChainID: defaultChainID.String(), - DS1BridgeName: bridge.Name.String(), - DS2BridgeName: bridge2.Name.String(), - TransmitterAddress: address.Hex(), - JobID: externalJobID.UUID.String(), - }) - jb2, err := ocr.ValidatedOracleSpecToml(legacyChains, spec2.Toml()) - require.NoError(t, err) - // Custom Chain Job - externalJobID = uuid.NullUUID{UUID: uuid.New(), Valid: true} - spec3 := testspecs.GenerateOCRSpec(testspecs.OCRSpecParams{ + externalJobID := uuid.NullUUID{UUID: uuid.New(), Valid: true} + spec := testspecs.GenerateOCRSpec(testspecs.OCRSpecParams{ Name: "job3", EVMChainID: customChainID.String(), DS1BridgeName: bridge.Name.String(), @@ -755,53 +731,32 @@ func TestORM_CreateJob_OCR_DuplicatedContractAddress(t *testing.T) { TransmitterAddress: address.Hex(), JobID: externalJobID.UUID.String(), }) - jb3, err := ocr.ValidatedOracleSpecToml(legacyChains, spec3.Toml()) + jb, err := ocr.ValidatedOracleSpecToml(legacyChains, spec.Toml()) require.NoError(t, err) - t.Run("with legacy NULL chain id", func(t *testing.T) { - err = jobORM.CreateJob(&jb) - require.NoError(t, err) - _, err := db.ExecContext(testutils.Context(t), - "UPDATE ocr_oracle_specs o SET evm_chain_id=NULL FROM jobs j WHERE o.id = j.ocr_oracle_spec_id AND j.id=$1", jb.ID) - require.NoError(t, err) - - cltest.AssertCount(t, db, "ocr_oracle_specs", 1) - cltest.AssertCount(t, db, "jobs", 1) - - err = jobORM.CreateJob(&jb2) // try adding job for same contract with default chain id - require.Error(t, err) - assert.Equal(t, fmt.Sprintf("CreateJobFailed: a job with contract address %s already exists for chain ID %d", jb2.OCROracleSpec.ContractAddress, jb2.OCROracleSpec.EVMChainID.ToInt()), err.Error()) - - err = jobORM.CreateJob(&jb3) // Try adding job with custom chain id - require.Error(t, err) - assert.Equal(t, fmt.Sprintf("CreateJobFailed: a job with contract address %s already exists for chain ID %d", jb3.OCROracleSpec.ContractAddress, jb3.OCROracleSpec.EVMChainID.ToInt()), err.Error()) - }) - - require.NoError(t, jobORM.DeleteJob(jb.ID)) - t.Run("with a set chain id", func(t *testing.T) { - err = jobORM.CreateJob(&jb3) // Add job with custom chain id + err = jobORM.CreateJob(&jb) // Add job with custom chain id require.NoError(t, err) cltest.AssertCount(t, db, "ocr_oracle_specs", 1) cltest.AssertCount(t, db, "jobs", 1) externalJobID = uuid.NullUUID{UUID: uuid.New(), Valid: true} - spec3.JobID = externalJobID.UUID.String() - jb3a, err := ocr.ValidatedOracleSpecToml(legacyChains, spec3.Toml()) + spec.JobID = externalJobID.UUID.String() + jba, err := ocr.ValidatedOracleSpecToml(legacyChains, spec.Toml()) require.NoError(t, err) - err = jobORM.CreateJob(&jb3a) // Try to add duplicate job with default id + err = jobORM.CreateJob(&jba) // Try to add duplicate job with default id require.Error(t, err) - assert.Equal(t, fmt.Sprintf("CreateJobFailed: a job with contract address %s already exists for chain ID %s", jb3.OCROracleSpec.ContractAddress, defaultChainID.String()), err.Error()) + assert.Equal(t, fmt.Sprintf("CreateJobFailed: a job with contract address %s already exists for chain ID %s", jb.OCROracleSpec.ContractAddress, defaultChainID.String()), err.Error()) externalJobID = uuid.NullUUID{UUID: uuid.New(), Valid: true} - spec3.JobID = externalJobID.UUID.String() - jb4, err := ocr.ValidatedOracleSpecToml(legacyChains, spec3.Toml()) + spec.JobID = externalJobID.UUID.String() + jb2, err := ocr.ValidatedOracleSpecToml(legacyChains, spec.Toml()) require.NoError(t, err) - err = jobORM.CreateJob(&jb4) // Try to add duplicate job with custom id + err = jobORM.CreateJob(&jb2) // Try to add duplicate job with custom id require.Error(t, err) - assert.Equal(t, fmt.Sprintf("CreateJobFailed: a job with contract address %s already exists for chain ID %s", jb4.OCROracleSpec.ContractAddress, customChainID), err.Error()) + assert.Equal(t, fmt.Sprintf("CreateJobFailed: a job with contract address %s already exists for chain ID %s", jb2.OCROracleSpec.ContractAddress, customChainID), err.Error()) }) } @@ -1060,9 +1015,8 @@ func Test_FindJobs(t *testing.T) { func Test_FindJob(t *testing.T) { t.Parallel() - // Create a config with multiple EVM chains. The test fixtures already load a 1337 and the - // default EVM chain ID. Additional chains will need additional fixture statements to add - // a chain to evm_chains. + // Create a config with multiple EVM chains. The test fixtures already load 1337 + // Additional chains will need additional fixture statements to add a chain to evm_chains. config := configtest.NewGeneralConfig(t, func(c *chainlink.Config, s *chainlink.Secrets) { chainID := utils.NewBigI(1337) enabled := true @@ -1146,28 +1100,12 @@ func Test_FindJob(t *testing.T) { jobOCR2WithFeedID2.Name = null.StringFrom("new name") require.NoError(t, err) - // Create a job with the legacy null evm chain id. - jobWithNullChain, err := ocr.ValidatedOracleSpecToml(legacyChains, - testspecs.GenerateOCRSpec(testspecs.OCRSpecParams{ - JobID: uuid.New().String(), - ContractAddress: "0xB47f9a6D281B2A82F8692F8dE058E4249363A6fc", - TransmitterAddress: address.Hex(), - Name: "ocr legacy null chain id", - DS1BridgeName: bridge.Name.String(), - DS2BridgeName: bridge2.Name.String(), - }).Toml(), - ) - require.NoError(t, err) - err = orm.CreateJob(&job) require.NoError(t, err) err = orm.CreateJob(&jobSameAddress) require.NoError(t, err) - err = orm.CreateJob(&jobWithNullChain) - require.NoError(t, err) - err = orm.CreateJob(&jobOCR2) require.NoError(t, err) @@ -1178,11 +1116,6 @@ func Test_FindJob(t *testing.T) { err = orm.CreateJob(&jobOCR2WithFeedID2) require.NoError(t, err) - // Set the ChainID to null manually since we can't do this in the test helper - _, err = db.ExecContext(testutils.Context(t), - "UPDATE ocr_oracle_specs o SET evm_chain_id=NULL FROM jobs j WHERE o.id = j.ocr_oracle_spec_id AND j.id=$1", jobWithNullChain.ID) - require.NoError(t, err) - t.Run("by id", func(t *testing.T) { ctx, cancel := context.WithTimeout(testutils.Context(t), 5*time.Second) defer cancel() @@ -1222,24 +1155,6 @@ func Test_FindJob(t *testing.T) { require.ErrorIs(t, err, sql.ErrNoRows) }) - t.Run("by address with legacy null evm chain id", func(t *testing.T) { - jbID, err := orm.FindJobIDByAddress( - jobWithNullChain.OCROracleSpec.ContractAddress, - jobWithNullChain.OCROracleSpec.EVMChainID, - ) - require.NoError(t, err) - - assert.Equal(t, jobWithNullChain.ID, jbID) - - jbID, err = orm.FindJobIDByAddress( - jobWithNullChain.OCROracleSpec.ContractAddress, - utils.NewBig(nil), - ) - require.NoError(t, err) - - assert.Equal(t, jobWithNullChain.ID, jbID) - }) - t.Run("by address yet chain scoped", func(t *testing.T) { commonAddr := jobSameAddress.OCROracleSpec.ContractAddress diff --git a/core/store/migrate/migrations/0195_add_not_null_to_evm_chain_id_in_job_specs.go b/core/store/migrate/migrations/0195_add_not_null_to_evm_chain_id_in_job_specs.go new file mode 100644 index 00000000000..4b722c08ba2 --- /dev/null +++ b/core/store/migrate/migrations/0195_add_not_null_to_evm_chain_id_in_job_specs.go @@ -0,0 +1,49 @@ +package migrations + +import ( + "context" + "database/sql" + + "github.com/pkg/errors" + "github.com/pressly/goose/v3" +) + +func init() { + goose.AddMigrationContext(Up195, Down195) +} + +const ( + addNullConstraintsToSpecs = ` + ALTER TABLE direct_request_specs ALTER COLUMN evm_chain_id SET NOT NULL; + ALTER TABLE flux_monitor_specs ALTER COLUMN evm_chain_id SET NOT NULL; + ALTER TABLE ocr_oracle_specs ALTER COLUMN evm_chain_id SET NOT NULL; + ALTER TABLE keeper_specs ALTER COLUMN evm_chain_id SET NOT NULL; + ALTER TABLE vrf_specs ALTER COLUMN evm_chain_id SET NOT NULL; + ALTER TABLE blockhash_store_specs ALTER COLUMN evm_chain_id SET NOT NULL; + ALTER TABLE block_header_feeder_specs ALTER COLUMN evm_chain_id SET NOT NULL; + ` + + dropNullConstraintsFromSpecs = ` + ALTER TABLE direct_request_specs ALTER COLUMN evm_chain_id DROP NOT NULL; + ALTER TABLE flux_monitor_specs ALTER COLUMN evm_chain_id DROP NOT NULL; + ALTER TABLE ocr_oracle_specs ALTER COLUMN evm_chain_id DROP NOT NULL; + ALTER TABLE keeper_specs ALTER COLUMN evm_chain_id DROP NOT NULL; + ALTER TABLE vrf_specs ALTER COLUMN evm_chain_id DROP NOT NULL; + ALTER TABLE blockhash_store_specs ALTER COLUMN evm_chain_id DROP NOT NULL; + ALTER TABLE block_header_feeder_specs ALTER COLUMN evm_chain_id DROP NOT NULL; + ` +) + +// nolint +func Up195(ctx context.Context, tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, addNullConstraintsToSpecs) + return errors.Wrap(err, "failed to add null constraints") +} + +// nolint +func Down195(ctx context.Context, tx *sql.Tx) error { + if _, err := tx.ExecContext(ctx, dropNullConstraintsFromSpecs); err != nil { + return err + } + return nil +} diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d2e3dfada9b..65701c9de60 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [dev] +### Added + +- Temporary helper function for proper migration of evmChainID not null in specs, that fills in missing chainIDs. This needs to be removed after one version after everyone migrated properly. For proper migrations all nodes must upgrade sequentially without skipping this version. + ### Removed - Removed support for sending telemetry to the deprecated Explorer service. All nodes will have to remove `Explorer` related keys from TOML configuration and env vars.