Merge branch 'develop' into tt_922_automation_seth

.changeset/cyan-crabs-explode.md

@@ -0,0 +1,5 @@
+---
+"chainlink": minor
+---
+
+Add support for workflow jobs to Operator UI #wip #added

.changeset/pretty-flies-fold.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+cor/services/relay/evm/mercury: switch to sqlutil.DataStore #internal

.changeset/real-numbers-taste.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+core/services/functions: switch to sqlutil.DataStore #internal

.changeset/rich-jars-flow.md

@@ -0,0 +1,5 @@
+---
+"chainlink": patch
+---
+
+Narrowing topic, data_word indexes by adding (evm_chain_id, address, event_sig) to the index definition #db_update

.github/workflows/changeset.yml

@@ -10,17 +10,17 @@ jobs:
   changeset:
     env:
       TAGS: |
-        #nops : For any feature that is NOP facing and needs to be in the official Release Notes for the release.
-        #added : For any new functionality added.
-        #changed : For any change to the existing functionality. 
-        #removed : For any functionality/config that is removed.
-        #updated : For any functionality that is updated.
-        #deprecation_notice : For any upcoming deprecation functionality.
-        #breaking_change : For any functionality that requires manual action for the node to boot.
-        #db_update : For any feature that introduces updates to database schema.
-        #wip : For any change that is not ready yet and external communication about it should be held off till it is feature complete.
-        #bugfix - For bug fixes.
-        #internal - For changesets that need to be excluded from the final changelog.
+        - `#added` For any new functionality added.
+        - `#breaking_change` For any functionality that requires manual action for the node to boot.
+        - `#bugfix` For bug fixes.
+        - `#changed` For any change to the existing functionality. 
+        - `#db_update` For any feature that introduces updates to database schema.
+        - `#deprecation_notice` For any upcoming deprecation functionality.
+        - `#internal` For changesets that need to be excluded from the final changelog.
+        - `#nops` For any feature that is NOP facing and needs to be in the official Release Notes for the release.
+        - `#removed` For any functionality/config that is removed.
+        - `#updated` For any functionality that is updated.
+        - `#wip` For any change that is not ready yet and external communication about it should be held off till it is feature complete.
 
     # For security reasons, GITHUB_TOKEN is read-only on forks, so we cannot leave comments on PRs.
     # This check skips the job if it is detected we are running on a fork.
@@ -66,9 +66,7 @@ jobs:
         with:
           message: |
             I see you updated files related to `core`. Please run `pnpm changeset` in the root directory to add a changeset as well as in the text include at least one of the following tags:
-            ```
             ${{ env.TAGS }}
-            ```
           reactions: eyes
           comment_tag: changeset-core
 
@@ -111,9 +109,7 @@ jobs:
         with:
           message: |
             I see you added a changeset file but it does not contain a tag. Please edit the text include at least one of the following tags:
-            ```
             ${{ env.TAGS }}
-            ```
           reactions: eyes
           comment_tag: changeset-core-tags
 

.github/workflows/goreleaser-build-publish-develop.yml

@@ -26,7 +26,7 @@ jobs:
           role-duration-seconds: ${{ secrets.AWS_ROLE_DURATION_SECONDS }}
           aws-region: ${{ secrets.AWS_REGION }}
           mask-aws-account-id: true
-          role-session-name: goreleaser-build-publish-chainlink.push-chainlink-develop-goreleaser
+          role-session-name: goreleaser-build-publish-chainlink.push-develop
       - name: Build, sign, and publish image
         id: build-sign-publish
         uses: ./.github/actions/goreleaser-build-sign-publish

.github/workflows/helm-chart-publish.yml

@@ -25,7 +25,7 @@ jobs:
 
       - name: Get Github Token
         id: get-gh-token
-        uses: smartcontractkit/chainlink-github-actions/github-app-token-issuer@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11
+        uses: smartcontractkit/chainlink-github-actions/github-app-token-issuer@5874ff7211cf5a5a2670bb010fbff914eaaae138 # v2.3.12
         with:
           url: ${{ secrets.GATI_LAMBDA_FUNCTION_URL }}
 

.github/workflows/operator-ui-ci.yml

@@ -38,7 +38,7 @@ jobs:
 
       - name: Get Github Token
         id: get-gh-token
-        uses: smartcontractkit/chainlink-github-actions/github-app-token-issuer@5dd916d08c03cb5f9a97304f4f174820421bb946 # v2.3.11
+        uses: smartcontractkit/chainlink-github-actions/github-app-token-issuer@5874ff7211cf5a5a2670bb010fbff914eaaae138 # v2.3.12
         with:
           url: ${{ secrets.AWS_INFRA_RELENG_TOKEN_ISSUER_LAMBDA_URL }}
 

.gitignore

@@ -98,3 +98,5 @@ override*.toml
 
 # Pythin venv
 .venv/
+
+ocr_soak_report.csv

charts/chainlink-cluster/README.md

@@ -151,6 +151,9 @@ We are using [Grabana](https://github.com/K-Phoen/grabana) lib to create dashboa
 
 You can also select dashboard platform in `INFRA_PLATFORM` either `kubernetes` or `docker`
 
+You can select the dashboard panels with `PANELS_INCLUDED` which is a list of panel names separated by comma
+If you don't specify it will include core panels by default
+
 ```
 export LOKI_TENANT_ID=promtail
 export LOKI_URL=...
@@ -159,7 +162,7 @@ export GRAFANA_TOKEN=...
 export PROMETHEUS_DATA_SOURCE_NAME=Thanos
 export LOKI_DATA_SOURCE_NAME=Loki
 export INFRA_PLATFORM=kubernetes
-export GRAFANA_FOLDER=CRIB
+export GRAFANA_FOLDER=DashboardCoreDebug
 export DASHBOARD_NAME=CL-Cluster
 
 devspace run dashboard_deploy

charts/chainlink-cluster/dashboard/cmd/delete.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	lib "github.com/smartcontractkit/chainlink/dashboard-lib"
+)
+
+func main() {
+	cfg := lib.ReadEnvDeployOpts()
+	db := lib.NewDashboard(cfg.Name, cfg, nil)
+	err := db.Delete()
+	if err != nil {
+		lib.L.Fatal().Err(err).Msg("failed to delete the dashboard")
+	}
+	lib.L.Info().
+		Str("Name", db.Name).
+		Str("GrafanaURL", db.DeployOpts.GrafanaURL).
+		Str("GrafanaFolder", db.DeployOpts.GrafanaFolder).
+		Msg("Dashboard deleted")
+}

charts/chainlink-cluster/dashboard/cmd/deploy.go

@@ -3,31 +3,48 @@ package main
 import (
 	"github.com/K-Phoen/grabana/dashboard"
 	lib "github.com/smartcontractkit/chainlink/dashboard-lib"
+	atlas_don "github.com/smartcontractkit/chainlink/dashboard-lib/atlas-don"
 	core_don "github.com/smartcontractkit/chainlink/dashboard-lib/core-don"
 	k8spods "github.com/smartcontractkit/chainlink/dashboard-lib/k8s-pods"
 	waspdb "github.com/smartcontractkit/wasp/dashboard"
-)
-
-const (
-	DashboardName = "Chainlink Cluster (DON)"
+	"strings"
 )
 
 func main() {
 	cfg := lib.ReadEnvDeployOpts()
-	db := lib.NewDashboard(DashboardName, cfg,
+	db := lib.NewDashboard(cfg.Name, cfg,
 		[]dashboard.Option{
 			dashboard.AutoRefresh("10s"),
 			dashboard.Tags([]string{"generated"}),
 		},
 	)
-	db.Add(
-		core_don.New(
-			core_don.Props{
-				PrometheusDataSource: cfg.DataSources.Prometheus,
-				PlatformOpts:         core_don.PlatformPanelOpts(cfg.Platform),
-			},
-		),
-	)
+	if len(cfg.PanelsIncluded) == 0 || cfg.PanelsIncluded["core"] {
+		db.Add(
+			core_don.New(
+				core_don.Props{
+					PrometheusDataSource: cfg.DataSources.Prometheus,
+					PlatformOpts:         core_don.PlatformPanelOpts(cfg.Platform),
+				},
+			),
+		)
+		// TODO: refactor as a component later
+		addWASPRows(db, cfg)
+	}
+	if cfg.PanelsIncluded["ocr"] || cfg.PanelsIncluded["ocr2"] || cfg.PanelsIncluded["ocr3"] {
+		for key := range cfg.PanelsIncluded {
+			if strings.Contains(key, "ocr") {
+				db.Add(
+					atlas_don.New(
+						atlas_don.Props{
+							PrometheusDataSource: cfg.DataSources.Prometheus,
+							PlatformOpts:         atlas_don.PlatformPanelOpts(cfg.Platform, key),
+							OcrVersion:           key,
+						},
+					),
+				)
+			}
+		}
+	}
 	if cfg.Platform == "kubernetes" {
 		db.Add(
 			k8spods.New(
@@ -38,13 +55,11 @@ func main() {
 			),
 		)
 	}
-	// TODO: refactor as a component later
-	addWASPRows(db, cfg)
 	if err := db.Deploy(); err != nil {
 		lib.L.Fatal().Err(err).Msg("failed to deploy the dashboard")
 	}
 	lib.L.Info().
-		Str("Name", DashboardName).
+		Str("Name", db.Name).
 		Str("GrafanaURL", db.DeployOpts.GrafanaURL).
 		Str("GrafanaFolder", db.DeployOpts.GrafanaFolder).
 		Msg("Dashboard deployed")

core/services/chainlink/application.go

@@ -310,10 +310,10 @@ func NewApplication(opts ApplicationOpts) (Application, error) {
 	var (
 		pipelineORM    = pipeline.NewORM(sqlxDB, globalLogger, cfg.Database(), cfg.JobPipeline().MaxSuccessfulRuns())
 		bridgeORM      = bridges.NewORM(sqlxDB)
-		mercuryORM     = mercury.NewORM(sqlxDB, globalLogger, cfg.Database())
+		mercuryORM     = mercury.NewORM(opts.DB)
 		pipelineRunner = pipeline.NewRunner(pipelineORM, bridgeORM, cfg.JobPipeline(), cfg.WebServer(), legacyEVMChains, keyStore.Eth(), keyStore.VRF(), globalLogger, restrictedHTTPClient, unrestrictedHTTPClient)
 		jobORM         = job.NewORM(sqlxDB, pipelineORM, bridgeORM, keyStore, globalLogger, cfg.Database())
-		txmORM         = txmgr.NewTxStore(sqlxDB, globalLogger)
+		txmORM         = txmgr.NewTxStore(opts.DB, globalLogger)
 		streamRegistry = streams.NewRegistry(globalLogger, pipelineRunner)
 	)
 
@@ -437,6 +437,7 @@ func NewApplication(opts ApplicationOpts) (Application, error) {
 
 		delegates[job.OffchainReporting2] = ocr2.NewDelegate(
 			sqlxDB,
+			opts.DB,
 			jobORM,
 			bridgeORM,
 			mercuryORM,

core/services/functions/listener.go

@@ -23,7 +23,6 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/services/job"
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/functions/config"
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/threshold"
-	"github.com/smartcontractkit/chainlink/v2/core/services/pg"
 	evmrelayTypes "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/types"
 	"github.com/smartcontractkit/chainlink/v2/core/services/s4"
 	"github.com/smartcontractkit/chainlink/v2/core/services/synchronization/telem"
@@ -270,7 +269,7 @@ func (l *functionsListener) setError(ctx context.Context, requestId RequestID, e
 		promRequestComputationError.WithLabelValues(l.contractAddressHex).Inc()
 	}
 	readyForProcessing := errType != INTERNAL_ERROR
-	if err := l.pluginORM.SetError(requestId, errType, errBytes, time.Now(), readyForProcessing, pg.WithParentCtx(ctx)); err != nil {
+	if err := l.pluginORM.SetError(ctx, requestId, errType, errBytes, time.Now(), readyForProcessing); err != nil {
 		l.logger.Errorw("call to SetError failed", "requestID", formatRequestId(requestId), "err", err)
 	}
 }
@@ -321,7 +320,7 @@ func (l *functionsListener) HandleOffchainRequest(ctx context.Context, request *
 		CoordinatorContractAddress: &senderAddr,
 		OnchainMetadata:            []byte(OffchainRequestMarker),
 	}
-	if err := l.pluginORM.CreateRequest(newReq, pg.WithParentCtx(ctx)); err != nil {
+	if err := l.pluginORM.CreateRequest(ctx, newReq); err != nil {
 		if errors.Is(err, ErrDuplicateRequestID) {
 			l.logger.Warnw("HandleOffchainRequest: received duplicate request ID", "requestID", formatRequestId(requestId), "err", err)
 		} else {
@@ -348,7 +347,7 @@ func (l *functionsListener) handleOracleRequestV1(request *evmrelayTypes.OracleR
 		CoordinatorContractAddress: &request.CoordinatorContract,
 		OnchainMetadata:            request.OnchainMetadata,
 	}
-	if err := l.pluginORM.CreateRequest(newReq, pg.WithParentCtx(ctx)); err != nil {
+	if err := l.pluginORM.CreateRequest(ctx, newReq); err != nil {
 		if errors.Is(err, ErrDuplicateRequestID) {
 			l.logger.Warnw("handleOracleRequestV1: received a log with duplicate request ID", "requestID", formatRequestId(request.RequestId), "err", err)
 		} else {
@@ -450,7 +449,7 @@ func (l *functionsListener) handleRequest(ctx context.Context, requestID Request
 		promRequestComputationSuccess.WithLabelValues(l.contractAddressHex).Inc()
 		promComputationResultSize.WithLabelValues(l.contractAddressHex).Set(float64(len(computationResult)))
 		l.logger.Debugw("saving computation result", "requestID", requestIDStr)
-		if err2 := l.pluginORM.SetResult(requestID, computationResult, time.Now(), pg.WithParentCtx(ctx)); err2 != nil {
+		if err2 := l.pluginORM.SetResult(ctx, requestID, computationResult, time.Now()); err2 != nil {
 			l.logger.Errorw("call to SetResult failed", "requestID", requestIDStr, "err", err2)
 			return err2
 		}
@@ -464,7 +463,7 @@ func (l *functionsListener) handleOracleResponseV1(response *evmrelayTypes.Oracl
 
 	ctx, cancel := l.getNewHandlerContext()
 	defer cancel()
-	if err := l.pluginORM.SetConfirmed(response.RequestId, pg.WithParentCtx(ctx)); err != nil {
+	if err := l.pluginORM.SetConfirmed(ctx, response.RequestId); err != nil {
 		l.logger.Errorw("setting CONFIRMED state failed", "requestID", formatRequestId(response.RequestId), "err", err)
 	}
 	promRequestConfirmed.WithLabelValues(l.contractAddressHex).Inc()
@@ -486,7 +485,7 @@ func (l *functionsListener) timeoutRequests() {
 		case <-ticker.C:
 			cutoff := time.Now().Add(-(time.Duration(timeoutSec) * time.Second))
 			ctx, cancel := l.getNewHandlerContext()
-			ids, err := l.pluginORM.TimeoutExpiredResults(cutoff, batchSize, pg.WithParentCtx(ctx))
+			ids, err := l.pluginORM.TimeoutExpiredResults(ctx, cutoff, batchSize)
 			cancel()
 			if err != nil {
 				l.logger.Errorw("error when calling FindExpiredResults", "err", err)
@@ -531,7 +530,7 @@ func (l *functionsListener) pruneRequests() {
 		case <-ticker.C:
 			ctx, cancel := l.getNewHandlerContext()
 			startTime := time.Now()
-			nTotal, nPruned, err := l.pluginORM.PruneOldestRequests(maxStoredRequests, batchSize, pg.WithParentCtx(ctx))
+			nTotal, nPruned, err := l.pluginORM.PruneOldestRequests(ctx, maxStoredRequests, batchSize)
 			cancel()
 			elapsedMillis := time.Since(startTime).Milliseconds()
 			if err != nil {