Add LOOP Plugin support to CRIB devspace images (#12350)

* Introduce LOOP plugin support for devspace goreleaser builds

* Ignore core dockerfile

* update ci changesets filtering

* update paths-filter version

* split shared to its own check and add pnpm .tool-versions

* Address feedback

---------

Co-authored-by: chainchad <[email protected]>
Co-authored-by: Frank Zhu <[email protected]>

.github/workflows/changeset.yml

@@ -16,29 +16,36 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
-      - uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd # v3.0.0
+      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
         id: files-changed
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
+          predicate-quantifier: every
           filters: |
-            shared: &shared
+            shared:
               - common/**
+              - '!common/**/*_test.go'
               - plugins/**
+              - '!plugins/**/*_test.go'
             core:
-              - *shared
               - core/**
-            changeset:
+              - '!core/**/*_test.go'
+              - '!core/**/*.md'
+              - '!core/**/*.json'
+              - '!core/chainlink.goreleaser.Dockerfile'
+              - '!core/chainlink.Dockerfile'
+            core-changeset:
               - added: '.changeset/**'
       - name: Make a comment
         uses: unsplash/comment-on-pr@ffe8f97ccc63ce12c3c23c6885b169db67958d3b # v1.3.0
-        if: ${{ steps.files-changed.outputs.core == 'true' && steps.files-changed.outputs.changeset == 'false' }}
+        if: ${{ (steps.files-changed.outputs.core == 'true' || steps.files-changed.outputs.shared == 'true') && steps.files-changed.outputs.core-changeset == 'false' }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           msg: "I see you updated files related to core. Please run `pnpm changeset` to add a changeset."
           check_for_duplicate_msg: true
       - name: Check for new changeset
-        if: ${{ steps.files-changed.outputs.core == 'true' && steps.files-changed.outputs.changeset == 'false' }}
+        if: ${{ (steps.files-changed.outputs.core == 'true' || steps.files-changed.outputs.shared == 'true') && steps.files-changed.outputs.core-changeset == 'false' }}
         shell: bash
         run: |
           echo "Please run pnpm changeset to add a changeset."

.goreleaser.develop.yaml

@@ -63,6 +63,7 @@ dockers:
     goarch: amd64
     extra_files:
       - tmp/linux_amd64/libs
+      - tmp/linux_amd64/plugins
       - tools/bin/ldd_fix
     build_flag_templates:
       - "--platform=linux/amd64"
@@ -86,6 +87,7 @@ dockers:
     goarch: arm64
     extra_files:
       - tmp/linux_arm64/libs
+      - tmp/linux_arm64/plugins
       - tools/bin/ldd_fix
     build_flag_templates:
       - "--platform=linux/arm64"
@@ -109,6 +111,7 @@ dockers:
     goarch: amd64
     extra_files:
       - tmp/linux_amd64/libs
+      - tmp/linux_amd64/plugins
       - tools/bin/ldd_fix
     build_flag_templates:
       - "--platform=linux/amd64"
@@ -133,6 +136,7 @@ dockers:
     goarch: arm64
     extra_files:
       - tmp/linux_arm64/libs
+      - tmp/linux_arm64/plugins
       - tools/bin/ldd_fix
     build_flag_templates:
       - "--platform=linux/arm64"

.goreleaser.devspace.yaml

@@ -43,12 +43,17 @@ dockers:
     goarch: amd64
     extra_files:
       - tmp/linux_amd64/libs
+      - tmp/linux_amd64/plugins
       - tools/bin/ldd_fix
     build_flag_templates:
       - "--platform=linux/amd64"
       - "--pull"
       - "--build-arg=CHAINLINK_USER=chainlink"
       - "--build-arg=COMMIT_SHA={{ .FullCommit }}"
+      - "--build-arg=CL_MEDIAN_CMD=chainlink-feeds"
+      - "--build-arg=CL_MERCURY_CMD=chainlink-mercury"
+      - "--build-arg=CL_SOLANA_CMD=chainlink-solana"
+      - "--build-arg=CL_STARKNET_CMD=chainlink-starknet"
       - "--label=org.opencontainers.image.created={{ .Date }}"
       - "--label=org.opencontainers.image.description={{ .Env.IMAGE_LABEL_DESCRIPTION }}"
       - "--label=org.opencontainers.image.licenses={{ .Env.IMAGE_LABEL_LICENSES }}"

.tool-versions

@@ -1,6 +1,7 @@
 golang 1.21.7
 mockery 2.38.0
 nodejs 16.16.0
+pnpm 8.11.0
 postgres 13.3
 helm 3.10.3
 zig 0.11.0

charts/chainlink-cluster/templates/chainlink-cm.yaml

@@ -68,4 +68,4 @@ data:
   {{ else }}
   {{ end }}
 ---
-{{- end }}
+{{- end }}

core/chainlink.goreleaser.Dockerfile

@@ -16,9 +16,21 @@ RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
   && rm -rf /var/lib/apt/lists/*
 
 COPY ./chainlink /usr/local/bin/
+
 # Copy native libs if cgo is enabled
 COPY ./tmp/linux_${TARGETARCH}/libs /usr/local/bin/libs
 
+# Copy plugins and enable them
+COPY ./tmp/linux_${TARGETARCH}/plugins/* /usr/local/bin/
+# Allow individual plugins to be enabled by supplying their path 
+ARG CL_MEDIAN_CMD
+ARG CL_MERCURY_CMD
+ARG CL_SOLANA_CMD
+ARG CL_STARKNET_CMD
+ENV CL_MEDIAN_CMD=${CL_MEDIAN_CMD} \
+  CL_MERCURY_CMD=${CL_MERCURY_CMD} \
+  CL_SOLANA_CMD=${CL_SOLANA_CMD} \
+  CL_STARKNET_CMD=${CL_STARKNET_CMD}
 # Temp fix to patch correctly link the libwasmvm.so
 COPY ./tools/bin/ldd_fix /usr/local/bin/ldd_fix
 RUN chmod +x /usr/local/bin/ldd_fix

tools/bin/goreleaser_utils

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+set -x
+
 
 # get machine / kernel name
 _get_platform() {
@@ -58,19 +60,76 @@ before_hook() {
   cp -f "$wasmvm_lib_path_darwin_amd64" "$lib_path/darwin_amd64/libs"
   mkdir -p "$lib_path/darwin_arm64/libs"
   cp -f "$wasmvm_lib_path_darwin_arm64" "$lib_path/darwin_arm64/libs"
+
+  # MOVE PLUGINS HERE
+  gobin=$(go env GOPATH)/bin
+
+  install_local_plugins  "linux" "amd64" "$gobin"/linux_amd64
+  install_remote_plugins "linux" "amd64" "$gobin"/linux_amd64
+  mkdir -p "$lib_path/linux_amd64/plugins"
+  cp "$gobin"/linux_amd64/chainlink* "$lib_path/linux_amd64/plugins"
+
+  install_local_plugins  "linux" "arm64" "$gobin"/linux_arm64
+  install_remote_plugins "linux" "arm64" "$gobin"/linux_arm64
+  mkdir -p "$lib_path/linux_arm64/plugins"
+  cp "$gobin"/linux_arm64/chainlink* "$lib_path/linux_arm64/plugins"
+}
+
+install_local_plugins() {
+  local -r goos=$1
+  local -r goarch=$2
+  local -r gobin=$3
+
+  GOBIN=$gobin GOARCH=$goarch GOOS=$goos make install-medianpoc 
+  GOBIN=$gobin GOARCH=$goarch GOOS=$goos make install-install-ocr3-capability
+}
+
+get_remote_plugin_paths() {
+  plugins=(
+    "github.com/smartcontractkit/chainlink-solana|/pkg/solana/cmd/chainlink-solana"
+    "github.com/smartcontractkit/chainlink-starknet/relayer|/pkg/chainlink/cmd/chainlink-starknet"
+    "github.com/smartcontractkit/chainlink-feeds|/cmd/chainlink-feeds"
+    "github.com/smartcontractkit/chainlink-data-streams|/mercury/cmd/chainlink-mercury"
+  )
+
+  for plugin in "${plugins[@]}"; do
+   plugin_dep_name=$(echo "$plugin" | cut -d"|" -f1)
+   plugin_main=$(echo "$plugin" | cut -d"|" -f2)
+
+   full_plugin_path=$(go list -m -f "{{.Dir}}" "$plugin_dep_name")"$plugin_main"
+    echo "$full_plugin_path"
+  done
+}
+
+install_remote_plugins() {
+  local -r goos=$1
+  local -r goarch=$2
+  local -r gobin=$(go env GOPATH)/bin
+
+  for plugin in $(get_remote_plugin_paths); do
+    GOBIN=$gobin GOARCH=$goarch GOOS=$goos go install "$plugin"
+  done  
+
 }
 
 # binary build post hook
 # moves native libraries to binary libs directory
 build_post_hook() {
   local -r dist_path=$1
-  local -r lib_path=$dist_path/libs
+  local -r lib_dest_path=$dist_path/libs
   local -r platform=$2
   local -r arch=$3
+  local -r plugin_src_path=./tmp/${platform}_${arch}/plugins
+  local -r plugin_dest_path=$dist_path/plugins
+
   # COPY NATIVE LIBRARIES HERE
   local -r wasmvm_lib_path=$(_get_wasmvm_lib_path "$platform" "$arch")
-  mkdir -p "$lib_path"
-  cp "$wasmvm_lib_path" "$lib_path"
+  mkdir -p "$lib_dest_path"
+  cp "$wasmvm_lib_path" "$lib_dest_path"
+
+  # COPY PLUGINS HERE
+  mkdir -p "$plugin_dest_path"
+  cp -r "$plugin_src_path/." "$plugin_dest_path"
 }
 
 "$@"