diff --git a/.github/actionlint.yml b/.github/actionlint.yml new file mode 100644 index 0000000..d4a0bdb --- /dev/null +++ b/.github/actionlint.yml @@ -0,0 +1,6 @@ +self-hosted-runner: + - ubuntu20.04-4cores-16GB + - ubuntu20.04-8cores-32GB + - ubuntu20.04-16cores-64GB + - ubuntu20.04-32cores-128GB + - ubuntu20.04-64cores-256GB \ No newline at end of file diff --git a/.github/actions/setup-postgres/action.yml b/.github/actions/setup-postgres/action.yml deleted file mode 100644 index f683934..0000000 --- a/.github/actions/setup-postgres/action.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Setup Postgresql -description: Setup postgres docker container via docker-compose, allowing usage of a custom command, see https://github.com/orgs/community/discussions/26688 -runs: - using: composite - steps: - - name: Start postgres service - run: docker compose up -d - shell: bash - working-directory: ./.github/actions/setup-postgres - - name: Wait for postgres service to be healthy - run: ./wait-for-healthy-postgres.sh - shell: bash - working-directory: ./.github/actions/setup-postgres diff --git a/.github/actions/setup-postgres/wait-for-healthy-postgres.sh b/.github/actions/setup-postgres/wait-for-healthy-postgres.sh deleted file mode 100755 index f53e186..0000000 --- a/.github/actions/setup-postgres/wait-for-healthy-postgres.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -RETRIES=10 - -until [ $RETRIES -eq 0 ]; do - if docker compose ps postgres --status running --format json | jq >/dev/null -e 'if (.Health == "healthy") then true else false end'; then - exit 0 - fi - echo "Waiting for postgres server, $((RETRIES--)) remaining attempts..." - sleep 2 -done -exit 1 diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..6566764 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,14 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: '/' + schedule: + interval: daily + - package-ecosystem: gomod + directory: '/' + schedule: + interval: daily + - package-ecosystem: docker + directory: '/' + schedule: + interval: daily \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index d8accad..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,138 +0,0 @@ -name: CI -on: - push: - branches: [main] - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - changes: - runs-on: ubuntu-latest - outputs: - src: ${{ steps.changes.outputs.src }} - steps: - - name: Checkout the repo - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 - id: changes - with: - filters: | - src: - - '**/*.go' - - '**/go.mod' - - '**/go.sum' - - '.golangci.yml' - - '.github/workflows/ci.yml' - - golangci-lint: - runs-on: ubuntu-latest - needs: [changes] - steps: - - uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab # v3.5.2 - with: - fetch-depth: 0 - - uses: actions/setup-go@v4 - with: - go-version-file: 'go.mod' - cache: false - - uses: smartcontractkit/golangci-lint-action@54ab6c5f11d66a92d14c3f7cc41ea13f676644bd # feature/multiple-output-formats-backup - if: needs.changes.outputs.src == 'true' - with: - version: v1.51.1 - only-new-issues: ${{github.event.schedule == ''}} # show only new issues, unless it's a scheduled run - allow-extra-out-format-args: true - args: --out-format checkstyle:golangci-lint-report.xml - - name: Print lint report artifact - if: always() - run: test -f golangci-lint-report.xml && cat golangci-lint-report.xml || true - - name: Store lint report artifact - if: always() - uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0 - with: - name: golangci-lint-report - path: golangci-lint-report.xml - - name: Collect Metrics - if: always() - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@90fcbaac8ebf86da9c4d55dba24f6fe3029f0e0b - with: - basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_CLOUD_HOST }} - this-job-name: golangci-lint - continue-on-error: true - - go-tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Setup Go - uses: actions/setup-go@v3 - with: - go-version-file: "go.mod" - cache: true - - name: Download Go vendor packages - run: go mod download - - name: Setup Postgres - uses: ./.github/actions/setup-postgres - - name: Build - run: go build -v ./... - - name: Run tests - run: ./tools/bin/go_feeds_tests - - name: Upload Go test results - if: always() - uses: actions/upload-artifact@v3 - with: - name: go-test-results - path: | - ./output.txt - ./coverage.txt - - name: Collect Metrics - if: always() - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@90fcbaac8ebf86da9c4d55dba24f6fe3029f0e0b - with: - basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_CLOUD_HOST }} - this-job-name: go-tests - continue-on-error: true - - sonarqube: - needs: [changes, golangci-lint, go-tests] - if: ${{ always() }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 # fetches all history for all tags and branches to provide more metadata for sonar reports - - name: Download all workflow run artifacts - uses: actions/download-artifact@9782bd6a9848b53b110e712e20e42d89988822b7 # v3.0.1 - - name: Set SonarQube Report Paths - id: sonarqube_report_paths - shell: bash - run: | - echo "sonarqube_tests_report_paths=$(find go-test-results -name output.txt | paste -sd "," -)" >> $GITHUB_OUTPUT - echo "sonarqube_coverage_report_paths=$(find go-test-results -name coverage.txt | paste -sd "," -)" >> $GITHUB_OUTPUT - - name: SonarQube scan with lint - if: needs.changes.outputs.src == 'true' - uses: sonarsource/sonarqube-scan-action@a6ba0aafc293e03de5437af7edbc97f7d3ebc91a # v1.2.0 - with: - args: > - -Dsonar.go.tests.reportPaths=${{ steps.sonarqube_report_paths.outputs.sonarqube_tests_report_paths }} - -Dsonar.go.coverage.reportPaths=${{ steps.sonarqube_report_paths.outputs.sonarqube_coverage_report_paths }} - -Dsonar.go.golangci-lint.reportPaths=golangci-lint-report/golangci-lint-report.xml - env: - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} - - name: SonarQube scan without lint - if: needs.changes.outputs.src != 'true' - uses: sonarsource/sonarqube-scan-action@a6ba0aafc293e03de5437af7edbc97f7d3ebc91a # v1.2.0 - with: - args: > - -Dsonar.go.tests.reportPaths=${{ steps.sonarqube_report_paths.outputs.sonarqube_tests_report_paths }} - -Dsonar.go.coverage.reportPaths=${{ steps.sonarqube_report_paths.outputs.sonarqube_coverage_report_paths }} - env: - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml deleted file mode 100644 index dbf0889..0000000 --- a/.github/workflows/dependency-check.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: Dependency Vulnerability Check - -on: - push: - -jobs: - changes: - name: Detect changes - runs-on: ubuntu-latest - outputs: - changes: ${{ steps.changes.outputs.src }} - steps: - - name: Checkout the repo - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 - id: changes - with: - filters: | - src: - - '**/*go.sum' - - '**/*go.mod' - - '.github/workflows/dependency-check.yml' - Go: - runs-on: ubuntu-latest - needs: [changes] - steps: - - name: Check out code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - - name: Set up Go - if: needs.changes.outputs.src == 'true' - uses: actions/setup-go@v4 - with: - go-version-file: 'go.mod' - id: go - - - name: Write Go Modules list - if: needs.changes.outputs.src == 'true' - run: go list -json -m all > go.list - - - name: Check vulnerabilities - if: needs.changes.outputs.src == 'true' - uses: sonatype-nexus-community/nancy-github-action@main - with: - nancyVersion: "v1.0.39" - - - name: Collect Metrics - if: always() - id: collect-gha-metrics - uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2 - with: - basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} - hostname: ${{ secrets.GRAFANA_CLOUD_HOST }} - this-job-name: Go - continue-on-error: true diff --git a/.github/workflows/pull-request-master.yml b/.github/workflows/pull-request-master.yml new file mode 100644 index 0000000..616aea8 --- /dev/null +++ b/.github/workflows/pull-request-master.yml @@ -0,0 +1,183 @@ +name: pull-request-master + +on: + merge_group: + pull_request: + branches: + - master +# Only run 1 of this workflow at a time per PR +concurrency: + group: chainlink-feeds-${{ github.ref }} + cancel-in-progress: true + +env: + PACKAGES: "median" + +jobs: + init: + runs-on: ubuntu-latest + outputs: + matrix_packages: ${{ steps.set-matrix-packages.outputs.matrix_packages }} + lint_args_packages: ${{ steps.set-matrix-packages.outputs.lint_args_packages }} + steps: + - name: Checkout code + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + - name: Set matrix packages + id: set-matrix-packages + shell: bash + env: + PACKAGES: ${{ env.PACKAGES }} + run: | + matrix_packages=$(echo "${PACKAGES}" | jq -R 'split(",")' | tr -d "\n\t") + echo "matrix_packages=${matrix_packages}" | tee -a "${GITHUB_OUTPUT}" + - name: Set lint args packages + id: set-lint-args-packages + shell: bash + env: + PACKAGES: ${{ env.PACKAGES }} + # Convert "producer,reports_consumer" to "./producer/... ./reports_consumer/..." + run: echo "lint_args_packages=$(echo "./$(echo $PACKAGES | sed 's/,/\/... .\//g;s/$/\/.../')")" | tee -a "${GITHUB_OUTPUT}" + + ci-lint: + runs-on: ubuntu-latest + needs: [init] + permissions: + id-token: write + contents: read + actions: read + steps: + - name: ci-lint + uses: smartcontractkit/.github/actions/ci-lint-go@main + with: + # grafana inputs + metrics-job-name: ci-lint + gc-basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} + gc-host: ${{ secrets.GRAFANA_CLOUD_HOST }} + # env inputs + use-env-files: "true" + env-files: ./tools/env/ci.env + # go inputs + use-go-cache: true + go-cache-dep-path: "**/go.sum" + go-version-file: go.mod + golangci-lint-version: "v1.53.2" + golangci-lint-args: --out-format checkstyle:golangci-lint-report.xml ${{ needs.init.outputs.lint_args_packages }} + + ci-lint-misc: + runs-on: ubuntu-latest + steps: + - name: ci-lint-misc + uses: smartcontractkit/.github/actions/ci-lint-misc@main + with: + # grafana inputs + metrics-job-name: ci-lint-misc + gc-basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} + gc-host: ${{ secrets.GRAFANA_CLOUD_HOST }} + + ci-test: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + actions: read + steps: + - name: ci-test + uses: smartcontractkit/.github/actions/ci-test-go@main + with: + # grafana inputs + metrics-job-name: ci-test + gc-basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} + gc-host: ${{ secrets.GRAFANA_CLOUD_HOST }} + # docker inputs + use-docker-compose: "true" + docker-compose-workdir: ./tools/docker/setup-postgres + # env inputs + use-env-files: "true" + env-files: ./tools/env/ci.env + # go inputs + use-go-cache: "true" + go-cache-dep-path: "**/go.sum" + go-version-file: go.mod + go-test-cmd: make test-ci + + ci-build-artifacts-matrix: + needs: [init] + runs-on: ubuntu20.04-16cores-64GB + strategy: + fail-fast: false + matrix: + package: ${{ fromJson(needs.init.outputs.matrix_packages) }} + permissions: + id-token: write + contents: read + actions: read + steps: + - name: ci-build-artifacts-matrix (${{ matrix.package }}) + uses: smartcontractkit/.github/actions/cicd-build-publish-artifacts-go@main + with: + # general inputs + app-name: chainlink-feeds-${{ matrix.package }} + publish: "true" + # grafana inputs + metrics-job-name: ci-build-artifacts-matrix (${{ matrix.package }}) + gc-basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} + gc-host: ${{ secrets.GRAFANA_CLOUD_HOST }} + # aws inputs + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_ARN }} + aws-account-number: ${{ secrets.AWS_ACCOUNT_NUMBER_PROD }} + # golang inputs + go-version-file: go.mod + # goreleaser inputs + goreleaser-args: "--nightly --config ./${{ matrix.package }}/.goreleaser.ci.yaml" + goreleaser-dist: goreleaser-pro + goreleaser-key: ${{ secrets.GORELEASER_KEY }} + # zig inputs + use-zig: "true" + zig-version: "0.11.0" + # docker inputs + docker-registry: aws + docker-image-tag: devel + - name: Save GoReleaser artifacts/metadata + uses: actions/upload-artifact@v3 + with: + name: goreleaser_${{ matrix.package }} + path: | + dist/artifacts.json + dist/metadata.json + + ci-build-artifacts: + runs-on: ubuntu-latest + needs: [ci-build-artifacts-matrix] + if: always() + steps: + - name: Check results + if: needs.ci-build-artifacts-matrix.result != 'success' + run: exit 1 + - name: Collect metrics + if: always() + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2 + with: + basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_CLOUD_HOST }} + this-job-name: ci-build-artifacts + continue-on-error: true + + ci-sonarqube: + needs: [ci-lint, ci-test] + if: ${{ always() }} + runs-on: ubuntu-latest + steps: + - name: ci-sonarqube + uses: smartcontractkit/.github/actions/ci-sonarqube@main + with: + # grafana inputs + metrics-job-name: ci-sonarqube + gc-basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} + gc-host: ${{ secrets.GRAFANA_CLOUD_HOST }} + # sonarqube inputs + sonar-token: ${{ secrets.SONAR_TOKEN }} + sonar-host-url: ${{ secrets.SONAR_HOST_URL }} \ No newline at end of file diff --git a/.github/workflows/push-master.yml b/.github/workflows/push-master.yml new file mode 100644 index 0000000..0fab5d5 --- /dev/null +++ b/.github/workflows/push-master.yml @@ -0,0 +1,168 @@ +name: push-master + +on: + push: + branches: + - master + +env: + PACKAGES: "median" + +jobs: + init: + runs-on: ubuntu-latest + outputs: + matrix_packages: ${{ steps.set-matrix-packages.outputs.matrix_packages }} + lint_args_packages: ${{ steps.set-matrix-packages.outputs.lint_args_packages }} + steps: + - name: Checkout code + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Set matrix packages + id: set-matrix-packages + shell: bash + env: + PACKAGES: ${{ env.PACKAGES }} + run: | + matrix_packages=$(echo "${PACKAGES}" | jq -R 'split(",")' | tr -d "\n\t") + echo "matrix_packages=${matrix_packages}" | tee -a "${GITHUB_OUTPUT}" + - name: Set lint args packages + id: set-lint-args-packages + shell: bash + env: + PACKAGES: ${{ env.PACKAGES }} + # Convert "producer,reports_consumer" to "./producer/... ./reports_consumer/..." + run: echo "lint_args_packages=$(echo "./$(echo $PACKAGES | sed 's/,/\/... .\//g;s/$/\/.../')")" | tee -a "${GITHUB_OUTPUT}" + + ci-lint: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + actions: read + steps: + - name: ci-lint + uses: smartcontractkit/.github/actions/ci-lint-go@main + with: + # grafana inputs + metrics-job-name: ci-lint + gc-basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} + gc-host: ${{ secrets.GRAFANA_CLOUD_HOST }} + # env inputs + use-env-files: "true" + env-files: ./tools/env/ci.env + # go inputs + use-go-cache: true + go-cache-dep-path: "**/go.sum" + go-version-file: go.mod + golangci-lint-version: "v1.53.2" + golangci-lint-args: --out-format checkstyle:golangci-lint-report.xml ${{ needs.init.outputs.lint_args_packages }} + + ci-test: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + actions: read + steps: + - name: ci-test + uses: smartcontractkit/.github/actions/ci-test-go@main + with: + # grafana inputs + metrics-job-name: ci-test + gc-basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} + gc-host: ${{ secrets.GRAFANA_CLOUD_HOST }} + # docker inputs + use-docker-compose: "true" + docker-compose-workdir: ./tools/docker/setup-postgres + # env inputs + use-env-files: "true" + env-files: ./tools/env/ci.env + # go inputs + use-go-cache: "true" + go-cache-dep-path: "**/go.sum" + go-version-file: go.mod + go-test-cmd: make test-ci + + cicd-build-publish-artifacts-dev-matrix: + needs: [init] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + package: ${{ fromJson(needs.init.outputs.matrix_packages) }} + permissions: + id-token: write + contents: write + actions: read + steps: + - name: cicd-build-publish-artifacts-dev-matrix (${{ matrix.package }}) + uses: smartcontractkit/.github/actions/cicd-build-publish-artifacts-go@main + with: + # general inputs + app-name: chainlink-feeds-${{ matrix.package }} + publish: "true" + # grafana inputs + metrics-job-name: cicd-build-publish-artifacts-dev-matrix (${{ matrix.package }}) + gc-basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} + gc-host: ${{ secrets.GRAFANA_CLOUD_HOST }} + # aws inputs + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_ARN }} + aws-account-number: ${{ secrets.AWS_ACCOUNT_NUMBER_PROD }} + # gati inputs + use-gati: "true" + aws-role-arn-gati: ${{ secrets.AWS_OIDC_IAM_ROLE_ARN_GATI }} + aws-lambda-url-gati: ${{ secrets.AWS_LAMBDA_URL_GATI }} + # golang inputs + go-version-file: go.mod + # goreleaser inputs + goreleaser-args: "--nightly --config ./${{ matrix.package }}/.goreleaser.ci.yaml" + goreleaser-dist: goreleaser-pro + goreleaser-key: ${{ secrets.GORELEASER_KEY }} + # zig inputs + use-zig: "true" + zig-version: "0.11.0" + # docker inputs + docker-registry: aws + docker-image-tag: devel + - name: Save GoReleaser artifacts/metadata + uses: actions/upload-artifact@v3 + with: + name: goreleaser_${{ matrix.package }} + path: | + dist/artifacts.json + dist/metadata.json + + cicd-build-publish-artifacts-dev: + runs-on: ubuntu-latest + needs: [cicd-build-publish-artifacts-dev-matrix] + if: always() + steps: + - name: Check results + if: needs.cicd-build-publish-artifacts-dev-matrix.result != 'success' + run: exit 1 + - name: Collect metrics + if: always() + id: collect-gha-metrics + uses: smartcontractkit/push-gha-metrics-action@d1618b772a97fd87e6505de97b872ee0b1f1729a # v2.0.2 + with: + basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} + hostname: ${{ secrets.GRAFANA_CLOUD_HOST }} + this-job-name: cicd-build-publish-artifacts-dev + continue-on-error: true + + ci-sonarqube: + needs: [ci-lint, ci-test] + if: ${{ always() }} + runs-on: ubuntu-latest + steps: + - name: ci-sonarqube + uses: smartcontractkit/.github/actions/ci-sonarqube@main + with: + # grafana inputs + metrics-job-name: ci-sonarqube + gc-basic-auth: ${{ secrets.GRAFANA_CLOUD_BASIC_AUTH }} + gc-host: ${{ secrets.GRAFANA_CLOUD_HOST }} + # sonarqube inputs + sonar-token: ${{ secrets.SONAR_TOKEN }} + sonar-host-url: ${{ secrets.SONAR_HOST_URL }} diff --git a/tools/bin/scrub_logs b/tools/bin/scrub_logs new file mode 100755 index 0000000..03cfe93 --- /dev/null +++ b/tools/bin/scrub_logs @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +set -e + +# Path to output.txt file produced from `go test ... | tee $OUTPUT_FILE` +OUTPUT_FILE=${1:-output.txt} +if [ ! -f "$OUTPUT_FILE" ]; then + printf "scrub_logs script error: No output file found at '$OUTPUT_FILE'" + exit 1 +fi + +# Scan for sensitive strings in the -v verbose go test output logs +# with constants from `core/internal/cltest/cltest.go` +declare -a arr=( + # General secret const strings and environment config + "2d25e62eaf9143e993acaf48691564b2" # APIKey of the fixture API user + "1eCP/w0llVkchejFaoBpfIGaLRxZK54lTXBCT22YLW+pdzE4Fafy/XO5LoJ2uwHi" # APISecret of the fixture API user + "1eCP%2fw0llVkchejFaoBpfIGaLRxZK54lTXBCT22YLW%2bpdzE4Fafy%2fXO5LoJ2uwHi" # URL Encoded + "T.tLHkcmwePT/p,]sYuntjwHKAsrhm#4eRs4LuKHwvHejWYAC2JP4M8HimwgmbaZ" # Tooling secret + "T.tLHkcmwePT%2fp%2c%5dsYuntjwHKAsrhm%234eRs4LuKHwvHejWYAC2JP4M8HimwgmbaZ" # URL Encoded + "16charlengthp4SsW0rD1!@#_" # Test user password + "16charlengthp4SsW0rD1%21%40%23_" # URL Encoded + "wss://" # Possible credentials + "psql://" # Possible DB connection string + "psql%3a%2f%2f" # URL Encoded + "postgresql://" # Possible DB connection string + "postgresql%3a%2f%2f" # URL Encoded + + # Functionality for key exports (JSON) are in the wallet v3 format, encrypted with the cltest.Password value. + # The actual value of the secret of these test seeds are 0x1. Secret keys are not as easily accessible and .String() methods are hardcoded to omit them + # but additionally check the following representations of that secret for %#v logs of that struct + "nat{0x1}" # Logged key struct could leak secret value, ex. &secp256k1.secp256k1Scalar{neg:false, abs:big.nat{0x1}} + "nat%7B0x1%7D" # URL Encoded + "nat{0x2}" + "nat%7B0x2%7D" + "134287936" # Int representation of internal Ed25519PrivateKey + # secret used by keys with KeyBigIntSeed seed (1) + "CAESQAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAzswVB9wd3XKVlRwpCIjwla25BE0bc9aW5t8GXWg71Pw=" # Base64 representation of internal Ed25519PrivateKey + # secret used by keys with KeyBigIntSeed seed (1) + "CAESQAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAzswVB9wd3XKVlRwpCIjwla25BE0bc9aW5t8GXWg71Pw%3D" # URL Encoded + "AQ==" + "AQ%3D%3D" + "Ag%3D%3D" +) + +declare -a testarr=( + "wss://foo.bar" + "wss://fake.com" + "wss://web.socket/test" +) + +# For each potential secret above, check for presence of string and encoded version in log files +MATCHED_LINES=() +for substr in "${arr[@]}"; do + MATCHES="$(grep -i "$substr" $OUTPUT_FILE || true)" + if [ -n "$MATCHES" ]; then + # check if the matched string is part of a known test case that is safe to pass + for safesubstr in "${testarr[@]}"; do + SAFE_MATCH="$(echo "${MATCHED_LINE}" | grep -i "$safesubstr" || true)" + if [ -n "$SAFE_MATCH" ]; then + MATCHED_LINES+=("$MATCHES") + fi + done + fi +done + +# No matches found in logs, return success +if [ ${#MATCHED_LINES[@]} -eq 0 ]; then + echo "No matches in test log output against known set of sensitive strings" + exit 0 +fi + +# Instances of secret strings were matched, exit the test script with an error +printf "\n" +printf "Sensitive string(s) found in test output logs:\n----------------------------------------------\n" +printf "%s\n" "${MATCHED_LINES[@]}" +printf "\n\n" +printf "❌ The above sensitive string(s) were found in the test harness logs output file - this means that secrets could be leaked/logged to external services when running in production.\n" +exit 1 diff --git a/.github/actions/setup-postgres/docker-compose.yml b/tools/docker/setup-postgres/docker-compose.yml similarity index 52% rename from .github/actions/setup-postgres/docker-compose.yml rename to tools/docker/setup-postgres/docker-compose.yml index ef8e23c..b6509b1 100644 --- a/.github/actions/setup-postgres/docker-compose.yml +++ b/tools/docker/setup-postgres/docker-compose.yml @@ -4,13 +4,13 @@ services: image: 'postgres:15.2-alpine' restart: always environment: - - POSTGRES_USER=postgres - - POSTGRES_PASSWORD=postgres - - POSTGRES_DB=mercury_test + - POSTGRES_USER=feeds + - POSTGRES_PASSWORD=feeds + - POSTGRES_DB=feeds_development ports: - '5432:5432' healthcheck: - test: "pg_isready -U postgres -d mercury_test" + test: 'pg_isready -U feeds -d feeds_development -p 5432' interval: 2s timeout: 5s - retries: 5 + retries: 5 \ No newline at end of file diff --git a/tools/env/ci.env b/tools/env/ci.env new file mode 100644 index 0000000..bcc2255 --- /dev/null +++ b/tools/env/ci.env @@ -0,0 +1 @@ +GOPRIVATE=github.com/smartcontractkit/* \ No newline at end of file