Merge remote-tracking branch 'origin/HH-241239' into EXP-103876

.env

@@ -6,11 +6,11 @@ SENTRY_EVENT_RETENTION_DAYS=90
 SENTRY_BIND=9000
 # Set SENTRY_MAIL_HOST to a valid FQDN (host/domain name) to be able to send emails!
 # SENTRY_MAIL_HOST=example.com
-SENTRY_IMAGE=getsentry/sentry:24.10.0
-SNUBA_IMAGE=getsentry/snuba:24.10.0
-RELAY_IMAGE=getsentry/relay:24.10.0
-SYMBOLICATOR_IMAGE=getsentry/symbolicator:24.10.0
-VROOM_IMAGE=getsentry/vroom:24.10.0
+SENTRY_IMAGE=getsentry/sentry:24.11.1
+SNUBA_IMAGE=getsentry/snuba:24.11.1
+RELAY_IMAGE=getsentry/relay:24.11.1
+SYMBOLICATOR_IMAGE=getsentry/symbolicator:24.11.1
+VROOM_IMAGE=getsentry/vroom:24.11.1
 HEALTHCHECK_INTERVAL=30s
 HEALTHCHECK_TIMEOUT=1m30s
 HEALTHCHECK_RETRIES=10

.github/workflows/docker-image.yml

@@ -14,6 +14,7 @@ jobs:
       security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
     runs-on: ubuntu-latest
     strategy:
+      max-parallel: 2
       fail-fast: false
       matrix:
         image-name:
@@ -31,27 +32,27 @@ jobs:
           file: 'docker/Dockerfile.${{ matrix.image-name }}'
 
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2
+        uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0
         with:
           image-ref: '${{ matrix.image-name }}-hh:${{ github.sha }}'
           format: sarif
           output: '${{ matrix.image-name }}-trivy-results.sarif'
           severity: 'CRITICAL,HIGH'
 
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v3
+        uses: github/codeql-action/upload-sarif@aa578102511db1f4524ed59b8cc2bae4f6e88195
         with:
           sarif_file: '${{ matrix.image-name }}-trivy-results.sarif'
 
       - name: Run the Anchore Grype scan action
-        uses: anchore/scan-action@f2ba85e044c8f5e5014c9a539328a9c78d3bfa49
+        uses: anchore/scan-action@869c549e657a088dc0441b08ce4fc0ecdac2bb65
         id: scan
         with:
           image: '${{ matrix.image-name }}-hh:${{ github.sha }}'
           fail-build: false
           severity-cutoff: critical
 
       - name: Upload vulnerability report
-        uses: github/codeql-action/upload-sarif@v3
+        uses: github/codeql-action/upload-sarif@aa578102511db1f4524ed59b8cc2bae4f6e88195
         with:
           sarif_file: '${{ steps.scan.outputs.sarif }}'

CHANGELOG.md

@@ -1,5 +1,26 @@
 # Changelog
 
+## 24.11.1
+
+### Various fixes & improvements
+
+- fix(redis): Use a safer eviction rule (#3432) by @BYK
+- feat: add Redis configuration for improved memory management (#3427) by @Hassanzadeh-sd
+- build(deps): bump codecov/codecov-action from 4 to 5 (#3429) by @dependabot
+
+## 24.11.0
+
+### Various fixes & improvements
+
+- feat(healthcheck): Improve redis healthcheck (#3422) by @hubertdeng123
+- fix: missing mime types and turning off autoindex for js-sdk endpoint (#3395) by @aldy505
+- fix: Use js.sentry-cdn.com for JS SDK downloads (#3417) by @BYK
+- fix(loader): provide js sdk assets from 4.x (#3415) by @aldy505
+- Revert "Revert "ref(feedback): remove issue platform flags after releasing issue types"" (#3403) by @BYK
+- Revert "ref(feedback): remove issue platform flags after releasing issue types" (#3402) by @BYK
+- ref(feedback): remove issue platform flags after releasing issue types (#3397) by @aliu39
+- fix(sentry-admin): Do not wait for command finish to display output (#3390) by @Makhonya
+
 ## 24.10.0
 
 ### Various fixes & improvements

CONTRIBUTING.md

@@ -1,7 +1,13 @@
 ## Testing
 
-Validate changes to the setup by running the integration test:
+### Running Tests with Pytest
 
-```shell
-./integration-test.sh
+We use pytest for running tests. To run the tests:
+
+1) Ensure that you are in the root directory of the project.
+2) Run the following command:
+```bash
+pytest
 ```
+
+This will automatically discover and run all test cases in the project.

_unit-test/js-sdk-assets-test.sh

@@ -10,17 +10,23 @@ source install/setup-js-sdk-assets.sh
 
 sdk_files=$(docker compose run --no-deps --rm -v "sentry-nginx-www:/var/www" nginx ls -lah /var/www/js-sdk/)
 sdk_tree=$(docker compose run --no-deps --rm -v "sentry-nginx-www:/var/www" nginx tree /var/www/js-sdk/ | tail -n 1)
+non_empty_file_count=$(docker compose run --no-deps --rm -v "sentry-nginx-www:/var/www" nginx find /var/www/js-sdk/ -type f -size +1k | wc -l)
 
-# `sdk_files` should contains 2 lines, `7.*` and `8.*`
+# `sdk_files` should contains 5 lines, '4.*', '5.*', '6.*', `7.*` and `8.*`
 echo $sdk_files
-total_directories=$(echo "$sdk_files" | grep -c '[78]\.[0-9]*\.[0-9]*$')
+total_directories=$(echo "$sdk_files" | grep -c '[45678]\.[0-9]*\.[0-9]*$')
 echo $total_directories
-test "2" == "$total_directories"
+test "5" == "$total_directories"
 echo "Pass"
 
-# `sdk_tree` should outputs "2 directories, 10 files"
+# `sdk_tree` should output "5 directories, 17 files"
 echo "$sdk_tree"
-test "2 directories, 10 files" == "$(echo "$sdk_tree")"
+test "5 directories, 17 files" == "$(echo "$sdk_tree")"
+echo "Pass"
+
+# Files should all be >1k (ensure they are not empty)
+echo "Testing file sizes"
+test "17" == "$non_empty_file_count"
 echo "Pass"
 
 report_success

docker-compose.yml

@@ -107,9 +107,13 @@ services:
     image: "redis:6.2.14-alpine"
     healthcheck:
       <<: *healthcheck_defaults
-      test: redis-cli ping
+      test: redis-cli ping | grep PONG
     volumes:
       - "sentry-redis:/data"
+      - type: bind
+        read_only: true
+        source: ./redis.conf
+        target: /usr/local/etc/redis/redis.conf
     ulimits:
       nofile:
         soft: 10032
@@ -457,8 +461,8 @@ services:
     volumes:
       - type: bind
         read_only: true
-        source: ./nginx
-        target: /etc/nginx
+        source: ./nginx.conf
+        target: /etc/nginx/nginx.conf
       - sentry-nginx-cache:/var/cache/nginx
       - sentry-nginx-www:/var/www
     depends_on:

docker/Dockerfile.relay

@@ -1,4 +1,4 @@
-ARG SENTRY_VERSION=24.10.0
+ARG SENTRY_VERSION=24.11.1
 ARG RELAY_IMAGE=getsentry/relay:${SENTRY_VERSION}
 ARG APPLICATION=relay-hh=24.10.0-4-3-1-7-1
 FROM ${RELAY_IMAGE}

docker/Dockerfile.sentry

@@ -1,4 +1,4 @@
-ARG SENTRY_VERSION=24.10.0
+ARG SENTRY_VERSION=24.11.1
 ARG SENTRY_IMAGE=getsentry/sentry:${SENTRY_VERSION}
 ARG APPLICATION=sentry-hh=24.10.0-4-3-1-7-1
 FROM ${SENTRY_IMAGE}

docker/Dockerfile.snuba

@@ -1,4 +1,4 @@
-ARG SENTRY_VERSION=24.10.0
+ARG SENTRY_VERSION=24.11.1
 ARG SNUBA_IMAGE=getsentry/snuba:${SENTRY_VERSION}
 ARG APPLICATION=snuba-hh=24.10.0-4-3-1-7-1
 FROM ${SNUBA_IMAGE}

docker/Dockerfile.symbolicator

@@ -1,4 +1,4 @@
-ARG SENTRY_VERSION=24.10.0
+ARG SENTRY_VERSION=24.11.1
 ARG SYMBOLICATOR_IMAGE=getsentry/symbolicator:${SENTRY_VERSION}
 ARG APPLICATION=symbolicator-hh=24.10.0-4-3-1-7-1
 FROM ${SYMBOLICATOR_IMAGE}

docker/Dockerfile.vroom

@@ -1,4 +1,4 @@
-ARG SENTRY_VERSION=24.10.0
+ARG SENTRY_VERSION=24.11.1
 ARG VROOM_IMAGE=getsentry/vroom:${SENTRY_VERSION}
 ARG APPLICATION=vroom-hh=24.10.0-4-3-1-7-1
 FROM ${VROOM_IMAGE}

install/setup-js-sdk-assets.sh

@@ -21,18 +21,20 @@ if [[ "${SETUP_JS_SDK_ASSETS:-}" == "1" ]]; then
   # We want to remove everything before the first '{'.
   loader_registry=$(echo "$loader_registry" | sed '0,/{/s/[^{]*//')
 
+  #  Sentry backend provides SDK versions from v4.x up to v8.x.
+  latest_js_v4=$(echo "$loader_registry" | $jq -r '.versions | reverse | map(select(.|any(.; startswith("4.")))) | .[0]')
+  latest_js_v5=$(echo "$loader_registry" | $jq -r '.versions | reverse | map(select(.|any(.; startswith("5.")))) | .[0]')
+  latest_js_v6=$(echo "$loader_registry" | $jq -r '.versions | reverse | map(select(.|any(.; startswith("6.")))) | .[0]')
   latest_js_v7=$(echo "$loader_registry" | $jq -r '.versions | reverse | map(select(.|any(.; startswith("7.")))) | .[0]')
   latest_js_v8=$(echo "$loader_registry" | $jq -r '.versions | reverse | map(select(.|any(.; startswith("8.")))) | .[0]')
 
-  echo "Found JS SDKs v${latest_js_v7} and v${latest_js_v8}, downloading from upstream.."
+  echo "Found JS SDKs: v${latest_js_v4}, v${latest_js_v5}, v${latest_js_v6}, v${latest_js_v7}, v${latest_js_v8}"
 
-  # Download those two using wget
-  for version in "${latest_js_v7}" "${latest_js_v8}"; do
-    $dcr --no-deps --rm -v "sentry-nginx-www:/var/www" nginx mkdir -p /var/www/js-sdk/${version}
-    for variant in "tracing" "tracing.replay" "replay" "tracing.replay.feedback" "feedback"; do
-      $dcr --no-deps --rm -v "sentry-nginx-www:/var/www" nginx wget -q -O /var/www/js-sdk/${version}/bundle.${variant}.min.js "https://browser.sentry-cdn.com/${version}/bundle.${variant}.min.js"
-    done
-  done
+  versions="{$latest_js_v4,$latest_js_v5,$latest_js_v6,$latest_js_v7,$latest_js_v8}"
+  variants="{bundle,bundle.tracing,bundle.tracing.replay,bundle.replay,bundle.tracing.replay.feedback,bundle.feedback}"
+
+  # Download those versions & variants using curl
+  $dcr --no-deps --rm -v "sentry-nginx-www:/var/www" nginx curl -w '%{response_code} %{url}\n' --no-progress-meter --compressed --retry 3 --create-dirs -fLo "/var/www/js-sdk/#1/#2.min.js" "https://browser.sentry-cdn.com/${versions}/${variants}.min.js" || true
 
   echo "${_endgroup}"
 fi

nginx/nginx.conf → nginx.conf

@@ -11,6 +11,7 @@ events {
 
 
 http {
+	include /etc/nginx/mime.types;
 	default_type application/octet-stream;
 
 	log_format main '$remote_addr - $remote_user [$time_local] "$request" '
@@ -89,8 +90,10 @@ http {
 			proxy_pass http://relay;
 		}
 		location ^~ /js-sdk/ {
-			autoindex on;
-			root  /var/www/js-sdk;
+			root /var/www/;
+			# This value is set to mimic the behavior of the upstream Sentry CDN. For security reasons,
+			# it is recommended to change this to your Sentry URL (in most cases same as system.url-prefix).
+			add_header Access-Control-Allow-Origin *;
 		}
 		location / {
 			proxy_pass http://sentry;

redis.conf

@@ -0,0 +1,27 @@
+# redis.conf
+
+# The 'maxmemory' directive controls the maximum amount of memory Redis is allowed to use.
+# Setting 'maxmemory 0' means there is no limit on memory usage, allowing Redis to use as much
+# memory as the operating system allows. This is suitable for environments where memory
+# constraints are not a concern.
+#
+# Alternatively, you can specify a limit, such as 'maxmemory 15gb', to restrict Redis to
+# using a maximum of 15 gigabytes of memory.
+#
+# Example:
+# maxmemory 0         # Unlimited memory usage
+# maxmemory 15gb     # Limit memory usage to 15 GB
+
+maxmemory 0
+
+# This setting determines how Redis evicts keys when it reaches the memory limit.
+# `allkeys-lru` evicts the least recently used keys from all keys stored in Redis,
+# allowing frequently accessed data to remain in memory while older data is removed.
+# That said we use `volatile-lru` as Redis is used both as a cache and processing
+# queue in self-hosted Sentry.
+# > The volatile-lru and volatile-random policies are mainly useful when you want to
+# > use a single Redis instance for both caching and for a set of persistent keys.
+# > However, you should consider running two separate Redis instances in a case like
+# > this, if possible.
+
+maxmemory-policy volatile-lru

requirements-dev.txt

@@ -5,3 +5,4 @@ pytest-rerunfailures>=11.0
 pytest-sentry>=0.1.11
 httpx>=0.25.2
 beautifulsoup4>=4.7.1
+cryptography>=43.0.3

sentry-admin.sh

@@ -22,8 +22,7 @@ on the host filesystem. Commands that write files should write them to the '/sen
 
 # Actual invocation that runs the command in the container.
 invocation() {
-  output=$($dc run -v "$VOLUME_MAPPING" --rm -T -e SENTRY_LOG_LEVEL=CRITICAL web "$@" 2>&1)
-  echo "$output"
+  $dc run -v "$VOLUME_MAPPING" --rm -T -e SENTRY_LOG_LEVEL=CRITICAL web "$@" 2>&1
 }
 
 # Function to modify lines starting with `Usage: sentry` to say `Usage: ./sentry-admin.sh` instead.

sentry/sentry.conf.example.py

@@ -318,9 +318,6 @@ def get_internal_network():
             "organizations:user-feedback-ingest",
             "organizations:user-feedback-replay-clip",
             "organizations:user-feedback-ui",
-            "organizations:feedback-visible",
-            "organizations:feedback-ingest",
-            "organizations:feedback-post-process-group",
         )
     }
 )

unit-test.sh

@@ -11,8 +11,9 @@ for test_file in _unit-test/*-test.sh; do
   fi
   echo "🙈 Running $test_file ..."
   $test_file
-  if [ $? != 0 ]; then
-    echo fail 👎
+  exit_code=$?
+  if [ $exit_code != 0 ]; then
+    echo fail 👎 with exit code $exit_code
     fail=1
   fi
 done