health error

[arturmon]

Issue submitter TODO list

• I've searched for an already existing issues here

Describe the bug (actual behavior)

spec:
  progressDeadlineSeconds: 600
  replicas: 3
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: sentry
      release: sentry
      role: snuba-outcomes-consumer
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      annotations:
        checksum/config.yaml: 0d19fb638ed987c58f1a992be276c75716d49a2e63e15d98286602fce3230c83
        checksum/snubaSettingsPy: d5f85a6a8afbc55eebe23801e1a51a0fb4c0428c9a73ef6708d8dc83e079cd49
      creationTimestamp: null
      labels:
        app: sentry
        release: sentry
        role: snuba-outcomes-consumer
    spec:
      containers:
      - command:
        - snuba
        - consumer
        - --storage
        - outcomes_raw
        - --consumer-group
        - snuba-consumers
        - --max-batch-size
        - "3"
        - --health-check-file
        - /tmp/health.txt
        env:
        - name: SNUBA_SETTINGS
          value: /etc/snuba/settings.py
        - name: DEFAULT_BROKERS
          value: sentry-kafka:9092
        - name: KAFKA_SECURITY_PROTOCOL
          value: PLAINTEXT
        - name: CLICKHOUSE_MAX_CONNECTIONS
          value: "100"
        - name: REDIS_PORT
          value: "6379"
        envFrom:
        - secretRef:
            name: sentry-snuba-env
        image: getsentry/snuba:24.9.0
        imagePullPolicy: IfNotPresent
        livenessProbe:
          exec:
            command:
            - rm
            - /tmp/health.txt
          failureThreshold: 3
          initialDelaySeconds: 30
          periodSeconds: 320
          successThreshold: 1
          timeoutSeconds: 1
        name: sentry-snuba
        ports:
        - containerPort: 1218
          protocol: TCP
        resources: {}
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /etc/snuba
          name: config
          readOnly: true
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      terminationGracePeriodSeconds: 30
      volumes:
      - configMap:
          defaultMode: 420
          name: sentry-snuba
        name: config

sentry-snuba-subscription-consumer-events

spec:
  progressDeadlineSeconds: 600
  replicas: 3
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: sentry
      release: sentry
      role: snuba-subscription-consumer-events
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      annotations:
        checksum/config.yaml: 0d19fb638ed987c58f1a992be276c75716d49a2e63e15d98286602fce3230c83
        checksum/snubaSettingsPy: d5f85a6a8afbc55eebe23801e1a51a0fb4c0428c9a73ef6708d8dc83e079cd49
      creationTimestamp: null
      labels:
        app: sentry
        release: sentry
        role: snuba-subscription-consumer-events
    spec:
      containers:
      - command:
        - snuba
        - subscriptions-scheduler-executor
        - --dataset=events
        - --entity=events
        - --consumer-group=snuba-events-subscriptions-consumers
        - --followed-consumer-group=snuba-consumers
        - --schedule-ttl=60
        - --stale-threshold-seconds=900
        - --health-check-file
        - /tmp/health.txt
        env:
        - name: SNUBA_SETTINGS
          value: /etc/snuba/settings.py
        - name: DEFAULT_BROKERS
          value: sentry-kafka:9092
        - name: KAFKA_SECURITY_PROTOCOL
          value: PLAINTEXT
        - name: CLICKHOUSE_MAX_CONNECTIONS
          value: "100"
        - name: REDIS_PORT
          value: "6379"
        envFrom:
        - secretRef:
            name: sentry-snuba-env
        image: getsentry/snuba:24.9.0
        imagePullPolicy: IfNotPresent
        livenessProbe:
          exec:
            command:
            - rm
            - /tmp/health.txt
          failureThreshold: 3
          initialDelaySeconds: 5
          periodSeconds: 320
          successThreshold: 1
          timeoutSeconds: 1
        name: sentry-snuba
        ports:
        - containerPort: 1218
          protocol: TCP
        resources: {}
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /etc/snuba
          name: config
          readOnly: true
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      terminationGracePeriodSeconds: 30
      volumes:
      - configMap:
          defaultMode: 420
          name: sentry-snuba
        name: config

sentry-snuba-subscription-consumer-transactions

spec:
  progressDeadlineSeconds: 600
  replicas: 3
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: sentry
      release: sentry
      role: snuba-subscription-consumer-transactions
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      annotations:
        checksum/config.yaml: 0d19fb638ed987c58f1a992be276c75716d49a2e63e15d98286602fce3230c83
        checksum/snubaSettingsPy: d5f85a6a8afbc55eebe23801e1a51a0fb4c0428c9a73ef6708d8dc83e079cd49
      creationTimestamp: null
      labels:
        app: sentry
        release: sentry
        role: snuba-subscription-consumer-transactions
    spec:
      containers:
      - command:
        - snuba
        - subscriptions-scheduler-executor
        - --dataset=transactions
        - --entity=transactions
        - --consumer-group=snuba-transactions-subscriptions-consumers
        - --followed-consumer-group=transactions_group
        - --schedule-ttl=60
        - --stale-threshold-seconds=900
        - --health-check-file
        - /tmp/health.txt
        env:
        - name: SNUBA_SETTINGS
          value: /etc/snuba/settings.py
        - name: DEFAULT_BROKERS
          value: sentry-kafka:9092
        - name: KAFKA_SECURITY_PROTOCOL
          value: PLAINTEXT
        - name: CLICKHOUSE_MAX_CONNECTIONS
          value: "100"
        - name: REDIS_PORT
          value: "6379"
        envFrom:
        - secretRef:
            name: sentry-snuba-env
        image: getsentry/snuba:24.9.0
        imagePullPolicy: IfNotPresent
        livenessProbe:
          exec:
            command:
            - rm
            - /tmp/health.txt
          failureThreshold: 3
          initialDelaySeconds: 5
          periodSeconds: 320
          successThreshold: 1
          timeoutSeconds: 1
        name: sentry-snuba
        ports:
        - containerPort: 1218
          protocol: TCP
        resources: {}
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /etc/snuba
          name: config
          readOnly: true
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      terminationGracePeriodSeconds: 30
      volumes:
      - configMap:
          defaultMode: 420
          name: sentry-snuba
        name: config

with these settings I get an error sentry-snuba-outcomes-consumer-7996565d6c-8qv8g: Liveness probe failed: rm: cannot remove '/tmp/health.txt': No such file or directory

the same with 4 more components sentry-snuba-subscription-consumer-transactions, sentry-subscription-consumer-events, sentry-subscription-consumer-transactions and sentry-snuba-subscription-consumer-events

Expected behavior

These errors leading to a pod reboot should not occur.

values.yaml

snuba:
...
subscriptionConsumerEvents:
replicas: 3
subscriptionConsumerTransactions:
replicas: 3
transactionsConsumer:
replicas: 3

Helm chart version

26.5.0

Steps to reproduce

helmfile apply -f helmfile.yaml

Screenshots

Logs

Additional context

No response