From 8876428191a493bfae22fb05286999f2d9d38b79 Mon Sep 17 00:00:00 2001 From: EarthlingDavey <15802017+EarthlingDavey@users.noreply.github.com> Date: Thu, 21 Nov 2024 16:22:01 +0000 Subject: [PATCH 1/3] CDPT-2264 Revert isolated cron deployments PR #773 and #772 --- deploy/development/deployment.tpl.yml | 161 -------------------------- deploy/production/deployment.tpl.yml | 161 -------------------------- 2 files changed, 322 deletions(-) diff --git a/deploy/development/deployment.tpl.yml b/deploy/development/deployment.tpl.yml index 56b2e465d..0c72976a6 100644 --- a/deploy/development/deployment.tpl.yml +++ b/deploy/development/deployment.tpl.yml @@ -62,167 +62,6 @@ spec: name: ${KUBE_NAMESPACE}-base64-secrets key: IPS_FORMATTED - - name: fpm - image: ${ECR_URL}:${IMAGE_TAG_FPM} - resources: - limits: - # If a pod exceeds its CPU limit, Kubernetes will simply throttle the pod. - cpu: "4" - # If a pod exceeds its memory limit, Kubernetes will kill the pod. - memory: 2000Mi - requests: - cpu: 500m - memory: 600Mi - volumeMounts: - - name: uploads - mountPath: /var/www/html/public/app/uploads - - name: php-socket - mountPath: /sock - securityContext: - runAsUser: 101 - # Check frequently during startup, so that scaling up can happen as fast as possible. - startupProbe: - exec: - command: - - /usr/local/bin/fpm-health/fpm-readiness.sh - failureThreshold: 20 - periodSeconds: 5 - # Don't route traffic to this pod if the container is not ready. - readinessProbe: - exec: - command: - - /usr/local/bin/fpm-health/fpm-readiness.sh - periodSeconds: 10 - failureThreshold: 1 - # Restart the container if it fails liveness script. - livenessProbe: - exec: - command: - - /usr/local/bin/fpm-health/fpm-liveness.sh - periodSeconds: 10 - env: - - name: AWS_S3_BUCKET - valueFrom: - secretKeyRef: - name: s3-bucket-output - key: bucket_name - - name: AWS_CLOUDFRONT_PUBLIC_KEYS_OBJECT - valueFrom: - secretKeyRef: - name: cloudfront-output - key: cloudfront_public_keys - - name: DB_HOST - valueFrom: - secretKeyRef: - name: rds-output - key: rds_instance_address - - name: DB_NAME - valueFrom: - secretKeyRef: - name: rds-output - key: database_name - - name: DB_USER - valueFrom: - secretKeyRef: - name: rds-output - key: database_username - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: rds-output - key: database_password - - name: OPENSEARCH_URL - valueFrom: - secretKeyRef: - name: central-digital-product-team-opensearch-proxy-url - key: proxy_url - - name: BASIC_AUTH - valueFrom: - secretKeyRef: - name: basic-auth-secret - key: auth - - name: CACHE_HOST - valueFrom: - secretKeyRef: - name: elasticache-output - key: primary_endpoint_address - - name: CACHE_PASSWORD - valueFrom: - secretKeyRef: - name: elasticache-output - key: auth_token - envFrom: - - configMapRef: - name: ${KUBE_NAMESPACE} - - secretRef: - name: ${KUBE_NAMESPACE}-secrets - - secretRef: - name: ${KUBE_NAMESPACE}-base64-secrets ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ${KUBE_NAMESPACE}-cron - namespace: ${KUBE_NAMESPACE} - labels: - app: ${KUBE_NAMESPACE}-cron -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 0 - maxSurge: 100% - selector: - matchLabels: - app: ${KUBE_NAMESPACE}-cron - template: - metadata: - labels: - app: ${KUBE_NAMESPACE}-cron - spec: - volumes: - - name: uploads - emptyDir: { } - - name: php-socket - emptyDir: { } - terminationGracePeriodSeconds: 35 - serviceAccountName: ${KUBE_NAMESPACE}-service - containers: - - name: nginx - image: ${ECR_URL}:${IMAGE_TAG_NGINX} - resources: - limits: - cpu: 500m - memory: 250Mi - requests: - cpu: 50m - memory: 100Mi - ports: - - containerPort: 8080 - name: http - volumeMounts: - - name: uploads - mountPath: /var/www/html/public/app/uploads - - name: php-socket - mountPath: /sock - securityContext: - runAsUser: 101 - readinessProbe: - httpGet: - path: /readiness - port: 8080 - livenessProbe: - httpGet: - path: /liveness - port: 8080 - env: - - name: IPS_FORMATTED - valueFrom: - secretKeyRef: - name: ${KUBE_NAMESPACE}-base64-secrets - key: IPS_FORMATTED - - name: cron image: ${ECR_URL}:${IMAGE_TAG_CRON} resources: diff --git a/deploy/production/deployment.tpl.yml b/deploy/production/deployment.tpl.yml index 6960c3817..9b2266ff7 100644 --- a/deploy/production/deployment.tpl.yml +++ b/deploy/production/deployment.tpl.yml @@ -62,167 +62,6 @@ spec: name: ${KUBE_NAMESPACE}-base64-secrets key: IPS_FORMATTED - - name: fpm - image: ${ECR_URL}:${IMAGE_TAG_FPM} - resources: - limits: - # If a pod exceeds its CPU limit, Kubernetes will simply throttle the pod. - cpu: "4" - # If a pod exceeds its memory limit, Kubernetes will kill the pod. - memory: 2000Mi - requests: - cpu: 500m - memory: 600Mi - volumeMounts: - - name: uploads - mountPath: /var/www/html/public/app/uploads - - name: php-socket - mountPath: /sock - securityContext: - runAsUser: 101 - # Check frequently during startup, so that scaling up can happen as fast as possible. - startupProbe: - exec: - command: - - /usr/local/bin/fpm-health/fpm-readiness.sh - failureThreshold: 20 - periodSeconds: 5 - # Don't route traffic to this pod if the container is not ready. - readinessProbe: - exec: - command: - - /usr/local/bin/fpm-health/fpm-readiness.sh - periodSeconds: 10 - failureThreshold: 1 - # Restart the container if it fails liveness script. - livenessProbe: - exec: - command: - - /usr/local/bin/fpm-health/fpm-liveness.sh - periodSeconds: 10 - env: - - name: AWS_S3_BUCKET - valueFrom: - secretKeyRef: - name: s3-bucket-output - key: bucket_name - - name: AWS_CLOUDFRONT_PUBLIC_KEYS_OBJECT - valueFrom: - secretKeyRef: - name: cloudfront-output - key: cloudfront_public_keys - - name: DB_HOST - valueFrom: - secretKeyRef: - name: rds-output - key: rds_instance_address - - name: DB_NAME - valueFrom: - secretKeyRef: - name: rds-output - key: database_name - - name: DB_USER - valueFrom: - secretKeyRef: - name: rds-output - key: database_username - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: rds-output - key: database_password - - name: OPENSEARCH_URL - valueFrom: - secretKeyRef: - name: opensearch-output - key: proxy_url - - name: BASIC_AUTH - valueFrom: - secretKeyRef: - name: basic-auth-secret - key: auth - - name: CACHE_HOST - valueFrom: - secretKeyRef: - name: elasticache-output - key: primary_endpoint_address - - name: CACHE_PASSWORD - valueFrom: - secretKeyRef: - name: elasticache-output - key: auth_token - envFrom: - - configMapRef: - name: ${KUBE_NAMESPACE} - - secretRef: - name: ${KUBE_NAMESPACE}-secrets - - secretRef: - name: ${KUBE_NAMESPACE}-base64-secrets ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: ${KUBE_NAMESPACE}-cron - namespace: ${KUBE_NAMESPACE} - labels: - app: ${KUBE_NAMESPACE}-cron -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 0 - maxSurge: 100% - selector: - matchLabels: - app: ${KUBE_NAMESPACE}-cron - template: - metadata: - labels: - app: ${KUBE_NAMESPACE}-cron - spec: - volumes: - - name: uploads - emptyDir: { } - - name: php-socket - emptyDir: { } - terminationGracePeriodSeconds: 35 - serviceAccountName: ${KUBE_NAMESPACE}-service - containers: - - name: nginx - image: ${ECR_URL}:${IMAGE_TAG_NGINX} - resources: - limits: - cpu: 500m - memory: 250Mi - requests: - cpu: 50m - memory: 100Mi - ports: - - containerPort: 8080 - name: http - volumeMounts: - - name: uploads - mountPath: /var/www/html/public/app/uploads - - name: php-socket - mountPath: /sock - securityContext: - runAsUser: 101 - readinessProbe: - httpGet: - path: /readiness - port: 8080 - livenessProbe: - httpGet: - path: /liveness - port: 8080 - env: - - name: IPS_FORMATTED - valueFrom: - secretKeyRef: - name: ${KUBE_NAMESPACE}-base64-secrets - key: IPS_FORMATTED - - name: cron image: ${ECR_URL}:${IMAGE_TAG_CRON} resources: From dbde2bd2238036577f70260375e4e3817d6d0568 Mon Sep 17 00:00:00 2001 From: EarthlingDavey <15802017+EarthlingDavey@users.noreply.github.com> Date: Thu, 21 Nov 2024 16:38:34 +0000 Subject: [PATCH 2/3] Lower min replicas to see if 5 can handle usual 8-9 AM traffic. --- deploy/production/hpa.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/production/hpa.yml b/deploy/production/hpa.yml index 002a97051..d49f0644e 100644 --- a/deploy/production/hpa.yml +++ b/deploy/production/hpa.yml @@ -8,7 +8,7 @@ spec: apiVersion: apps/v1 kind: Deployment name: intranet-production - minReplicas: 6 + minReplicas: 5 maxReplicas: 18 metrics: # fpm From 2bfa01a3804208e3fc644e4698434cb466b23b74 Mon Sep 17 00:00:00 2001 From: EarthlingDavey <15802017+EarthlingDavey@users.noreply.github.com> Date: Thu, 21 Nov 2024 16:49:28 +0000 Subject: [PATCH 3/3] Remove CDPT_Debug logging --- public/app/themes/clarity/functions.php | 54 ------------------------- 1 file changed, 54 deletions(-) diff --git a/public/app/themes/clarity/functions.php b/public/app/themes/clarity/functions.php index 6bf51e58b..969eecf0b 100644 --- a/public/app/themes/clarity/functions.php +++ b/public/app/themes/clarity/functions.php @@ -125,57 +125,3 @@ $search = new MOJ\Intranet\Search(); $search->hooks(); - -/** - * Add logging on potential causes of high CPU usage - */ - -// Log on document_serve -add_action('serve_document', function ( $post_id, $file ) { - error_log('CDPT_Debug: document_serve: ' . $post_id . ' ' . $file); -}, 10, 2); - -// Log on document_serve_done -add_action('document_serve_done', function ($file, $attach_id ) { - error_log('CDPT_Debug: document_serve_done: ' . $file . ' ' . $attach_id); -}, 10, 2); - -// Log on document edit -add_action('document_edit', function () { - error_log('CDPT_Debug: document_edit'); -}); - -// Log on document saved -add_action('document_saved', function ($doc_id, $attach_id) { - error_log('CDPT_Debug: document_saved: ' . $doc_id . ' ' . $attach_id); -}, 10, 2); - -// Log on attachment upload -add_action('add_attachment', function ($attach_id) { - error_log('CDPT_Debug: add_attachment: ' . $attach_id); -}); - -// Log on edit post -add_action('edit_post', function ($post_id) { - error_log('CDPT_Debug: edit_post: ' . $post_id); -}); - -// Log on post save -add_action('save_post', function ($post_id) { - error_log('CDPT_Debug: save_post: ' . $post_id); -}); - -// Log on S3 upload - as3cf_post_upload_attachment -add_action('as3cf_post_upload_attachment', function ($source_id, $item) { - error_log('CDPT_Debug: as3cf_post_upload_attachment: ' . $source_id); -}, 10, 2); - -// Log on as3cf_pre_upload_object -add_action('as3cf_pre_upload_object', function ($args, $file) { - error_log('CDPT_Debug: as3cf_pre_upload_object'); -}, 10, 2); - -// Log on as3cf_post_upload_item -add_action('as3cf_post_upload_item', function ($item) { - error_log('CDPT_Debug: as3cf_post_upload_item'); -});