run book commands for overcome DB CPU max utilization

ministryofjustice · Nov 14, 2023 · b64e406 · b64e406
1 parent 48d4a76
commit b64e406
Show file tree

Hide file tree

Showing 8 changed files with 89 additions and 0 deletions.
diff --git a/.idea/.gitignore b/.idea/.gitignore
diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml
diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml
diff --git a/.idea/misc.xml b/.idea/misc.xml
diff --git a/.idea/modules.xml b/.idea/modules.xml
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
diff --git a/hmpps-interventions-docs.iml b/hmpps-interventions-docs.iml
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="WEB_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>
diff --git a/source/runbooks/runbook-commands.html.md.erb b/source/runbooks/runbook-commands.html.md.erb
@@ -0,0 +1,36 @@
+---
+title: Production support commands
+weight: 54
+last_reviewed_on: 2023-11-14
+review_in: 1 year
+---
+
+# When the DB cpu is 100 %, try restarting the pods first
+
+## commands restarting the pods, do it one by one
+
+```
+kubectl rollout restart deployment hmpps-interventions-service-dashboard -n hmpps-interventions-prod
+kubectl rollout restart deployment hmpps-interventions-service-api -n hmpps-interventions-prod
+kubectl rollout restart deployment hmpps-interventions-ui -n hmpps-interventions-prod
+kubectl rollout restart deployment hmpps-interventions-service-performance-report -n hmpps-interventions-prod
+```
+
+## If the above commands does not bring the db cpu down, trying scaling down and scale support up
+
+### commands for scaling down
+
+```
+kubectl scale deployment hmpps-interventions-service-api -n hmpps-interventions-prod --replicas=0
+kubectl scale deployment hmpps-interventions-service-dashboard -n hmpps-interventions-prod --replicas=0
+kubectl scale deployment hmpps-interventions-service-performance-report -n hmpps-interventions-prod --replicas=0
+kubectl scale deployment hmpps-interventions-ui -n hmpps-interventions-prod --replicas=0
+```
+### commands for scaling up
+
+```
+kubectl scale deployment hmpps-interventions-service-api -n hmpps-interventions-prod --replicas=3
+kubectl scale deployment hmpps-interventions-service-dashboard -n hmpps-interventions-prod --replicas=2
+kubectl scale deployment hmpps-interventions-service-performance-report -n hmpps-interventions-prod --replicas=1
+kubectl scale deployment hmpps-interventions-ui -n hmpps-interventions-prod --replicas=3
+```